diff --git a/README.md b/README.md index 839c2c05..70ca04d1 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,10 @@ Examples are built by default into `build/bin` and are prefixed with `nvbench.ex Example output from `nvbench.example.throughput` ``` +# Command Line + +./bin/nvbench.example.throughput + # Devices ## [0] `Quadro GV100` diff --git a/nvbench/json_printer.cu b/nvbench/json_printer.cu index e06f25c9..bc951c4f 100644 --- a/nvbench/json_printer.cu +++ b/nvbench/json_printer.cu @@ -381,7 +381,10 @@ void json_printer::do_process_bulk_data_float64(state &state, } // end hint == sample_freqs } -static void add_devices_section(nlohmann::ordered_json &root) +namespace +{ + +void add_devices_section(nlohmann::ordered_json &root) { auto &devices = root["devices"]; for (const auto &dev_info : nvbench::device_manager::get().get_devices()) @@ -409,6 +412,19 @@ static void add_devices_section(nlohmann::ordered_json &root) } } +void add_argv_array(nlohmann::ordered_json &metadata, + const char *key, + const std::vector &args) +{ + auto &argv = metadata[key]; + for (const auto &arg : args) + { + argv.push_back(arg); + } +} + +} // namespace + void json_printer::do_print_benchmark_results(const benchmark_vector &benches) { nlohmann::ordered_json root; @@ -416,13 +432,8 @@ void json_printer::do_print_benchmark_results(const benchmark_vector &benches) { auto &metadata = root["meta"]; - { - auto &argv = metadata["argv"]; - for (const auto &arg : m_argv) - { - argv.push_back(arg); - } - } // "argv" + add_argv_array(metadata, "argv", this->get_argv()); + add_argv_array(metadata, "raw_argv", this->get_raw_argv()); { auto &version = metadata["version"]; diff --git a/nvbench/json_printer.cuh b/nvbench/json_printer.cuh index 5b3637e7..a8b1ce97 100644 --- a/nvbench/json_printer.cuh +++ b/nvbench/json_printer.cuh @@ -76,7 +76,6 @@ struct json_printer : nvbench::printer_base protected: // Virtual API from printer_base: - void do_log_argv(const std::vector &argv) override { m_argv = argv; } void do_process_bulk_data_float64(nvbench::state &state, const std::string &tag, const std::string &hint, @@ -87,8 +86,6 @@ protected: bool m_enable_binary_output{false}; std::size_t m_num_jsonbin_files{}; std::size_t m_num_jsonbin_freq_files{}; - - std::vector m_argv; }; } // namespace nvbench diff --git a/nvbench/main.cuh b/nvbench/main.cuh index 24029866..ab60a886 100644 --- a/nvbench/main.cuh +++ b/nvbench/main.cuh @@ -39,6 +39,7 @@ #include #include #include +#include #include // Advanced users can rebuild NVBench's `main` function using the macros in this file, or replace @@ -57,12 +58,16 @@ // Customization point, called before NVBench parsing. Update argc/argv if needed. // argc/argv are the usual command line arguments types. The ARGS version of this // macro is a bit more convenient. +// NVBench captures raw argv before this handler runs. Changes made here do not +// alter the raw argv that printers may report separately from the parsed args. #ifndef NVBENCH_MAIN_CUSTOM_ARGC_ARGV_HANDLER #define NVBENCH_MAIN_CUSTOM_ARGC_ARGV_HANDLER(argc, argv) []() {}() #endif // Customization point, called before NVBench parsing. Update args if needed. // Args is a vector of strings, each element is an argument. +// NVBench captures raw argv before this handler runs. Changes made here do not +// alter the raw argv that printers may report separately from the parsed args. #ifndef NVBENCH_MAIN_CUSTOM_ARGS_HANDLER #define NVBENCH_MAIN_CUSTOM_ARGS_HANDLER(args) []() {}() #endif @@ -132,10 +137,12 @@ #ifndef NVBENCH_MAIN_PARSE #define NVBENCH_MAIN_PARSE(argc, argv) \ + std::vector raw_args = nvbench::detail::main_convert_args(argc, argv); \ NVBENCH_MAIN_CUSTOM_ARGC_ARGV_HANDLER(argc, argv); \ std::vector args = nvbench::detail::main_convert_args(argc, argv); \ NVBENCH_MAIN_CUSTOM_ARGS_HANDLER(args); \ nvbench::option_parser parser; \ + parser.set_raw_args(std::move(raw_args)); \ NVBENCH_MAIN_PARSE_CUSTOM_PRE(parser, args); \ parser.parse(args); \ NVBENCH_MAIN_PARSE_CUSTOM_POST(parser) @@ -209,6 +216,7 @@ inline void main_print_preamble(option_parser &parser) { auto &printer = parser.get_printer(); + printer.print_argv(); printer.print_device_info(); printer.print_log_preamble(); } diff --git a/nvbench/markdown_printer.cu b/nvbench/markdown_printer.cu index f80b17b2..b3c3381b 100644 --- a/nvbench/markdown_printer.cu +++ b/nvbench/markdown_printer.cu @@ -41,6 +41,146 @@ namespace nvbench { +namespace +{ + +#ifdef _WIN32 + +// Quote an argument with the backslash/double-quote rules used by +// CommandLineToArgvW. This is not a full cmd.exe escaping layer; cmd.exe +// expansion rules may still apply when pasted into a shell. +std::string shell_quote(const std::string &arg) +{ + if (!arg.empty() && arg.find_first_of(" \t\n\v\"^&|<>()%!") == std::string::npos) + { + return arg; + } + + // Follow the rules of CommandLineToArgvW: a run of backslashes is only special + // when a double quote comes after it. + std::string result; + + result.reserve((4 * arg.size()) + 2); + result += '"'; + for (auto iter = arg.begin(); iter != arg.end(); ++iter) + { + std::size_t num_backslashes = 0; + while (iter != arg.end() && *iter == '\\') + { + ++num_backslashes; + ++iter; + } + + if (iter == arg.end()) + { // Double the backslashes that come before the closing quote. + result.append(num_backslashes * 2, '\\'); + break; + } + + if (*iter == '"') + { // Double the backslashes that come before a quote, then escape the quote. + result.append(num_backslashes * 2, '\\'); + result += "\\\""; + } + else + { + result.append(num_backslashes, '\\'); + result += *iter; + } + } + result += '"'; + return result; +} + +#else + +// Quote an argument for POSIX shells (sh, bash, zsh), so that the printed +// command line can be copied and pasted. +std::string shell_quote(const std::string &arg) +{ + if (!arg.empty() && arg.find_first_of(" \t\n\"'\\$`|&;<>()*?[]{}#~!") == std::string::npos) + { + return arg; + } + + std::string result; + + result.reserve((4 * arg.size()) + 2); + result += '\''; + for (const char c : arg) + { + if (c == '\'') + { // A single quote cannot appear inside single quotes; close, escape, reopen. + result += "'\\''"; + } + else + { + result += c; + } + } + result += '\''; + return result; +} + +#endif // _WIN32 + +std::size_t max_backtick_run(const std::string &str) +{ + std::size_t max_run{}; + std::size_t current_run{}; + for (const char c : str) + { + if (c == '`') + { + ++current_run; + max_run = current_run > max_run ? current_run : max_run; + } + else + { + current_run = 0; + } + } + + return max_run; +} + +std::string markdown_code_fence(const std::string &contents) +{ + const auto fence_size = max_backtick_run(contents) + 1; + return std::string(fence_size < 3 ? 3 : fence_size, '`'); +} + +} // namespace + +void markdown_printer::do_print_argv() +{ + const auto &argv = this->get_raw_argv(); + if (argv.empty()) + { + return; + } + + std::string command_line; + for (std::size_t i = 0; i < argv.size(); ++i) + { + if (i != 0) + { + command_line += ' '; + } + command_line += shell_quote(argv[i]); + } + + const auto fence = markdown_code_fence(command_line); + + fmt::memory_buffer buffer; + fmt::format_to(fmt::appender(buffer), + "# Command Line\n\n{}\n{}\n{}\n\n", + fence, + command_line, + fence); + m_ostream << fmt::to_string(buffer); +} + void markdown_printer::do_print_device_info() { fmt::memory_buffer buffer; diff --git a/nvbench/markdown_printer.cuh b/nvbench/markdown_printer.cuh index 73c38c60..078b9d78 100644 --- a/nvbench/markdown_printer.cuh +++ b/nvbench/markdown_printer.cuh @@ -31,6 +31,7 @@ #include #include +#include namespace nvbench { @@ -62,6 +63,7 @@ struct markdown_printer : nvbench::printer_base protected: // Virtual API from printer_base: + void do_print_argv() override; void do_print_device_info() override; void do_print_log_preamble() override; void do_print_log_epilogue() override; diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index 4f865d7f..762f4b14 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -496,6 +496,7 @@ void option_parser::parse_impl() this->update_used_device_state(); m_printer.log_argv(m_args); + m_printer.log_raw_argv(this->get_raw_args()); } void option_parser::parse_range(option_parser::arg_iterator_t first, diff --git a/nvbench/option_parser.cuh b/nvbench/option_parser.cuh index 9a8ee405..f6dfe9aa 100644 --- a/nvbench/option_parser.cuh +++ b/nvbench/option_parser.cuh @@ -37,6 +37,7 @@ #include #include #include +#include #include namespace nvbench @@ -62,11 +63,30 @@ struct option_parser void parse(int argc, char const *const argv[]); void parse(std::vector args); + /*! + * Set the command line that invoked the executable, before any modification. + * + * Call this before `parse`. `parse` sends these args to the printers as raw + * args alongside the args that NVBench parsed. + */ + void set_raw_args(std::vector raw_args) { m_raw_args = std::move(raw_args); } + [[nodiscard]] benchmark_vector &get_benchmarks() { return m_benchmarks; }; [[nodiscard]] const benchmark_vector &get_benchmarks() const { return m_benchmarks; }; + /*! + * The args given to `parse` after customization handlers have modified them. + */ [[nodiscard]] const std::vector &get_args() const { return m_args; } + /*! + * The args given to `set_raw_args`, or `get_args` if it was not called. + */ + [[nodiscard]] const std::vector &get_raw_args() const + { + return m_raw_args ? *m_raw_args : m_args; + } + /*! * Returns the output format requested by the parse options. * @@ -141,6 +161,9 @@ private: // Command line args std::vector m_args; + // The unmodified command line, if the caller supplied one. + std::optional> m_raw_args; + // Store benchmark modifiers passed in before any benchmarks are requested as // "global args". Replay them after every benchmark. std::vector m_global_benchmark_args; diff --git a/nvbench/printer_base.cuh b/nvbench/printer_base.cuh index 2347a2f3..57975d6a 100644 --- a/nvbench/printer_base.cuh +++ b/nvbench/printer_base.cuh @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -93,11 +94,33 @@ struct printer_base printer_base &operator=(const printer_base &) = delete; printer_base &operator=(printer_base &&) = delete; + /*! + * Called once with the command line arguments that NVBench parsed. + */ + void log_argv(const std::vector &argv) + { + m_argv = argv; + this->do_log_argv(argv); + } + /*! * Called once with the command line arguments used to invoke the current - * executable. + * executable, before any customization handlers modify them. */ - void log_argv(const std::vector &argv) { this->do_log_argv(argv); } + void log_raw_argv(const std::vector &argv) + { + m_raw_argv = argv; + this->do_log_raw_argv(argv); + } + + /*! + * Print the command line used to invoke the current executable, if supported. + * + * Called before running benchmarks for active terminal output. Must be called + * after `log_argv`. If `log_raw_argv` has been called, printers can use the + * raw command line. + */ + void print_argv() { this->do_print_argv(); } /*! * Print a summary of all detected devices, if supported. @@ -192,8 +215,16 @@ struct printer_base /*!@}*/ protected: + [[nodiscard]] const std::vector &get_argv() const { return m_argv; } + [[nodiscard]] const std::vector &get_raw_argv() const + { + return m_raw_argv ? *m_raw_argv : m_argv; + } + // Implementation hooks for subclasses: virtual void do_log_argv(const std::vector &) {} + virtual void do_log_raw_argv(const std::vector &) {} + virtual void do_print_argv() {} virtual void do_print_device_info() {} virtual void do_print_log_preamble() {} virtual void do_print_log_epilogue() {} @@ -226,6 +257,10 @@ protected: std::size_t m_completed_state_count{}; std::size_t m_total_state_count{}; + +private: + std::vector m_argv; + std::optional> m_raw_argv; }; } // namespace nvbench diff --git a/nvbench/printer_multiplex.cuh b/nvbench/printer_multiplex.cuh index b9090864..c4bd8512 100644 --- a/nvbench/printer_multiplex.cuh +++ b/nvbench/printer_multiplex.cuh @@ -57,6 +57,8 @@ struct printer_multiplex : nvbench::printer_base protected: void do_log_argv(const std::vector &argv) override; + void do_log_raw_argv(const std::vector &argv) override; + void do_print_argv() override; void do_print_device_info() override; void do_print_log_preamble() override; void do_print_log_epilogue() override; diff --git a/nvbench/printer_multiplex.cxx b/nvbench/printer_multiplex.cxx index 5cede21d..0cd4f3ae 100644 --- a/nvbench/printer_multiplex.cxx +++ b/nvbench/printer_multiplex.cxx @@ -30,6 +30,14 @@ printer_multiplex::printer_multiplex() : printer_base(std::cerr) // Nothing should write to this. {} +void printer_multiplex::do_print_argv() +{ + for (auto &format_ptr : m_printers) + { + format_ptr->print_argv(); + } +} + void printer_multiplex::do_print_device_info() { for (auto &format_ptr : m_printers) @@ -124,11 +132,18 @@ void printer_multiplex::do_set_total_state_count(std::size_t states) } void printer_multiplex::do_log_argv(const std::vector &argv) { - printer_base::do_log_argv(argv); for (auto &format_ptr : m_printers) { format_ptr->log_argv(argv); } } +void printer_multiplex::do_log_raw_argv(const std::vector &argv) +{ + for (auto &format_ptr : m_printers) + { + format_ptr->log_raw_argv(argv); + } +} + } // namespace nvbench diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 79f59ddf..5eb5ca62 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -11,12 +11,14 @@ set(test_srcs custom_main_custom_args.cu custom_main_custom_exceptions.cu custom_main_global_state_raii.cu + custom_main_raw_argv.cu enum_type_list.cu entropy_criterion.cu exception_safety.cu float64_axis.cu int64_axis.cu json_printer.cu + markdown_printer.cu measure_timeout_warnings.cu named_values.cu option_parser.cu @@ -38,6 +40,8 @@ set(test_srcs # CTest commands+args can't be modified after creation, so we need to rely on substitution. set(NVBench_TEST_ARGS_nvbench.test.custom_main_custom_args "--quiet" "--my-custom-arg" "--profile" "-d" "0") set(NVBench_TEST_ARGS_nvbench.test.custom_main_custom_exceptions "--quiet" "--profile" "-d" "0") +set(NVBench_TEST_ARGS_nvbench.test.custom_main_raw_argv + "--my-custom-arg" "-d" "0" "--json" "custom_main_raw_argv.json") # Metatarget for all tests: add_custom_target(nvbench.test.all) @@ -60,6 +64,10 @@ endforeach() set_tests_properties(nvbench.test.custom_main_custom_exceptions PROPERTIES PASS_REGULAR_EXPRESSION "Custom error detected: Expected exception thrown." ) +set_tests_properties(nvbench.test.custom_main_raw_argv PROPERTIES + PASS_REGULAR_EXPRESSION "custom_main_raw_argv --my-custom-arg -d 0" + FAIL_REGULAR_EXPRESSION "custom_main_raw_argv --profile" +) set_tests_properties(nvbench.test.exception_safety PROPERTIES TIMEOUT 20) add_subdirectory(cmake) diff --git a/testing/custom_main_raw_argv.cu b/testing/custom_main_raw_argv.cu new file mode 100644 index 00000000..f7ca91f5 --- /dev/null +++ b/testing/custom_main_raw_argv.cu @@ -0,0 +1,131 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +const char *json_output_path = "custom_main_raw_argv.json"; + +std::string read_json_output() +{ + std::ifstream stream{json_output_path}; + if (!stream) + { + throw std::runtime_error("JSON output was not written."); + } + + std::ostringstream buffer; + buffer << stream.rdbuf(); + return buffer.str(); +} + +void verify_json_array(const std::string &json, + const char *key, + const char *expected, + const char *unexpected) +{ + const auto key_pos = json.find(key); + if (key_pos == std::string::npos) + { + throw std::runtime_error(std::string{"JSON output is missing "} + key + "."); + } + + const auto array_end = json.find("]", key_pos); + if (array_end == std::string::npos) + { + throw std::runtime_error(std::string{"JSON output has an unterminated "} + key + " array."); + } + + const auto array = json.substr(key_pos, array_end - key_pos); + if (array.find(expected) == std::string::npos) + { + throw std::runtime_error(std::string{"JSON "} + key + " lost its expected argument."); + } + if (array.find(unexpected) != std::string::npos) + { + throw std::runtime_error(std::string{"JSON "} + key + " contains the wrong argument."); + } +} + +} // namespace + +// Rewrite "--my-custom-arg" into "--profile". The printed command line must keep +// the original argument. +void custom_arg_handler(std::vector &args) +{ + auto it = std::find(args.begin(), args.end(), "--my-custom-arg"); + if (it == args.end()) + { + throw std::runtime_error("Custom argument not found."); + } + *it = "--profile"; +} + +#undef NVBENCH_MAIN_CUSTOM_ARGS_HANDLER +#define NVBENCH_MAIN_CUSTOM_ARGS_HANDLER(args) custom_arg_handler(args) + +void verify(nvbench::option_parser &parser) +{ + const auto &raw = parser.get_raw_args(); + const auto &parsed = parser.get_args(); + + if (std::find(raw.begin(), raw.end(), "--my-custom-arg") == raw.end()) + { + throw std::runtime_error("Raw args lost the original argument."); + } + if (std::find(raw.begin(), raw.end(), "--profile") != raw.end()) + { + throw std::runtime_error("Raw args contain the rewritten argument."); + } + if (std::find(parsed.begin(), parsed.end(), "--profile") == parsed.end()) + { + throw std::runtime_error("Parsed args lost the rewritten argument."); + } +} + +#undef NVBENCH_MAIN_PARSE_CUSTOM_POST +#define NVBENCH_MAIN_PARSE_CUSTOM_POST(parser) verify(parser) + +void verify_json_output() +{ + const auto json = read_json_output(); + verify_json_array(json, "\"argv\"", "\"--profile\"", "\"--my-custom-arg\""); + verify_json_array(json, "\"raw_argv\"", "\"--my-custom-arg\"", "\"--profile\""); + std::remove(json_output_path); +} + +#undef NVBENCH_MAIN_FINALIZE_CUSTOM_PRE +#define NVBENCH_MAIN_FINALIZE_CUSTOM_PRE() verify_json_output() + +void bench(nvbench::state &state) +{ + state.exec([](nvbench::launch &) {}); +} +NVBENCH_BENCH(bench); + +NVBENCH_MAIN diff --git a/testing/markdown_printer.cu b/testing/markdown_printer.cu new file mode 100644 index 00000000..a74e160e --- /dev/null +++ b/testing/markdown_printer.cu @@ -0,0 +1,42 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include "test_asserts.cuh" + +void test_argv_fence_grows_for_backticks() +{ + std::ostringstream output; + nvbench::markdown_printer printer{output}; + + printer.log_argv({"benchmark"}); + printer.log_raw_argv({"benchmark", "contains```fence"}); + printer.print_argv(); + + const auto markdown = output.str(); + ASSERT(markdown.find("# Command Line\n\n````\n") != std::string::npos); + ASSERT(markdown.find("contains```fence") != std::string::npos); + ASSERT(markdown.find("\n````\n\n") != std::string::npos); +} + +int main() { test_argv_fence_grows_for_backticks(); }