From 68c3ea19fcabe4cf020d7c6315814a41e00adac7 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Thu, 27 Aug 2026 09:51:50 -0400 Subject: [PATCH 1/4] Print argv at the top for all benchmarking runs --- README.md | 4 ++++ nvbench/main.cuh | 1 + nvbench/markdown_printer.cu | 44 +++++++++++++++++++++++++++++++++++ nvbench/markdown_printer.cuh | 4 ++++ nvbench/printer_base.cuh | 9 +++++++ nvbench/printer_multiplex.cuh | 1 + nvbench/printer_multiplex.cxx | 8 +++++++ 7 files changed, 71 insertions(+) diff --git a/README.md b/README.md index 839c2c05..70ca04d1 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,10 @@ Examples are built by default into `build/bin` and are prefixed with `nvbench.ex Example output from `nvbench.example.throughput` ``` +# Command Line + +./bin/nvbench.example.throughput + # Devices ## [0] `Quadro GV100` diff --git a/nvbench/main.cuh b/nvbench/main.cuh index 24029866..b00848d1 100644 --- a/nvbench/main.cuh +++ b/nvbench/main.cuh @@ -209,6 +209,7 @@ inline void main_print_preamble(option_parser &parser) { auto &printer = parser.get_printer(); + printer.print_argv(); printer.print_device_info(); printer.print_log_preamble(); } diff --git a/nvbench/markdown_printer.cu b/nvbench/markdown_printer.cu index f80b17b2..a78bc32f 100644 --- a/nvbench/markdown_printer.cu +++ b/nvbench/markdown_printer.cu @@ -41,6 +41,50 @@ namespace nvbench { +void markdown_printer::do_log_argv(const std::vector &argv) { m_argv = argv; } + +void markdown_printer::do_print_argv() +{ + if (m_argv.empty()) + { + return; + } + + // Quote arguments that contain characters a shell would interpret, so that the + // printed command line can be copied and pasted. + auto quote = [](const std::string &arg) -> std::string { + if (!arg.empty() && arg.find_first_of(" \t\n\"'\\$`|&;<>()*?[]{}#~!") == std::string::npos) + { + return arg; + } + + std::string result{"'"}; + for (const char c : arg) + { + if (c == '\'') + { // A single quote cannot appear inside single quotes; close, escape, reopen. + result += "'\\''"; + } + else + { + result += c; + } + } + result += '\''; + return result; + }; + + fmt::memory_buffer buffer; + fmt::format_to(fmt::appender(buffer), "# Command Line\n\n```\n"); + for (std::size_t i = 0; i < m_argv.size(); ++i) + { + fmt::format_to(fmt::appender(buffer), "{}{}", i == 0 ? "" : " ", quote(m_argv[i])); + } + fmt::format_to(fmt::appender(buffer), "\n```\n\n"); + + m_ostream << fmt::to_string(buffer); +} + void markdown_printer::do_print_device_info() { fmt::memory_buffer buffer; diff --git a/nvbench/markdown_printer.cuh b/nvbench/markdown_printer.cuh index 73c38c60..9f5775e1 100644 --- a/nvbench/markdown_printer.cuh +++ b/nvbench/markdown_printer.cuh @@ -31,6 +31,7 @@ #include #include +#include namespace nvbench { @@ -62,6 +63,8 @@ struct markdown_printer : nvbench::printer_base protected: // Virtual API from printer_base: + void do_log_argv(const std::vector &argv) override; + void do_print_argv() override; void do_print_device_info() override; void do_print_log_preamble() override; void do_print_log_epilogue() override; @@ -80,6 +83,7 @@ protected: virtual std::string do_format_sample_size(const nvbench::summary &count); virtual std::string do_format_percentage(const nvbench::summary &percentage); + std::vector m_argv; bool m_color{false}; }; diff --git a/nvbench/printer_base.cuh b/nvbench/printer_base.cuh index 2347a2f3..b372f7e3 100644 --- a/nvbench/printer_base.cuh +++ b/nvbench/printer_base.cuh @@ -99,6 +99,14 @@ struct printer_base */ void log_argv(const std::vector &argv) { this->do_log_argv(argv); } + /*! + * Print the command line used to invoke the current executable, if supported. + * + * Called before running benchmarks for active terminal output. Must be called + * after `log_argv`. + */ + void print_argv() { this->do_print_argv(); } + /*! * Print a summary of all detected devices, if supported. * @@ -194,6 +202,7 @@ struct printer_base protected: // Implementation hooks for subclasses: virtual void do_log_argv(const std::vector &) {} + virtual void do_print_argv() {} virtual void do_print_device_info() {} virtual void do_print_log_preamble() {} virtual void do_print_log_epilogue() {} diff --git a/nvbench/printer_multiplex.cuh b/nvbench/printer_multiplex.cuh index b9090864..ef4f9f48 100644 --- a/nvbench/printer_multiplex.cuh +++ b/nvbench/printer_multiplex.cuh @@ -57,6 +57,7 @@ struct printer_multiplex : nvbench::printer_base protected: void do_log_argv(const std::vector &argv) override; + void do_print_argv() override; void do_print_device_info() override; void do_print_log_preamble() override; void do_print_log_epilogue() override; diff --git a/nvbench/printer_multiplex.cxx b/nvbench/printer_multiplex.cxx index 5cede21d..6def8800 100644 --- a/nvbench/printer_multiplex.cxx +++ b/nvbench/printer_multiplex.cxx @@ -30,6 +30,14 @@ printer_multiplex::printer_multiplex() : printer_base(std::cerr) // Nothing should write to this. {} +void printer_multiplex::do_print_argv() +{ + for (auto &format_ptr : m_printers) + { + format_ptr->print_argv(); + } +} + void printer_multiplex::do_print_device_info() { for (auto &format_ptr : m_printers) From e7e391b3e468d803df44e41a2874fa7fa8e4ae23 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Thu, 27 Aug 2026 13:33:20 -0400 Subject: [PATCH 2/4] fixup! Print argv at the top for all benchmarking runs --- nvbench/markdown_printer.cu | 101 ++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 23 deletions(-) diff --git a/nvbench/markdown_printer.cu b/nvbench/markdown_printer.cu index a78bc32f..d2f2a23f 100644 --- a/nvbench/markdown_printer.cu +++ b/nvbench/markdown_printer.cu @@ -41,44 +41,99 @@ namespace nvbench { -void markdown_printer::do_log_argv(const std::vector &argv) { m_argv = argv; } +namespace +{ -void markdown_printer::do_print_argv() +// Quote an argument for the shell of the current platform, so that the printed +// command line can be copied and pasted. +#ifdef _WIN32 + +// The Windows command processor (cmd.exe) does not group text inside single +// quotes, so use double quotes with backslash escapes. +std::string shell_quote(const std::string &arg) { - if (m_argv.empty()) + if (!arg.empty() && arg.find_first_of(" \t\n\v\"^&|<>()%!") == std::string::npos) { - return; + return arg; } - // Quote arguments that contain characters a shell would interpret, so that the - // printed command line can be copied and pasted. - auto quote = [](const std::string &arg) -> std::string { - if (!arg.empty() && arg.find_first_of(" \t\n\"'\\$`|&;<>()*?[]{}#~!") == std::string::npos) + // Follow the rules of CommandLineToArgvW: a run of backslashes is only special + // when a double quote comes after it. + std::string result{'"'}; + for (auto iter = arg.begin(); iter != arg.end(); ++iter) + { + std::size_t num_backslashes = 0; + while (iter != arg.end() && *iter == '\\') { - return arg; + ++num_backslashes; + ++iter; + } + + if (iter == arg.end()) + { // Double the backslashes that come before the closing quote. + result.append(num_backslashes * 2, '\\'); + break; } - std::string result{"'"}; - for (const char c : arg) + if (*iter == '"') + { // Double the backslashes that come before a quote, then escape the quote. + result.append(num_backslashes * 2, '\\'); + result += "\\\""; + } + else { - if (c == '\'') - { // A single quote cannot appear inside single quotes; close, escape, reopen. - result += "'\\''"; - } - else - { - result += c; - } + result.append(num_backslashes, '\\'); + result += *iter; } - result += '\''; - return result; - }; + } + result += '"'; + return result; +} + +#else + +// POSIX shells (sh, bash, zsh) take single quotes. +std::string shell_quote(const std::string &arg) +{ + if (!arg.empty() && arg.find_first_of(" \t\n\"'\\$`|&;<>()*?[]{}#~!") == std::string::npos) + { + return arg; + } + + std::string result{"'"}; + for (const char c : arg) + { + if (c == '\'') + { // A single quote cannot appear inside single quotes; close, escape, reopen. + result += "'\\''"; + } + else + { + result += c; + } + } + result += '\''; + return result; +} + +#endif // _WIN32 + +} // namespace + +void markdown_printer::do_log_argv(const std::vector &argv) { m_argv = argv; } + +void markdown_printer::do_print_argv() +{ + if (m_argv.empty()) + { + return; + } fmt::memory_buffer buffer; fmt::format_to(fmt::appender(buffer), "# Command Line\n\n```\n"); for (std::size_t i = 0; i < m_argv.size(); ++i) { - fmt::format_to(fmt::appender(buffer), "{}{}", i == 0 ? "" : " ", quote(m_argv[i])); + fmt::format_to(fmt::appender(buffer), "{}{}", i == 0 ? "" : " ", shell_quote(m_argv[i])); } fmt::format_to(fmt::appender(buffer), "\n```\n\n"); From 789a8447605e008d983b87a016c903b6b0b17ed3 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Thu, 27 Aug 2026 13:46:30 -0400 Subject: [PATCH 3/4] fixup! Print argv at the top for all benchmarking runs --- nvbench/main.cuh | 7 ++++ nvbench/option_parser.cu | 2 +- nvbench/option_parser.cuh | 23 +++++++++++ nvbench/printer_base.cuh | 4 ++ testing/CMakeLists.txt | 6 +++ testing/custom_main_raw_argv.cu | 69 +++++++++++++++++++++++++++++++++ 6 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 testing/custom_main_raw_argv.cu diff --git a/nvbench/main.cuh b/nvbench/main.cuh index b00848d1..e45526db 100644 --- a/nvbench/main.cuh +++ b/nvbench/main.cuh @@ -39,6 +39,7 @@ #include #include #include +#include #include // Advanced users can rebuild NVBench's `main` function using the macros in this file, or replace @@ -57,12 +58,16 @@ // Customization point, called before NVBench parsing. Update argc/argv if needed. // argc/argv are the usual command line arguments types. The ARGS version of this // macro is a bit more convenient. +// NVBench captures the command line before this handler runs. Changes made here +// do not alter the reported command line. #ifndef NVBENCH_MAIN_CUSTOM_ARGC_ARGV_HANDLER #define NVBENCH_MAIN_CUSTOM_ARGC_ARGV_HANDLER(argc, argv) []() {}() #endif // Customization point, called before NVBench parsing. Update args if needed. // Args is a vector of strings, each element is an argument. +// NVBench captures the command line before this handler runs. Changes made here +// do not alter the reported command line. #ifndef NVBENCH_MAIN_CUSTOM_ARGS_HANDLER #define NVBENCH_MAIN_CUSTOM_ARGS_HANDLER(args) []() {}() #endif @@ -132,10 +137,12 @@ #ifndef NVBENCH_MAIN_PARSE #define NVBENCH_MAIN_PARSE(argc, argv) \ + std::vector raw_args = nvbench::detail::main_convert_args(argc, argv); \ NVBENCH_MAIN_CUSTOM_ARGC_ARGV_HANDLER(argc, argv); \ std::vector args = nvbench::detail::main_convert_args(argc, argv); \ NVBENCH_MAIN_CUSTOM_ARGS_HANDLER(args); \ nvbench::option_parser parser; \ + parser.set_raw_args(std::move(raw_args)); \ NVBENCH_MAIN_PARSE_CUSTOM_PRE(parser, args); \ parser.parse(args); \ NVBENCH_MAIN_PARSE_CUSTOM_POST(parser) diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index 4f865d7f..a7a88394 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -495,7 +495,7 @@ void option_parser::parse_impl() this->update_used_device_state(); - m_printer.log_argv(m_args); + m_printer.log_argv(this->get_raw_args()); } void option_parser::parse_range(option_parser::arg_iterator_t first, diff --git a/nvbench/option_parser.cuh b/nvbench/option_parser.cuh index 9a8ee405..a83ed9f7 100644 --- a/nvbench/option_parser.cuh +++ b/nvbench/option_parser.cuh @@ -37,6 +37,7 @@ #include #include #include +#include #include namespace nvbench @@ -62,11 +63,30 @@ struct option_parser void parse(int argc, char const *const argv[]); void parse(std::vector args); + /*! + * Set the command line that invoked the executable, before any modification. + * + * Call this before `parse`. `parse` sends these args to the printers instead + * of its own args. + */ + void set_raw_args(std::vector raw_args) { m_raw_args = std::move(raw_args); } + [[nodiscard]] benchmark_vector &get_benchmarks() { return m_benchmarks; }; [[nodiscard]] const benchmark_vector &get_benchmarks() const { return m_benchmarks; }; + /*! + * The args given to `parse`. A customization handler can modify these. + */ [[nodiscard]] const std::vector &get_args() const { return m_args; } + /*! + * The args given to `set_raw_args`, or `get_args` if it was not called. + */ + [[nodiscard]] const std::vector &get_raw_args() const + { + return m_raw_args ? *m_raw_args : m_args; + } + /*! * Returns the output format requested by the parse options. * @@ -141,6 +161,9 @@ private: // Command line args std::vector m_args; + // The unmodified command line, if the caller supplied one. + std::optional> m_raw_args; + // Store benchmark modifiers passed in before any benchmarks are requested as // "global args". Replay them after every benchmark. std::vector m_global_benchmark_args; diff --git a/nvbench/printer_base.cuh b/nvbench/printer_base.cuh index b372f7e3..3b86a48a 100644 --- a/nvbench/printer_base.cuh +++ b/nvbench/printer_base.cuh @@ -96,6 +96,10 @@ struct printer_base /*! * Called once with the command line arguments used to invoke the current * executable. + * + * `NVBENCH_MAIN` supplies the command line as the user typed it, not the + * arguments that the customization handlers produce. Use + * `nvbench::option_parser::get_args` for the parsed arguments. */ void log_argv(const std::vector &argv) { this->do_log_argv(argv); } diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 79f59ddf..019678bf 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -11,6 +11,7 @@ set(test_srcs custom_main_custom_args.cu custom_main_custom_exceptions.cu custom_main_global_state_raii.cu + custom_main_raw_argv.cu enum_type_list.cu entropy_criterion.cu exception_safety.cu @@ -38,6 +39,7 @@ set(test_srcs # CTest commands+args can't be modified after creation, so we need to rely on substitution. set(NVBench_TEST_ARGS_nvbench.test.custom_main_custom_args "--quiet" "--my-custom-arg" "--profile" "-d" "0") set(NVBench_TEST_ARGS_nvbench.test.custom_main_custom_exceptions "--quiet" "--profile" "-d" "0") +set(NVBench_TEST_ARGS_nvbench.test.custom_main_raw_argv "--my-custom-arg" "-d" "0") # Metatarget for all tests: add_custom_target(nvbench.test.all) @@ -60,6 +62,10 @@ endforeach() set_tests_properties(nvbench.test.custom_main_custom_exceptions PROPERTIES PASS_REGULAR_EXPRESSION "Custom error detected: Expected exception thrown." ) +set_tests_properties(nvbench.test.custom_main_raw_argv PROPERTIES + PASS_REGULAR_EXPRESSION "custom_main_raw_argv --my-custom-arg -d 0" + FAIL_REGULAR_EXPRESSION "custom_main_raw_argv --profile" +) set_tests_properties(nvbench.test.exception_safety PROPERTIES TIMEOUT 20) add_subdirectory(cmake) diff --git a/testing/custom_main_raw_argv.cu b/testing/custom_main_raw_argv.cu new file mode 100644 index 00000000..3fa675aa --- /dev/null +++ b/testing/custom_main_raw_argv.cu @@ -0,0 +1,69 @@ +/* + * Copyright 2026 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 with the LLVM exception + * (the "License"); you may not use this file except in compliance with + * the License. + * + * You may obtain a copy of the License at + * + * http://llvm.org/foundation/relicensing/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +// Rewrite "--my-custom-arg" into "--profile". The reported command line must keep +// the original argument. +void custom_arg_handler(std::vector &args) +{ + auto it = std::find(args.begin(), args.end(), "--my-custom-arg"); + if (it == args.end()) + { + throw std::runtime_error("Custom argument not found."); + } + *it = "--profile"; +} + +#undef NVBENCH_MAIN_CUSTOM_ARGS_HANDLER +#define NVBENCH_MAIN_CUSTOM_ARGS_HANDLER(args) custom_arg_handler(args) + +void verify(nvbench::option_parser &parser) +{ + const auto &raw = parser.get_raw_args(); + const auto &parsed = parser.get_args(); + + if (std::find(raw.begin(), raw.end(), "--my-custom-arg") == raw.end()) + { + throw std::runtime_error("Raw args lost the original argument."); + } + if (std::find(raw.begin(), raw.end(), "--profile") != raw.end()) + { + throw std::runtime_error("Raw args contain the rewritten argument."); + } + if (std::find(parsed.begin(), parsed.end(), "--profile") == parsed.end()) + { + throw std::runtime_error("Parsed args lost the rewritten argument."); + } +} + +#undef NVBENCH_MAIN_PARSE_CUSTOM_POST +#define NVBENCH_MAIN_PARSE_CUSTOM_POST(parser) verify(parser) + +void bench(nvbench::state &state) +{ + state.exec([](nvbench::launch &) {}); +} +NVBENCH_BENCH(bench); + +NVBENCH_MAIN From 6b2d43398996122ea9c2234afd82cc250234eea9 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Fri, 28 Aug 2026 07:40:17 -0400 Subject: [PATCH 4/4] fixup! Print argv at the top for all benchmarking runs --- nvbench/markdown_printer.cu | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nvbench/markdown_printer.cu b/nvbench/markdown_printer.cu index d2f2a23f..489aa3b3 100644 --- a/nvbench/markdown_printer.cu +++ b/nvbench/markdown_printer.cu @@ -59,7 +59,10 @@ std::string shell_quote(const std::string &arg) // Follow the rules of CommandLineToArgvW: a run of backslashes is only special // when a double quote comes after it. - std::string result{'"'}; + std::string result; + + result.reserve((4 * arg.size()) + 2); + result += '\''; for (auto iter = arg.begin(); iter != arg.end(); ++iter) { std::size_t num_backslashes = 0; @@ -100,7 +103,10 @@ std::string shell_quote(const std::string &arg) return arg; } - std::string result{"'"}; + std::string result; + + result.reserve((4 * arg.size()) + 2); + result += '\''; for (const char c : arg) { if (c == '\'')