diff --git a/CMakeLists.txt b/CMakeLists.txt index 93731d3..238662b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ include(CTest) include(cmake/LaghuCapabilities.cmake) include(cmake/LaghuDependencyDag.cmake) include(cmake/LaghuApiBoundaries.cmake) +include(cmake/LaghuBenchmarks.cmake) include(cmake/LaghuDependencies.cmake) include(cmake/LaghuFeatures.cmake) include(cmake/LaghuFuzzing.cmake) @@ -44,6 +45,7 @@ laghu_configure_dependency_registry() laghu_configure_dependency_modes() laghu_configure_capability_header() laghu_configure_build_identity() +laghu_configure_benchmarks() add_library(laghu_core STATIC src/core/clocks.cpp @@ -104,6 +106,8 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") PROPERTIES COMPILE_OPTIONS -Wno-unsafe-buffer-usage) endif() +laghu_add_benchmark_targets() + laghu_declare_subsystem_graph() laghu_configure_api_boundaries() laghu_apply_first_party_contract(laghu_os) @@ -305,6 +309,7 @@ set_property(TARGET visibility_negative_fixture PROPERTY CXX_VISIBILITY_PRESET h set_property(TARGET visibility_negative_fixture PROPERTY VISIBILITY_INLINES_HIDDEN YES) laghu_add_validation_tests() +laghu_add_benchmark_validation_tests() laghu_add_static_analysis_validation_tests() laghu_add_static_analysis_target() laghu_add_install_layout_test() @@ -326,6 +331,7 @@ set(laghu_verify_targets laghu_core_shared_offsets_test laghu_core_mapped_regions_test laghu_core_binary_envelope_test laghu_core_digest_primitives_test laghu_os_iovec_translation_test laghu_test_support laghu_test_faults laghu_test_time_entropy + laghu_benchmark_metrics_test laghu_benchmark_workload_counters_test laghu_test_time_entropy_test laghu_test_support_fixtures_test laghu_test_support_runner_contract_test laghu_test_fault_injection_test laghu laghu_capability_header_parity laghu_visibility_probe visibility_negative_fixture) diff --git a/bench/.gitkeep b/bench/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/bench/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/bench/core_foundation.cpp b/bench/core_foundation.cpp new file mode 100644 index 0000000..d78ca7b --- /dev/null +++ b/bench/core_foundation.cpp @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: AGPL-3.0-only +#include + +#include + +#include + +std::uint64_t laghu::benchmark::internal::run_core_foundation( + std::uint64_t seed, WorkloadCounters& counters) noexcept { + std::uint64_t state = seed; + for (std::uint64_t iteration = 0U; iteration < core_foundation_operations_per_interval; + ++iteration) { + const auto incremented = laghu::core::checked_add(state, std::uint64_t{0x9e3779b9U}); + if (!incremented.has_value()) { + return state; + } + state = *incremented ^ (state >> 13U); + } + // The foundation workload deliberately performs no allocation or Laghu OS + // operation. It records no events, so the caller reports these metrics as + // uninstrumented rather than inferring them from host metrics. + static_cast(counters); + return state; +} diff --git a/bench/private/laghu/benchmark/internal/metrics.hpp b/bench/private/laghu/benchmark/internal/metrics.hpp new file mode 100644 index 0000000..c68eccb --- /dev/null +++ b/bench/private/laghu/benchmark/internal/metrics.hpp @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: AGPL-3.0-only +#pragma once + +#include +#include +#include +#include + +namespace laghu::benchmark::internal { + +struct Percentiles final { + std::uint64_t p50{}; + std::uint64_t p95{}; + std::uint64_t p99{}; + std::uint64_t p999{}; +}; + +[[nodiscard]] constexpr std::size_t nearest_rank_index(std::size_t sample_count, + std::uint64_t per_mille) noexcept { + if (sample_count == 0U || per_mille == 0U || per_mille > 1000U) { + return 0U; + } + const std::size_t numerator = sample_count * static_cast(per_mille); + return (numerator + 999U) / 1000U - 1U; +} + +template +[[nodiscard]] bool summarize_percentiles(std::array& samples, + std::size_t count, Percentiles& output) noexcept { + if (count == 0U || count > Capacity) { + return false; + } + for (std::size_t left = 0U; left < count; ++left) { + std::size_t minimum = left; + for (std::size_t right = left + 1U; right < count; ++right) { + if (samples[right] < samples[minimum]) { + minimum = right; + } + } + const std::uint64_t value = samples[left]; + samples[left] = samples[minimum]; + samples[minimum] = value; + } + output = Percentiles{ + samples[nearest_rank_index(count, 500U)], + samples[nearest_rank_index(count, 950U)], + samples[nearest_rank_index(count, 990U)], + samples[nearest_rank_index(count, 999U)], + }; + return true; +} + +} // namespace laghu::benchmark::internal diff --git a/bench/private/laghu/benchmark/internal/workload.hpp b/bench/private/laghu/benchmark/internal/workload.hpp new file mode 100644 index 0000000..18964fe --- /dev/null +++ b/bench/private/laghu/benchmark/internal/workload.hpp @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: AGPL-3.0-only +#pragma once + +#include +#include + +namespace laghu::benchmark::internal { + +class WorkloadCounters final { + public: + [[nodiscard]] bool record_allocation_events(std::uint64_t count = 1U) noexcept { + return record(allocation_count_, allocation_instrumented_, count); + } + + [[nodiscard]] bool record_laghu_syscall_events(std::uint64_t count = 1U) noexcept { + return record(laghu_syscall_count_, laghu_syscall_instrumented_, count); + } + + [[nodiscard]] bool allocation_instrumented() const noexcept { return allocation_instrumented_; } + [[nodiscard]] bool laghu_syscall_instrumented() const noexcept { + return laghu_syscall_instrumented_; + } + [[nodiscard]] std::uint64_t allocation_count() const noexcept { return allocation_count_; } + [[nodiscard]] std::uint64_t laghu_syscall_count() const noexcept { return laghu_syscall_count_; } + + private: + [[nodiscard]] static bool record(std::uint64_t& total, bool& instrumented, + std::uint64_t count) noexcept { + if (count == 0U) { + return true; + } + if (count > std::numeric_limits::max() - total) { + return false; + } + total += count; + instrumented = true; + return true; + } + + std::uint64_t allocation_count_{}; + std::uint64_t laghu_syscall_count_{}; + bool allocation_instrumented_{}; + bool laghu_syscall_instrumented_{}; +}; + +inline constexpr std::uint64_t core_foundation_operations_per_interval = 4096U; + +[[nodiscard]] std::uint64_t run_core_foundation(std::uint64_t seed, + WorkloadCounters& counters) noexcept; + +} // namespace laghu::benchmark::internal diff --git a/bench/runner.cpp b/bench/runner.cpp new file mode 100644 index 0000000..c8e4916 --- /dev/null +++ b/bench/runner.cpp @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: AGPL-3.0-only +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if defined(__APPLE__) || defined(__FreeBSD__) +#include +#endif + +#include +#include +#include + +namespace { + +using laghu::benchmark::internal::Percentiles; +using laghu::benchmark::internal::WorkloadCounters; + +constexpr std::size_t maximum_intervals = 1000U; +constexpr std::uint64_t maximum_warmup_intervals = 10000U; +constexpr std::size_t diagnostic_capacity = 256U; +constexpr std::size_t json_capacity = 8192U; + +enum class ExitCode : int { + success = 0, + invalid_arguments = 64, + metric_unavailable = 69, + output_failure = 74, +}; + +struct Options final { + std::uint64_t warmup_intervals{}; + std::size_t measured_intervals{}; + std::string_view workload{}; +}; + +struct TextMetric final { + bool available{}; + std::array text{}; + std::size_t size{}; +}; + +struct NumericMetric final { + bool available{}; + std::uint64_t value{}; + std::string_view reason{}; +}; + +class JsonWriter final { + public: + [[nodiscard]] bool append(std::string_view text) noexcept { + if (text.size() > output_.size() - size_) { + return false; + } + std::memcpy(output_.data() + size_, text.data(), text.size()); + size_ += text.size(); + return true; + } + + [[nodiscard]] bool append_number(std::uint64_t value) noexcept { + const auto converted = std::to_chars(output_.data() + size_, output_.data() + output_.size(), value); + if (converted.ec != std::errc{}) { + return false; + } + size_ = static_cast(converted.ptr - output_.data()); + return true; + } + + [[nodiscard]] bool append_json_string(std::string_view value) noexcept { + if (!append("\"")) { + return false; + } + for (const char character : value) { + switch (character) { + case '\\': + if (!append("\\\\")) { + return false; + } + break; + case '"': + if (!append("\\\"")) { + return false; + } + break; + case '\n': + if (!append("\\n")) { + return false; + } + break; + case '\r': + if (!append("\\r")) { + return false; + } + break; + case '\t': + if (!append("\\t")) { + return false; + } + break; + default: + if (static_cast(character) < 0x20U || !append({&character, 1U})) { + return false; + } + break; + } + } + return append("\""); + } + + [[nodiscard]] std::string_view view() const noexcept { return {output_.data(), size_}; } + + private: + std::array output_{}; + std::size_t size_{}; +}; + +[[nodiscard]] bool write_all(int descriptor, std::string_view text) noexcept { + while (!text.empty()) { + const ssize_t written = ::write(descriptor, text.data(), text.size()); + if (written > 0) { + text.remove_prefix(static_cast(written)); + continue; + } + if (written < 0 && errno == EINTR) { + continue; + } + return false; + } + return true; +} + +[[nodiscard]] bool parse_positive(std::string_view text, std::uint64_t maximum, + std::uint64_t& output) noexcept { + if (text.empty()) { + return false; + } + const auto result = std::from_chars(text.data(), text.data() + text.size(), output); + return result.ec == std::errc{} && result.ptr == text.data() + text.size() && output <= maximum; +} + +[[nodiscard]] bool parse_options(int argc, char** argv, Options& options) noexcept { + bool workload_seen = false; + bool warmup_seen = false; + bool intervals_seen = false; + for (int argument = 1; argument < argc; ++argument) { + const std::string_view option{argv[argument]}; + if (argument + 1 >= argc) { + return false; + } + const std::string_view value{argv[++argument]}; + if (option == "--workload" && !workload_seen && value == "core-foundation") { + options.workload = value; + workload_seen = true; + continue; + } + if (option == "--warmup" && !warmup_seen && + parse_positive(value, maximum_warmup_intervals, options.warmup_intervals)) { + warmup_seen = true; + continue; + } + std::uint64_t intervals{}; + if (option == "--intervals" && !intervals_seen && + parse_positive(value, maximum_intervals, intervals) && intervals != 0U) { + options.measured_intervals = static_cast(intervals); + intervals_seen = true; + continue; + } + return false; + } + return workload_seen && warmup_seen && intervals_seen; +} + +[[nodiscard]] bool timespec_nanoseconds(const timespec& value, std::uint64_t& output) noexcept { + constexpr std::uint64_t nanoseconds_per_second = 1000000000U; + if (value.tv_sec < 0 || value.tv_nsec < 0 || value.tv_nsec >= static_cast(nanoseconds_per_second)) { + return false; + } + const auto seconds = static_cast(value.tv_sec); + const auto nanoseconds = static_cast(value.tv_nsec); + if (seconds > (std::numeric_limits::max() - nanoseconds) / nanoseconds_per_second) { + return false; + } + output = seconds * nanoseconds_per_second + nanoseconds; + return true; +} + +[[nodiscard]] bool monotonic_now(std::uint64_t& output) noexcept { + timespec value{}; + return ::clock_gettime(CLOCK_MONOTONIC, &value) == 0 && timespec_nanoseconds(value, output); +} + +[[nodiscard]] NumericMetric process_cpu_time() noexcept { + timespec value{}; + std::uint64_t nanoseconds{}; + if (::clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &value) != 0 || !timespec_nanoseconds(value, nanoseconds)) { + return NumericMetric{false, 0U, "clock_gettime_process_cpu_time_unavailable"}; + } + return NumericMetric{true, nanoseconds, {}}; +} + +[[nodiscard]] NumericMetric peak_rss_bytes() noexcept { + rusage usage{}; + if (::getrusage(RUSAGE_SELF, &usage) != 0 || usage.ru_maxrss < 0) { + return NumericMetric{false, 0U, "getrusage_peak_rss_unavailable"}; + } +#if defined(__APPLE__) + return NumericMetric{true, static_cast(usage.ru_maxrss), {}}; +#elif defined(__linux__) || defined(__FreeBSD__) + const auto kilobytes = static_cast(usage.ru_maxrss); + if (kilobytes > std::numeric_limits::max() / 1024U) { + return NumericMetric{false, 0U, "peak_rss_overflow"}; + } + return NumericMetric{true, kilobytes * 1024U, {}}; +#else + return NumericMetric{false, 0U, "peak_rss_unit_platform_unsupported"}; +#endif +} + +#if defined(__linux__) +[[nodiscard]] bool copy_metric_text(TextMetric& metric, std::string_view text) noexcept { + if (text.empty() || text.size() >= metric.text.size()) { + return false; + } + std::memmove(metric.text.data(), text.data(), text.size()); + metric.size = text.size(); + metric.available = true; + return true; +} +#endif + +[[nodiscard]] TextMetric cpu_description() noexcept { + TextMetric metric{}; +#if defined(__APPLE__) || defined(__FreeBSD__) + #if defined(__APPLE__) + constexpr std::string_view cpu_sysctl{"machdep.cpu.brand_string"}; + #else + constexpr std::string_view cpu_sysctl{"hw.model"}; + #endif + std::size_t size = metric.text.size() - 1U; + if (::sysctlbyname(cpu_sysctl.data(), metric.text.data(), &size, nullptr, 0U) == 0 && + size != 0U && size < metric.text.size()) { + metric.size = size; + if (metric.text[metric.size - 1U] == '\0') { + --metric.size; + } + metric.text[metric.size] = '\0'; + metric.available = metric.size != 0U; + } +#elif defined(__linux__) + int flags = O_RDONLY; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif + const int descriptor = ::open("/proc/cpuinfo", flags); + if (descriptor >= 0) { + const ssize_t read_count = ::read(descriptor, metric.text.data(), metric.text.size() - 1U); + static_cast(::close(descriptor)); + if (read_count > 0) { + const auto size = static_cast(read_count); + metric.text[size] = '\0'; + const std::string_view source{metric.text.data(), size}; + for (const std::string_view label : {std::string_view{"model name"}, std::string_view{"Hardware"}}) { + const std::size_t label_position = source.find(label); + if (label_position == std::string_view::npos) { + continue; + } + const std::size_t separator = source.find(':', label_position + label.size()); + const std::size_t line_end = source.find('\n', separator); + if (separator == std::string_view::npos || line_end == std::string_view::npos) { + continue; + } + std::string_view value = source.substr(separator + 1U, line_end - separator - 1U); + while (!value.empty() && (value.front() == ' ' || value.front() == '\t')) { + value.remove_prefix(1U); + } + while (!value.empty() && (value.back() == ' ' || value.back() == '\t')) { + value.remove_suffix(1U); + } + if (copy_metric_text(metric, value)) { + return metric; + } + } + } + } +#endif + return metric; +} + +[[nodiscard]] bool append_numeric_metric(JsonWriter& writer, const NumericMetric& metric) noexcept { + if (!writer.append("{\"status\":\"")) { + return false; + } + if (metric.available) { + return writer.append("available\",\"value\":") && writer.append_number(metric.value) && writer.append("}"); + } + return writer.append("unavailable\",\"reason\":") && writer.append_json_string(metric.reason) && + writer.append("}"); +} + +[[nodiscard]] bool write_report(const Options& options, const Percentiles& percentiles, + std::uint64_t throughput, bool throughput_available, + const NumericMetric& cpu_time, const NumericMetric& peak_rss, + const TextMetric& cpu, const WorkloadCounters& counters, + std::uint64_t checksum) noexcept { + JsonWriter writer; + const std::string_view cpu_text{cpu.text.data(), cpu.size}; + const bool complete = + writer.append("{\"schema_version\":") && + writer.append_json_string(laghu::benchmark::internal::schema_version) && + writer.append(",\"build\":{\"build_id\":") && + writer.append_json_string(laghu::benchmark::internal::build_id) && + writer.append(",\"compiler\":{\"id\":") && + writer.append_json_string(laghu::benchmark::internal::compiler_id) && + writer.append(",\"version\":") && + writer.append_json_string(laghu::benchmark::internal::compiler_version) && + writer.append("},\"dependencies\":") && + writer.append(laghu::benchmark::internal::dependencies_json) && + writer.append(",\"features\":") && + writer.append(laghu::benchmark::internal::features_json) && + writer.append(",\"target\":{\"architecture\":") && + writer.append_json_string(laghu::benchmark::internal::target_architecture) && + writer.append(",\"os\":") && + writer.append_json_string(laghu::benchmark::internal::target_os) && + writer.append("}},\"cpu\":{\"description\":{") && + writer.append(cpu.available ? "\"status\":\"available\",\"value\":" + : "\"status\":\"unavailable\",\"reason\":") && + writer.append_json_string(cpu.available ? cpu_text : std::string_view{"cpu_description_unavailable"}) && + writer.append("}},\"metrics\":{\"allocation_count\":{\"instrumented\":") && + writer.append(counters.allocation_instrumented() ? "true" : "false") && + writer.append(",\"value\":") && writer.append_number(counters.allocation_count()) && + writer.append("},\"cpu_time_ns\":") && + append_numeric_metric(writer, cpu_time) && + writer.append(",\"laghu_syscall_count\":{\"instrumented\":") && + writer.append(counters.laghu_syscall_instrumented() ? "true" : "false") && + writer.append(",\"value\":") && writer.append_number(counters.laghu_syscall_count()) && + writer.append("},\"latency_ns_per_interval\":{\"p50\":") && + writer.append_number(percentiles.p50) && + writer.append(",\"p95\":") && writer.append_number(percentiles.p95) && + writer.append(",\"p99\":") && writer.append_number(percentiles.p99) && + writer.append(",\"p99_9\":") && writer.append_number(percentiles.p999) && + writer.append("},\"peak_rss_bytes\":") && append_numeric_metric(writer, peak_rss) && + writer.append(",\"throughput_operations_per_second\":"); + if (!complete) { + return false; + } + const bool throughput_written = throughput_available + ? writer.append("{\"status\":\"available\",\"value\":") && writer.append_number(throughput) && writer.append("}") + : writer.append("{\"status\":\"unavailable\",\"reason\":\"zero_elapsed_time\"}"); + if (!throughput_written || !writer.append("},\"parameters\":{\"intervals\":") || + !writer.append_number(options.measured_intervals) || + !writer.append(",\"operations_per_interval\":") || + !writer.append_number(laghu::benchmark::internal::core_foundation_operations_per_interval) || + !writer.append(",\"warmup\":") || !writer.append_number(options.warmup_intervals) || + !writer.append("},\"workload\":") || !writer.append_json_string(options.workload) || + !writer.append(",\"workload_checksum\":") || !writer.append_number(checksum) || !writer.append("}\n")) { + return false; + } + return write_all(STDOUT_FILENO, writer.view()); +} + +} // namespace + +int main(int argc, char** argv) { + Options options{}; + if (!parse_options(argc, argv, options)) { + static_cast(write_all(STDERR_FILENO, + "usage: laghu_benchmark_core_foundation --workload core-foundation --warmup <0..10000> " + "--intervals <1..1000>\n")); + return static_cast(ExitCode::invalid_arguments); + } + + WorkloadCounters warmup_counters{}; + std::uint64_t checksum{}; + for (std::uint64_t interval = 0U; interval < options.warmup_intervals; ++interval) { + checksum ^= laghu::benchmark::internal::run_core_foundation(interval + 1U, warmup_counters); + } + + std::array durations{}; + WorkloadCounters measured_counters{}; + NumericMetric cpu_start = process_cpu_time(); + std::uint64_t total_duration{}; + for (std::size_t interval = 0U; interval < options.measured_intervals; ++interval) { + std::uint64_t begin{}; + std::uint64_t end{}; + if (!monotonic_now(begin)) { + static_cast(write_all(STDERR_FILENO, "laghu-benchmark: monotonic clock is unavailable\n")); + return static_cast(ExitCode::metric_unavailable); + } + checksum ^= laghu::benchmark::internal::run_core_foundation( + static_cast(interval) + options.warmup_intervals + 1U, measured_counters); + if (!monotonic_now(end) || end < begin) { + static_cast(write_all(STDERR_FILENO, "laghu-benchmark: monotonic clock is invalid\n")); + return static_cast(ExitCode::metric_unavailable); + } + durations[interval] = end - begin; + if (durations[interval] > std::numeric_limits::max() - total_duration) { + static_cast(write_all(STDERR_FILENO, "laghu-benchmark: duration overflow\n")); + return static_cast(ExitCode::metric_unavailable); + } + total_duration += durations[interval]; + } + NumericMetric cpu_end = process_cpu_time(); + NumericMetric cpu_time{false, 0U, "clock_gettime_process_cpu_time_unavailable"}; + if (cpu_start.available && cpu_end.available && cpu_end.value >= cpu_start.value) { + cpu_time = NumericMetric{true, cpu_end.value - cpu_start.value, {}}; + } + + Percentiles percentiles{}; + if (!laghu::benchmark::internal::summarize_percentiles( + durations, options.measured_intervals, percentiles)) { + static_cast(write_all(STDERR_FILENO, "laghu-benchmark: percentile calculation failed\n")); + return static_cast(ExitCode::metric_unavailable); + } + bool throughput_available = total_duration != 0U; + std::uint64_t throughput{}; + const std::uint64_t operations = + static_cast(options.measured_intervals) * + laghu::benchmark::internal::core_foundation_operations_per_interval; + if (throughput_available) { + constexpr std::uint64_t nanoseconds_per_second = 1000000000U; + if (operations <= std::numeric_limits::max() / nanoseconds_per_second) { + throughput = operations * nanoseconds_per_second / total_duration; + } else { + throughput_available = false; + } + } + if (!write_report(options, percentiles, throughput, throughput_available, cpu_time, peak_rss_bytes(), + cpu_description(), measured_counters, checksum)) { + static_cast(write_all(STDERR_FILENO, "laghu-benchmark: report exceeds bounded output capacity\n")); + return static_cast(ExitCode::output_failure); + } + return static_cast(ExitCode::success); +} diff --git a/cmake/ExpectBenchmarkReleaseExclusion.cmake b/cmake/ExpectBenchmarkReleaseExclusion.cmake new file mode 100644 index 0000000..17a71a7 --- /dev/null +++ b/cmake/ExpectBenchmarkReleaseExclusion.cmake @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: AGPL-3.0-only +if(NOT DEFINED BUILD_DIRECTORY OR NOT DEFINED ARCHIVE OR NOT DEFINED EXECUTABLE OR NOT DEFINED NM OR + NOT DEFINED STAGE_DIRECTORY OR NOT DEFINED INSTALL_PREFIX) + message(FATAL_ERROR "Laghu benchmark exclusion expectation requires BUILD_DIRECTORY ARCHIVE EXECUTABLE NM STAGE_DIRECTORY and INSTALL_PREFIX") +endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" --build "${BUILD_DIRECTORY}" --target laghu + RESULT_VARIABLE build_result + OUTPUT_VARIABLE build_output + ERROR_VARIABLE build_diagnostics) +if(NOT build_result EQUAL 0) + message(FATAL_ERROR "Laghu benchmark exclusion expectation failed: build=${build_output}${build_diagnostics}") +endif() + +file(REMOVE_RECURSE "${STAGE_DIRECTORY}") +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env "DESTDIR=${STAGE_DIRECTORY}" + "${CMAKE_COMMAND}" --install "${BUILD_DIRECTORY}" + RESULT_VARIABLE install_result + OUTPUT_VARIABLE install_output + ERROR_VARIABLE install_diagnostics) +if(NOT install_result EQUAL 0) + message(FATAL_ERROR "Laghu benchmark exclusion expectation failed: install=${install_output}${install_diagnostics}") +endif() + +set(stage_root "${STAGE_DIRECTORY}${INSTALL_PREFIX}") +file(GLOB_RECURSE staged_files LIST_DIRECTORIES false RELATIVE "${stage_root}" "${stage_root}/*") +list(SORT staged_files) +set(expected_staged_files + bin/laghu + lib/laghu/liblaghu_core.a + share/laghu/laghu-build-manifest-v1.json) +if(NOT staged_files STREQUAL expected_staged_files) + message(FATAL_ERROR "Laghu benchmark exclusion expectation failed: staged_layout_invalid actual=${staged_files}") +endif() + +set(staged_archive "${stage_root}/lib/laghu/liblaghu_core.a") +set(staged_executable "${stage_root}/bin/laghu") +foreach(artifact IN ITEMS "${ARCHIVE}" "${EXECUTABLE}" "${staged_archive}" "${staged_executable}") + execute_process(COMMAND "${NM}" -a "${artifact}" + RESULT_VARIABLE nm_result + OUTPUT_VARIABLE symbols + ERROR_VARIABLE diagnostics) + if(NOT nm_result EQUAL 0) + message(FATAL_ERROR "Laghu benchmark exclusion expectation failed: nm=${artifact}; diagnostics=${diagnostics}") + endif() + if(symbols MATCHES "laghu.*benchmark|benchmark_core_foundation") + message(FATAL_ERROR "Laghu benchmark exclusion expectation failed: instrumentation_leaked artifact=${artifact}") + endif() +endforeach() diff --git a/cmake/ExpectBenchmarkRunner.cmake b/cmake/ExpectBenchmarkRunner.cmake new file mode 100644 index 0000000..215bddc --- /dev/null +++ b/cmake/ExpectBenchmarkRunner.cmake @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: AGPL-3.0-only +if(NOT DEFINED SCRIPT OR NOT DEFINED BUILD_DIRECTORY OR NOT DEFINED SOURCE_DIRECTORY OR + NOT DEFINED EXPECTED_FEATURE_COUNT OR NOT DEFINED EXPECTED_DEPENDENCY_COUNT) + message(FATAL_ERROR "Laghu benchmark expectation requires SCRIPT BUILD_DIRECTORY SOURCE_DIRECTORY EXPECTED_FEATURE_COUNT and EXPECTED_DEPENDENCY_COUNT") +endif() + +function(laghu_benchmark_run output warmup) + execute_process( + COMMAND "${SCRIPT}" --build "${BUILD_DIRECTORY}" --workload core-foundation --warmup "${warmup}" --intervals 5 + RESULT_VARIABLE result + OUTPUT_VARIABLE standard_output + ERROR_VARIABLE diagnostics) + if(NOT result EQUAL 0) + message(FATAL_ERROR "Laghu benchmark expectation failed: positive=${standard_output}${diagnostics}") + endif() + set(${output} "${standard_output}" PARENT_SCOPE) +endfunction() + +laghu_benchmark_run(first 1) +laghu_benchmark_run(second 1) +laghu_benchmark_run(no_warmup 0) +laghu_benchmark_run(extended_warmup 7) + +foreach(output IN ITEMS "${first}" "${second}") + if(NOT output MATCHES "^\\{\\\"schema_version\\\":\\\"laghu-benchmark-v1\\\"") + message(FATAL_ERROR "Laghu benchmark expectation failed: canonical_schema_prefix_missing") + endif() + string(FIND "${output}" "${SOURCE_DIRECTORY}" source_path) + string(FIND "${output}" "${BUILD_DIRECTORY}" build_path) + if(NOT source_path EQUAL -1 OR NOT build_path EQUAL -1) + message(FATAL_ERROR "Laghu benchmark expectation failed: path_leakage") + endif() + string(JSON schema GET "${output}" schema_version) + string(JSON workload GET "${output}" workload) + string(JSON build_id GET "${output}" build build_id) + string(JSON feature_count LENGTH "${output}" build features) + string(JSON dependency_count LENGTH "${output}" build dependencies) + string(JSON interval_count GET "${output}" parameters intervals) + string(JSON warmup_count GET "${output}" parameters warmup) + string(JSON operations GET "${output}" parameters operations_per_interval) + string(JSON p50 GET "${output}" metrics latency_ns_per_interval p50) + string(JSON p95 GET "${output}" metrics latency_ns_per_interval p95) + string(JSON p99 GET "${output}" metrics latency_ns_per_interval p99) + string(JSON p999 GET "${output}" metrics latency_ns_per_interval p99_9) + string(JSON allocation_instrumented GET "${output}" metrics allocation_count instrumented) + string(JSON allocation_count GET "${output}" metrics allocation_count value) + string(JSON syscall_instrumented GET "${output}" metrics laghu_syscall_count instrumented) + string(JSON syscall_count GET "${output}" metrics laghu_syscall_count value) + if(NOT schema STREQUAL "laghu-benchmark-v1" OR NOT workload STREQUAL "core-foundation" OR + build_id STREQUAL "" OR NOT feature_count EQUAL EXPECTED_FEATURE_COUNT OR + NOT dependency_count EQUAL EXPECTED_DEPENDENCY_COUNT OR + NOT interval_count EQUAL 5 OR NOT warmup_count EQUAL 1 OR NOT operations EQUAL 4096 OR + p50 GREATER p95 OR p95 GREATER p99 OR p99 GREATER p999 OR + allocation_instrumented OR syscall_instrumented OR + NOT allocation_count EQUAL 0 OR NOT syscall_count EQUAL 0) + message(FATAL_ERROR "Laghu benchmark expectation failed: schema_or_metric_invalid") + endif() + string(JSON cpu_description_status GET "${output}" cpu description status) + string(JSON cpu_time_status GET "${output}" metrics cpu_time_ns status) + string(JSON peak_rss_status GET "${output}" metrics peak_rss_bytes status) + foreach(status IN ITEMS "${cpu_description_status}" "${cpu_time_status}" "${peak_rss_status}") + if(NOT status STREQUAL "available" AND NOT status STREQUAL "unavailable") + message(FATAL_ERROR "Laghu benchmark expectation failed: host_metric_status_invalid") + endif() + endforeach() +endforeach() + +string(JSON first_build GET "${first}" build) +string(JSON second_build GET "${second}" build) +string(JSON first_parameters GET "${first}" parameters) +string(JSON second_parameters GET "${second}" parameters) +if(NOT first_build STREQUAL second_build OR NOT first_parameters STREQUAL second_parameters) + message(FATAL_ERROR "Laghu benchmark expectation failed: deterministic_identity_or_parameters_invalid") +endif() + +foreach(metric IN ITEMS allocation_count laghu_syscall_count) + string(JSON no_warmup_instrumented GET "${no_warmup}" metrics ${metric} instrumented) + string(JSON extended_warmup_instrumented GET "${extended_warmup}" metrics ${metric} instrumented) + string(JSON no_warmup_value GET "${no_warmup}" metrics ${metric} value) + string(JSON extended_warmup_value GET "${extended_warmup}" metrics ${metric} value) + if(NOT no_warmup_instrumented STREQUAL extended_warmup_instrumented OR + NOT no_warmup_value EQUAL extended_warmup_value) + message(FATAL_ERROR "Laghu benchmark expectation failed: warmup_counter_leak metric=${metric}") + endif() +endforeach() + +foreach(case IN ITEMS missing-workload invalid-workload zero-intervals zero-padded-intervals + zero-tripled-intervals missing-build missing-directory) + if(case STREQUAL "missing-workload") + set(arguments --build "${BUILD_DIRECTORY}" --warmup 1 --intervals 1) + elseif(case STREQUAL "invalid-workload") + set(arguments --build "${BUILD_DIRECTORY}" --workload invalid --warmup 1 --intervals 1) + elseif(case STREQUAL "zero-intervals") + set(arguments --build "${BUILD_DIRECTORY}/missing" --workload core-foundation --warmup 1 --intervals 0) + elseif(case STREQUAL "zero-padded-intervals") + set(arguments --build "${BUILD_DIRECTORY}/missing" --workload core-foundation --warmup 1 --intervals 00) + elseif(case STREQUAL "zero-tripled-intervals") + set(arguments --build "${BUILD_DIRECTORY}/missing" --workload core-foundation --warmup 1 --intervals 000) + elseif(case STREQUAL "missing-directory") + set(arguments --build "${BUILD_DIRECTORY}/missing" --workload core-foundation --warmup 1 --intervals 1) + else() + set(arguments --workload core-foundation --warmup 1 --intervals 1) + endif() + execute_process(COMMAND "${SCRIPT}" ${arguments} + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE diagnostics) + if(case STREQUAL "missing-directory") + set(expected_exit 66) + else() + set(expected_exit 64) + endif() + if(NOT result EQUAL expected_exit) + message(FATAL_ERROR "Laghu benchmark expectation failed: case=${case}; expected_exit=${expected_exit}; actual_exit=${result}; output=${output}${diagnostics}") + endif() +endforeach() + +execute_process( + COMMAND "${SCRIPT}" --build "${BUILD_DIRECTORY}" --workload core-foundation --warmup 1 --intervals 001 + RESULT_VARIABLE padded_result + OUTPUT_VARIABLE padded_output + ERROR_VARIABLE padded_diagnostics) +if(NOT padded_result EQUAL 0) + message(FATAL_ERROR "Laghu benchmark expectation failed: nonzero_padded_intervals=${padded_output}${padded_diagnostics}") +endif() +string(JSON padded_interval_count GET "${padded_output}" parameters intervals) +if(NOT padded_interval_count EQUAL 1) + message(FATAL_ERROR "Laghu benchmark expectation failed: nonzero_padded_intervals_invalid") +endif() diff --git a/cmake/ExpectBuildIdentityDeterminism.cmake b/cmake/ExpectBuildIdentityDeterminism.cmake index 305b912..564af67 100644 --- a/cmake/ExpectBuildIdentityDeterminism.cmake +++ b/cmake/ExpectBuildIdentityDeterminism.cmake @@ -16,13 +16,14 @@ if(NOT active_binary_relative MATCHES "^\\.\\." AND NOT active_binary_relative S list(REMOVE_ITEM source_entries "${active_binary_root}") endif() endif() -foreach(copy_name IN ITEMS source-a source-b source-changed) +foreach(copy_name IN ITEMS source-a source-b source-changed source-benchmark-changed) file(MAKE_DIRECTORY "${work}/${copy_name}") foreach(entry IN LISTS source_entries) file(COPY "${LAGHU_SOURCE}/${entry}" DESTINATION "${work}/${copy_name}") endforeach() endforeach() file(APPEND "${work}/source-changed/VERSION" "\n") +file(APPEND "${work}/source-benchmark-changed/bench/runner.cpp" "\n") function(laghu_configure_identity source binary) execute_process( @@ -39,18 +40,24 @@ endfunction() laghu_configure_identity("${work}/source-a" "${work}/build-a") laghu_configure_identity("${work}/source-b" "${work}/build-b") laghu_configure_identity("${work}/source-changed" "${work}/build-changed") +laghu_configure_identity("${work}/source-benchmark-changed" "${work}/build-benchmark-changed") file(READ "${work}/build-a/config/laghu-build-manifest-v1.json" manifest_a) file(READ "${work}/build-b/config/laghu-build-manifest-v1.json" manifest_b) file(READ "${work}/build-changed/config/laghu-build-manifest-v1.json" manifest_changed) +file(READ "${work}/build-benchmark-changed/config/laghu-build-manifest-v1.json" manifest_benchmark_changed) if(NOT manifest_a STREQUAL manifest_b) message(FATAL_ERROR "Laghu build identity determinism expectation failed: equivalent_manifests_differ") endif() string(JSON build_id_a GET "${manifest_a}" build_id) string(JSON build_id_changed GET "${manifest_changed}" build_id) +string(JSON build_id_benchmark_changed GET "${manifest_benchmark_changed}" build_id) if(build_id_a STREQUAL build_id_changed) message(FATAL_ERROR "Laghu build identity determinism expectation failed: material_input_did_not_change_build_id") endif() -foreach(manifest IN ITEMS "${manifest_a}" "${manifest_changed}") +if(build_id_a STREQUAL build_id_benchmark_changed) + message(FATAL_ERROR "Laghu build identity determinism expectation failed: benchmark_input_did_not_change_build_id") +endif() +foreach(manifest IN ITEMS "${manifest_a}" "${manifest_changed}" "${manifest_benchmark_changed}") if(manifest MATCHES "${LAGHU_SOURCE}" OR manifest MATCHES "${work}") message(FATAL_ERROR "Laghu build identity determinism expectation failed: filesystem_path_leak") endif() diff --git a/cmake/ExpectReproducibleStaging.cmake b/cmake/ExpectReproducibleStaging.cmake index fd48d01..d16ce40 100644 --- a/cmake/ExpectReproducibleStaging.cmake +++ b/cmake/ExpectReproducibleStaging.cmake @@ -10,7 +10,7 @@ file(REMOVE_RECURSE "${work}") file(MAKE_DIRECTORY "${work}") function(laghu_copy_source destination) - foreach(entry IN ITEMS CMakeLists.txt CMakePresets.json VERSION cmake docs src tests) + foreach(entry IN ITEMS CMakeLists.txt CMakePresets.json VERSION bench cmake docs src tests) set(source_entry "${LAGHU_SOURCE}/${entry}") if(NOT EXISTS "${source_entry}") message(FATAL_ERROR "Laghu reproducibility expectation failed: source_entry_missing=${entry}") diff --git a/cmake/LaghuBenchmarks.cmake b/cmake/LaghuBenchmarks.cmake new file mode 100644 index 0000000..c51746f --- /dev/null +++ b/cmake/LaghuBenchmarks.cmake @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: AGPL-3.0-only +include_guard(GLOBAL) + +set(LAGHU_BENCHMARK_SCHEMA_VERSION laghu-benchmark-v1) + +function(laghu_benchmark_fail detail) + message(FATAL_ERROR "Laghu benchmark configuration failed: ${detail}") +endfunction() + +function(laghu_configure_benchmarks) + if(NOT DEFINED LAGHU_BUILD_ID OR LAGHU_BUILD_ID STREQUAL "") + laghu_benchmark_fail("rule=build_identity_missing") + endif() + if(NOT DEFINED LAGHU_BUILD_IDENTITY_FEATURES_JSON OR + NOT DEFINED LAGHU_BUILD_IDENTITY_DEPENDENCIES_JSON) + laghu_benchmark_fail("rule=build_identity_components_missing") + endif() + + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/generated/laghu") + set(identity_header "${CMAKE_BINARY_DIR}/generated/laghu/benchmark_identity.hpp") + file(WRITE "${identity_header}" +"// SPDX-License-Identifier: AGPL-3.0-only\n#pragma once\n\n#include \n\nnamespace laghu::benchmark::internal {\ninline constexpr std::string_view schema_version = \"${LAGHU_BENCHMARK_SCHEMA_VERSION}\";\ninline constexpr std::string_view build_id = \"${LAGHU_BUILD_ID}\";\ninline constexpr std::string_view compiler_id = \"${CMAKE_CXX_COMPILER_ID}\";\ninline constexpr std::string_view compiler_version = \"${CMAKE_CXX_COMPILER_VERSION}\";\ninline constexpr std::string_view target_architecture = \"${CMAKE_SYSTEM_PROCESSOR}\";\ninline constexpr std::string_view target_os = \"${CMAKE_SYSTEM_NAME}\";\ninline constexpr std::string_view features_json = R\"laghu(${LAGHU_BUILD_IDENTITY_FEATURES_JSON})laghu\";\ninline constexpr std::string_view dependencies_json = R\"laghu(${LAGHU_BUILD_IDENTITY_DEPENDENCIES_JSON})laghu\";\n} // namespace laghu::benchmark::internal\n") + set(LAGHU_BENCHMARK_IDENTITY_HEADER "${identity_header}" CACHE INTERNAL + "Laghu build-local benchmark identity header" FORCE) +endfunction() + +function(laghu_add_benchmark_targets) + if(NOT DEFINED LAGHU_BENCHMARK_IDENTITY_HEADER) + laghu_benchmark_fail("rule=identity_header_missing") + endif() + + add_executable(laghu_benchmark_core_foundation EXCLUDE_FROM_ALL + bench/core_foundation.cpp + bench/runner.cpp) + laghu_apply_first_party_contract(laghu_benchmark_core_foundation) + laghu_configure_api_consumer(laghu_benchmark_core_foundation core) + target_include_directories(laghu_benchmark_core_foundation PRIVATE + "${CMAKE_BINARY_DIR}/generated" + "${CMAKE_SOURCE_DIR}/bench/private") + target_link_libraries(laghu_benchmark_core_foundation PRIVATE laghu_core) + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + # The runner's bounded POSIX argv, write, sysctl, and /proc buffers require + # pointer-based APIs. Keep the waiver confined to this non-installed tool. + set_source_files_properties(bench/runner.cpp PROPERTIES + COMPILE_OPTIONS -Wno-unsafe-buffer-usage) + endif() + + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/config") + file(GENERATE OUTPUT "${CMAKE_BINARY_DIR}/config/laghu-benchmarks-v1.tsv" + CONTENT "# laghu-benchmarks-v1\n# workload\tcmake_target\texecutable\ncore-foundation\tlaghu_benchmark_core_foundation\t$\n") + + add_executable(laghu_benchmark_metrics_test tests/benchmarks/metrics.cpp) + laghu_apply_first_party_contract(laghu_benchmark_metrics_test) + target_include_directories(laghu_benchmark_metrics_test PRIVATE + "${CMAKE_SOURCE_DIR}/bench/private") + target_link_libraries(laghu_benchmark_metrics_test PRIVATE laghu_test_support) + laghu_add_native_test(laghu.benchmark.metrics laghu_benchmark_metrics_test) + + add_executable(laghu_benchmark_workload_counters_test tests/benchmarks/workload_counters.cpp) + laghu_apply_first_party_contract(laghu_benchmark_workload_counters_test) + target_include_directories(laghu_benchmark_workload_counters_test PRIVATE + "${CMAKE_SOURCE_DIR}/bench/private") + target_link_libraries(laghu_benchmark_workload_counters_test PRIVATE laghu_test_support) + laghu_add_native_test(laghu.benchmark.workload_counters laghu_benchmark_workload_counters_test) +endfunction() + +function(laghu_add_benchmark_validation_tests) + if(CMAKE_CROSSCOMPILING) + return() + endif() + list(LENGTH LAGHU_BUILD_EFFECTIVE_FEATURES benchmark_expected_feature_count) + list(LENGTH LAGHU_ACTIVE_DEPENDENCIES benchmark_expected_dependency_count) + add_test(NAME laghu.benchmark.runner + COMMAND "${CMAKE_COMMAND}" + "-DSCRIPT=${CMAKE_SOURCE_DIR}/scripts/benchmark" + "-DBUILD_DIRECTORY=${CMAKE_BINARY_DIR}" + "-DSOURCE_DIRECTORY=${CMAKE_SOURCE_DIR}" + "-DEXPECTED_FEATURE_COUNT=${benchmark_expected_feature_count}" + "-DEXPECTED_DEPENDENCY_COUNT=${benchmark_expected_dependency_count}" + -P "${CMAKE_SOURCE_DIR}/cmake/ExpectBenchmarkRunner.cmake") + add_test(NAME laghu.benchmark.release_exclusion + COMMAND "${CMAKE_COMMAND}" + "-DBUILD_DIRECTORY=${CMAKE_BINARY_DIR}" + "-DARCHIVE=$" + "-DEXECUTABLE=$" + "-DNM=${CMAKE_NM}" + "-DSTAGE_DIRECTORY=${CMAKE_BINARY_DIR}/tests/benchmark-install" + "-DINSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + -P "${CMAKE_SOURCE_DIR}/cmake/ExpectBenchmarkReleaseExclusion.cmake") +endfunction() diff --git a/cmake/LaghuBuildIdentity.cmake b/cmake/LaghuBuildIdentity.cmake index 57d733e..f4ee599 100644 --- a/cmake/LaghuBuildIdentity.cmake +++ b/cmake/LaghuBuildIdentity.cmake @@ -25,7 +25,12 @@ function(laghu_build_identity_input_hashes output) CMakeLists.txt CMakePresets.json VERSION + bench/core_foundation.cpp + bench/private/laghu/benchmark/internal/metrics.hpp + bench/private/laghu/benchmark/internal/workload.hpp + bench/runner.cpp cmake/LaghuApiBoundaries.cmake + cmake/LaghuBenchmarks.cmake cmake/LaghuBuildIdentity.cmake cmake/LaghuBuildVariants.cmake cmake/LaghuCapabilities.cmake @@ -37,6 +42,7 @@ function(laghu_build_identity_input_hashes output) cmake/LaghuToolchain.cmake tests/hardening/probes/clean.cpp tests/hardening/probes/fortification.cpp + tests/benchmarks/workload_counters.cpp src/cli/main.cpp src/cli/private/laghu/cli/internal/build_manifest.hpp src/core/clocks.cpp @@ -191,4 +197,8 @@ function(laghu_configure_build_identity) set(LAGHU_BUILD_IDENTITY_PREIMAGE "${preimage_path}" CACHE INTERNAL "Laghu build identity preimage") set(LAGHU_BUILD_ID "${build_id}" CACHE INTERNAL "Laghu stable build identifier") set(LAGHU_BUILD_MANIFEST_SOURCE "${generated_source}" CACHE INTERNAL "Laghu generated build manifest source") + set(LAGHU_BUILD_IDENTITY_FEATURES_JSON "${effective_json}" CACHE INTERNAL + "Laghu effective feature identity JSON" FORCE) + set(LAGHU_BUILD_IDENTITY_DEPENDENCIES_JSON "${dependencies_json}" CACHE INTERNAL + "Laghu effective dependency identity JSON" FORCE) endfunction() diff --git a/cmake/LaghuStaticAnalysis.cmake b/cmake/LaghuStaticAnalysis.cmake index 865c786..c710448 100644 --- a/cmake/LaghuStaticAnalysis.cmake +++ b/cmake/LaghuStaticAnalysis.cmake @@ -17,7 +17,7 @@ function(laghu_collect_static_analysis_targets output) endif() get_filename_component(source_absolute "${source}" ABSOLUTE BASE_DIR "${CMAKE_SOURCE_DIR}") file(RELATIVE_PATH source_relative "${CMAKE_SOURCE_DIR}" "${source_absolute}") - if(source_relative MATCHES "^src/") + if(source_relative MATCHES "^(src|bench)/") list(APPEND analysis_targets "${target}") break() endif() diff --git a/cmake/RunStaticAnalysis.cmake b/cmake/RunStaticAnalysis.cmake index f80b9bd..ccc6089 100644 --- a/cmake/RunStaticAnalysis.cmake +++ b/cmake/RunStaticAnalysis.cmake @@ -24,7 +24,7 @@ if(command_count GREATER 0) string(JSON source GET "${compile_database}" ${index} file) file(REAL_PATH "${source}" source_real) file(RELATIVE_PATH relative_source "${SOURCE}" "${source_real}") - if(relative_source MATCHES "^src/.*\\.(cpp|cc|cxx)$") + if(relative_source MATCHES "^(src|bench)/.*\\.(cpp|cc|cxx)$") list(APPEND owned_sources "${source_real}") list(APPEND owned_command_indexes "${index}") endif() @@ -36,7 +36,9 @@ if(owned_sources STREQUAL "") endif() file(TO_CMAKE_PATH "${SOURCE}/src" owned_header_root) string(REGEX REPLACE "([][+.*^$(){}|\\\\?])" "\\\\\\1" owned_header_root_regex "${owned_header_root}") -set(owned_header_filter "^${owned_header_root_regex}/.*") +file(TO_CMAKE_PATH "${SOURCE}/bench" benchmark_header_root) +string(REGEX REPLACE "([][+.*^$(){}|\\\\?])" "\\\\\\1" benchmark_header_root_regex "${benchmark_header_root}") +set(owned_header_filter "^(${owned_header_root_regex}|${benchmark_header_root_regex})/.*") execute_process( COMMAND "${CLANG_TIDY}" "--config-file=${CLANG_TIDY_CONFIG}" @@ -53,7 +55,7 @@ file(STRINGS "${CPPCHECK_CONFIG}" cppcheck_options) list(FILTER cppcheck_options EXCLUDE REGEX "^[ \\t]*(#|$)") execute_process( COMMAND "${CPPCHECK}" ${cppcheck_options} "--project=${compile_commands}" - "--file-filter=${SOURCE}/src/*" + "--file-filter=${SOURCE}/src/*" "--file-filter=${SOURCE}/bench/*" RESULT_VARIABLE cppcheck_result OUTPUT_VARIABLE cppcheck_output ERROR_VARIABLE cppcheck_diagnostics) diff --git a/scripts/benchmark b/scripts/benchmark new file mode 100755 index 0000000..c6de5eb --- /dev/null +++ b/scripts/benchmark @@ -0,0 +1,71 @@ +#!/bin/sh +# SPDX-License-Identifier: AGPL-3.0-only + +usage() { + printf '%s\n' 'usage: scripts/benchmark --build --workload --warmup --intervals ' >&2 +} + +build_directory='' +workload='' +warmup='' +intervals='' + +while [ "$#" -gt 0 ]; do + case "$1" in + --build) + if [ "$#" -lt 2 ] || [ -n "$build_directory" ] || [ -z "$2" ]; then usage; exit 64; fi + build_directory=$2; shift 2 ;; + --workload) + if [ "$#" -lt 2 ] || [ -n "$workload" ] || [ -z "$2" ]; then usage; exit 64; fi + workload=$2; shift 2 ;; + --warmup) + if [ "$#" -lt 2 ] || [ -n "$warmup" ]; then usage; exit 64; fi + case "$2" in ''|*[!0-9]*) usage; exit 64 ;; esac + warmup=$2; shift 2 ;; + --intervals) + if [ "$#" -lt 2 ] || [ -n "$intervals" ]; then usage; exit 64; fi + case "$2" in + ''|*[!0-9]*) usage; exit 64 ;; + *[1-9]*) ;; + *) usage; exit 64 ;; + esac + intervals=$2; shift 2 ;; + *) usage; exit 64 ;; + esac +done + +if [ -z "$build_directory" ] || [ -z "$workload" ] || [ -z "$warmup" ] || [ -z "$intervals" ]; then + usage + exit 64 +fi +if [ ! -d "$build_directory" ]; then + printf '%s\n' "scripts/benchmark: build directory does not exist: $build_directory" >&2 + exit 66 +fi +if ! command -v cmake >/dev/null 2>&1; then + printf '%s\n' 'scripts/benchmark: cmake is unavailable' >&2 + exit 69 +fi + +registry="$build_directory/config/laghu-benchmarks-v1.tsv" +if [ ! -r "$registry" ]; then + printf '%s\n' "scripts/benchmark: benchmark registry is unavailable: $registry" >&2 + exit 66 +fi +record=$(awk -F '\t' -v requested="$workload" '$1 == requested && NF == 3 { print $2 "\t" $3; exit }' "$registry") +if [ -z "$record" ]; then + printf '%s\n' "scripts/benchmark: workload is not configured: $workload" >&2 + exit 64 +fi +cmake_target=$(printf '%s\n' "$record" | awk -F '\t' '{ print $1 }') +executable=$(printf '%s\n' "$record" | awk -F '\t' '{ print $2 }') + +if ! cmake --build "$build_directory" --target "$cmake_target" >&2; then + printf '%s\n' "scripts/benchmark: workload build failed: $workload" >&2 + exit 70 +fi +if [ ! -x "$executable" ]; then + printf '%s\n' "scripts/benchmark: configured executable is unavailable: $workload" >&2 + exit 70 +fi +exec "$executable" --workload "$workload" --warmup "$warmup" --intervals "$intervals" diff --git a/tests/benchmarks/metrics.cpp b/tests/benchmarks/metrics.cpp new file mode 100644 index 0000000..66c0671 --- /dev/null +++ b/tests/benchmarks/metrics.cpp @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: AGPL-3.0-only +#include +#include + +#include + +#include "laghu_test_support.hpp" + +namespace { + +[[nodiscard]] bool check_nearest_rank_percentiles() noexcept { + std::array samples{10U, 2U, 8U, 4U, 6U, 1U, 9U, 3U, 7U, 5U}; + laghu::benchmark::internal::Percentiles summary{}; + return laghu::benchmark::internal::summarize_percentiles(samples, samples.size(), summary) && + summary.p50 == 5U && summary.p95 == 10U && summary.p99 == 10U && summary.p999 == 10U; +} + +[[nodiscard]] bool check_single_sample_percentiles() noexcept { + std::array samples{42U, 0U}; + laghu::benchmark::internal::Percentiles summary{}; + return laghu::benchmark::internal::summarize_percentiles(samples, 1U, summary) && + summary.p50 == 42U && summary.p95 == 42U && summary.p99 == 42U && summary.p999 == 42U; +} + +[[nodiscard]] bool check_invalid_samples_are_rejected() noexcept { + std::array samples{1U}; + laghu::benchmark::internal::Percentiles summary{}; + return !laghu::benchmark::internal::summarize_percentiles(samples, 0U, summary) && + !laghu::benchmark::internal::summarize_percentiles(samples, 2U, summary); +} + +} // namespace + +int main() { + static constexpr std::array tests{ + laghu::test::TestCase{"benchmark.nearest-rank", check_nearest_rank_percentiles}, + laghu::test::TestCase{"benchmark.single-sample", check_single_sample_percentiles}, + laghu::test::TestCase{"benchmark.invalid-samples", check_invalid_samples_are_rejected}, + }; + return laghu::test::run_tests(tests); +} diff --git a/tests/benchmarks/workload_counters.cpp b/tests/benchmarks/workload_counters.cpp new file mode 100644 index 0000000..7fa1cbc --- /dev/null +++ b/tests/benchmarks/workload_counters.cpp @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: AGPL-3.0-only +#include +#include +#include + +#include + +#include "laghu_test_support.hpp" + +namespace { + +[[nodiscard]] bool run_instrumented_fixture( + laghu::benchmark::internal::WorkloadCounters& counters) noexcept { + return counters.record_allocation_events(2U) && counters.record_laghu_syscall_events(3U); +} + +[[nodiscard]] bool run_fixture(std::uint64_t warmup_intervals, std::uint64_t measured_intervals, + laghu::benchmark::internal::WorkloadCounters& reported) noexcept { + laghu::benchmark::internal::WorkloadCounters warmup{}; + for (std::uint64_t interval = 0U; interval < warmup_intervals; ++interval) { + if (!run_instrumented_fixture(warmup)) { + return false; + } + } + for (std::uint64_t interval = 0U; interval < measured_intervals; ++interval) { + if (!run_instrumented_fixture(reported)) { + return false; + } + } + return true; +} + +[[nodiscard]] bool check_exact_instrumentation_accounting() noexcept { + laghu::benchmark::internal::WorkloadCounters counters{}; + return !counters.allocation_instrumented() && !counters.laghu_syscall_instrumented() && + counters.allocation_count() == 0U && counters.laghu_syscall_count() == 0U && + counters.record_allocation_events(0U) && counters.record_laghu_syscall_events(0U) && + !counters.allocation_instrumented() && !counters.laghu_syscall_instrumented() && + run_instrumented_fixture(counters) && counters.allocation_instrumented() && + counters.laghu_syscall_instrumented() && counters.allocation_count() == 2U && + counters.laghu_syscall_count() == 3U; +} + +[[nodiscard]] bool check_warmup_counters_are_not_reported() noexcept { + laghu::benchmark::internal::WorkloadCounters no_warmup{}; + laghu::benchmark::internal::WorkloadCounters extended_warmup{}; + return run_fixture(0U, 4U, no_warmup) && run_fixture(7U, 4U, extended_warmup) && + no_warmup.allocation_instrumented() && no_warmup.laghu_syscall_instrumented() && + extended_warmup.allocation_instrumented() && extended_warmup.laghu_syscall_instrumented() && + no_warmup.allocation_count() == 8U && no_warmup.laghu_syscall_count() == 12U && + extended_warmup.allocation_count() == 8U && extended_warmup.laghu_syscall_count() == 12U; +} + +[[nodiscard]] bool check_counter_overflow_is_rejected() noexcept { + laghu::benchmark::internal::WorkloadCounters counters{}; + constexpr std::uint64_t maximum = std::numeric_limits::max(); + return counters.record_allocation_events(maximum) && !counters.record_allocation_events() && + counters.allocation_instrumented() && counters.allocation_count() == maximum && + counters.record_laghu_syscall_events(maximum - 1U) && + !counters.record_laghu_syscall_events(2U) && counters.laghu_syscall_instrumented() && + counters.laghu_syscall_count() == maximum - 1U; +} + +} // namespace + +int main() { + static constexpr std::array tests{ + laghu::test::TestCase{"benchmark.workload-counters.exact", check_exact_instrumentation_accounting}, + laghu::test::TestCase{"benchmark.workload-counters.warmup", check_warmup_counters_are_not_reported}, + laghu::test::TestCase{"benchmark.workload-counters.overflow", check_counter_overflow_is_rejected}, + }; + return laghu::test::run_tests(tests); +}