From 9f3e9ca624d896f5e69aff14b819039b34f5130b Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 24 Sep 2026 14:01:25 +0200 Subject: [PATCH 1/4] ci: declare FLAKY_BENCHMARKS_REGEX and document benchmarks CI Co-Authored-By: Claude Opus 5.5 --- .gitlab/BENCHMARKS.md | 30 ++++++++++++++++++++++++++++++ .gitlab/benchmarks.yml | 2 ++ 2 files changed, 32 insertions(+) create mode 100644 .gitlab/BENCHMARKS.md diff --git a/.gitlab/BENCHMARKS.md b/.gitlab/BENCHMARKS.md new file mode 100644 index 000000000..9546e0fa1 --- /dev/null +++ b/.gitlab/BENCHMARKS.md @@ -0,0 +1,30 @@ +# Benchmarks + +GitLab CI configuration for the benchmarks that run on the +[Benchmarking Platform](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/2419261562/Benchmarking+Platform). + +## Layout + +- `benchmarks.yml`: Google Benchmark microbenchmarks. + - `benchmarks` clones `benchmarking-platform` (`dd-trace-cpp` branch), runs the benchmark, + converts and analyzes results, uploads and comments on the PR. + - `check-big-regressions` fails on regressions above the threshold defined on + `bp-runner.fail-on-regression.yml` in the `dd-trace-cpp` branch of + [benchmarking-platform](https://github.com/DataDog/benchmarking-platform). + +## Marking a benchmark as flaky + +Add it to `FLAKY_BENCHMARKS_REGEX` in the `benchmarks` job's `variables` in `benchmarks.yml`. + +The benchmark still runs and reports, but doesn't fail the gate. + +- The regex matches anywhere in the scenario name. + - `BM_Trace` quarantines every scenario containing it, like `BM_TraceTinyCCSource`. + - Anchor with `^...$` to target one scenario. + +```yaml +FLAKY_BENCHMARKS_REGEX: "^BM_TraceTinyCCSource$" +``` + +Open a ticket to fix or remove it. See +[Flaky Benchmarks Monitoring](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring). diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 8e5184467..7a864278d 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -32,6 +32,8 @@ benchmarks: UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-cpp" UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built. UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for. + # See https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring for more details. + FLAKY_BENCHMARKS_REGEX: "" # This repository is using PR-level performance quality gates. # Verify that the check-big-regressions CI job has passed. If any regression happened, merging this PR will be blocked. From 7ca83b880bc84cf7681ae67647863046370200b9 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 1 Oct 2026 15:29:15 +0200 Subject: [PATCH 2/4] docs: use permanent confluence links --- .gitlab/BENCHMARKS.md | 2 +- .gitlab/benchmarks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/BENCHMARKS.md b/.gitlab/BENCHMARKS.md index 9546e0fa1..491939d0b 100644 --- a/.gitlab/BENCHMARKS.md +++ b/.gitlab/BENCHMARKS.md @@ -27,4 +27,4 @@ FLAKY_BENCHMARKS_REGEX: "^BM_TraceTinyCCSource$" ``` Open a ticket to fix or remove it. See -[Flaky Benchmarks Monitoring](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring). +[Flaky Benchmarks Monitoring](https://datadoghq.atlassian.net/wiki/x/dAKLrgE) for more details. diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 7a864278d..14250c46d 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -32,7 +32,7 @@ benchmarks: UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-cpp" UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built. UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for. - # See https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring for more details. + # See https://datadoghq.atlassian.net/wiki/x/dAKLrgE for more details. FLAKY_BENCHMARKS_REGEX: "" # This repository is using PR-level performance quality gates. From de8128146ba9d88bd7618c75fca9745560f95660 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 1 Oct 2026 15:30:21 +0200 Subject: [PATCH 3/4] docs: improved wording for bench doc intro --- .gitlab/BENCHMARKS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/BENCHMARKS.md b/.gitlab/BENCHMARKS.md index 491939d0b..6cfa3ab99 100644 --- a/.gitlab/BENCHMARKS.md +++ b/.gitlab/BENCHMARKS.md @@ -1,6 +1,6 @@ # Benchmarks -GitLab CI configuration for the benchmarks that run on the +This describes the GitLab CI configuration for the benchmarks that run on the [Benchmarking Platform](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/2419261562/Benchmarking+Platform). ## Layout From bf73099a99fbeefcf4c6de1627ce1d4979cbe978 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 1 Oct 2026 15:30:35 +0200 Subject: [PATCH 4/4] docs: capitalized title --- .gitlab/BENCHMARKS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/BENCHMARKS.md b/.gitlab/BENCHMARKS.md index 6cfa3ab99..efcec2fdc 100644 --- a/.gitlab/BENCHMARKS.md +++ b/.gitlab/BENCHMARKS.md @@ -12,7 +12,7 @@ This describes the GitLab CI configuration for the benchmarks that run on the `bp-runner.fail-on-regression.yml` in the `dd-trace-cpp` branch of [benchmarking-platform](https://github.com/DataDog/benchmarking-platform). -## Marking a benchmark as flaky +## Marking a Benchmark as Flaky Add it to `FLAKY_BENCHMARKS_REGEX` in the `benchmarks` job's `variables` in `benchmarks.yml`.