From fafcb0271a0d22e46b09642d7d69483ce9c09417 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Thu, 24 Sep 2026 14:01:26 +0200 Subject: [PATCH 1/3] ci: declare FLAKY_BENCHMARKS_REGEX and document benchmarks CI Co-Authored-By: Claude Opus 5.5 --- .gitlab/BENCHMARKS.md | 31 +++++++++++++++++++++++++++++++ .gitlab/benchmarks.yml | 3 +++ 2 files changed, 34 insertions(+) create mode 100644 .gitlab/BENCHMARKS.md diff --git a/.gitlab/BENCHMARKS.md b/.gitlab/BENCHMARKS.md new file mode 100644 index 00000000..54f0050a --- /dev/null +++ b/.gitlab/BENCHMARKS.md @@ -0,0 +1,31 @@ +# Benchmarks + +GitLab CI configuration for the benchmarks that run on the +[Benchmarking Platform](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/2419261562/Benchmarking+Platform). + +## Layout + +- `benchmarks.yml`: k6 load tests against an Nginx server with the Datadog module. + - `baseline` and `only-tracing` run the `normal_operation` and `high_load` scenarios via the + `.benchmarks` template. + - `check-slo-breaches` gates releases based on SLOs defined on `bp-runner.fail-on-breach.yml`. + - Steps live in the `cpp/nginx` branch of + [benchmarking-platform](https://github.com/DataDog/benchmarking-platform). + +## Marking a benchmark as flaky + +Add it to `FLAKY_BENCHMARKS_REGEX` in `.benchmarks` in `benchmarks.yml`. + +The benchmark still runs and reports, but doesn't fail the gate. + +- The regex matches anywhere in the scenario name. + - `normal_operation` quarantines every `normal_operation` scenario, across both + `baseline` and `only-tracing`. + - Anchor with `^...$` to target one scenario. + +```yaml +FLAKY_BENCHMARKS_REGEX: "^high_load--only-tracing--nginx-utilization$" +``` + +Open a ticket to fix or remove it. See +[Flaky Benchmarks Monitoring](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring). diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 55b9146f..51dcb951 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -70,6 +70,9 @@ build-nginx-module: DD_TRACE_DEBUG: "false" DD_RUNTIME_METRICS_ENABLED: "true" + # See https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring for more details. + FLAKY_BENCHMARKS_REGEX: "" + DD_BENCHMARKS_NGINX_IMAGE_TAG: amd64-${NGINX_VERSION} K6_OPTIONS_NORMAL_OPERATION_RATE: 1000 From 48cf81dfa5acf948e817549118d82b27c2bddc5d Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Tue, 29 Sep 2026 14:51:45 +0200 Subject: [PATCH 2/3] docs: name the quality gates flaky benchmarks skip Co-Authored-By: Claude Sonnet 5.5 --- .gitlab/BENCHMARKS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/BENCHMARKS.md b/.gitlab/BENCHMARKS.md index 54f0050a..263758b4 100644 --- a/.gitlab/BENCHMARKS.md +++ b/.gitlab/BENCHMARKS.md @@ -16,7 +16,8 @@ GitLab CI configuration for the benchmarks that run on the Add it to `FLAKY_BENCHMARKS_REGEX` in `.benchmarks` in `benchmarks.yml`. -The benchmark still runs and reports, but doesn't fail the gate. +The benchmark still runs and reports, but doesn't fail the `check-slo-breaches` performance +quality gate (SLO-based). - The regex matches anywhere in the scenario name. - `normal_operation` quarantines every `normal_operation` scenario, across both From f728b302e4be43f5f1f7d2c35082b1a53cc655c6 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Fri, 2 Oct 2026 09:54:11 +0200 Subject: [PATCH 3/3] docs: fixes for clarity --- .gitlab/BENCHMARKS.md | 8 ++++---- .gitlab/benchmarks.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/BENCHMARKS.md b/.gitlab/BENCHMARKS.md index 263758b4..3d801320 100644 --- a/.gitlab/BENCHMARKS.md +++ b/.gitlab/BENCHMARKS.md @@ -1,7 +1,7 @@ # Benchmarks -GitLab CI configuration for the benchmarks that run on the -[Benchmarking Platform](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/2419261562/Benchmarking+Platform). +This describes the GitLab CI configuration for the benchmarks that run on the +[Benchmarking Platform](https://datadoghq.atlassian.net/wiki/x/egAzk). ## Layout @@ -12,7 +12,7 @@ GitLab CI configuration for the benchmarks that run on the - Steps live in the `cpp/nginx` branch of [benchmarking-platform](https://github.com/DataDog/benchmarking-platform). -## Marking a benchmark as flaky +## Marking a Benchmark as Flaky Add it to `FLAKY_BENCHMARKS_REGEX` in `.benchmarks` in `benchmarks.yml`. @@ -29,4 +29,4 @@ FLAKY_BENCHMARKS_REGEX: "^high_load--only-tracing--nginx-utilization$" ``` Open a ticket to fix or remove it. See -[Flaky Benchmarks Monitoring](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring). +[Flaky Benchmarks Monitoring](https://datadoghq.atlassian.net/wiki/x/dAKLrgE). diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 5dcdf556..3574d765 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -70,7 +70,7 @@ build-nginx-module: DD_TRACE_DEBUG: "false" DD_RUNTIME_METRICS_ENABLED: "true" - # See https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/7223313012/Flaky+Benchmarks+Monitoring for more details. + # See https://datadoghq.atlassian.net/wiki/x/dAKLrgE for more details. FLAKY_BENCHMARKS_REGEX: "" DD_BENCHMARKS_NGINX_IMAGE_TAG: amd64-${NGINX_VERSION}