From a5de38b81f84cee4579fadfa0f4227d7552fd00e Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Mon, 31 Aug 2026 14:35:03 +0200 Subject: [PATCH 1/9] Fix various minor typos --- src/datadog/limiter.h | 10 +++--- src/datadog/random.cpp | 2 +- src/datadog/remote_config/remote_config.h | 4 +-- src/datadog/telemetry/telemetry_impl.h | 42 ++++++++++------------- 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/datadog/limiter.h b/src/datadog/limiter.h index ddddb5efb..7071f90aa 100644 --- a/src/datadog/limiter.h +++ b/src/datadog/limiter.h @@ -1,12 +1,10 @@ #pragma once -// This component provides a `class`, `Limiter`, that is an implementation of -// the [token bucket][1] rate limiter. +// The `Limiter` class is an implementation of the [token +// bucket](https://en.wikipedia.org/wiki/Token_bucket) rate limiter. // -// `Limiter` is used by the `TraceSampler` and the `SpanSampler` to enforce -// their respective `max_per_second` configuration parameters. -// -// [1]: https://en.wikipedia.org/wiki/Token_bucket +// `Limiter` is used by the `TraceSampler` and the `SpanSampler` to enforce their respective +// `max_per_second` configuration parameters. #include #include diff --git a/src/datadog/random.cpp b/src/datadog/random.cpp index 78f52e65f..6d52df538 100644 --- a/src/datadog/random.cpp +++ b/src/datadog/random.cpp @@ -22,7 +22,7 @@ class Uint64Generator { // If a process links to this library and then calls `fork`, the // `generator_` in the parent and child processes will produce the exact // same sequence of values, which is bad. - // A subsequent call to `exec` would remedy this, but nginx in particular + // A subsequent call to `exec` would remedy this, but Nginx in particular // does not call `exec` after forking its worker processes. // So, we use `at_fork_in_child` to re-seed `generator_` in the child // process after `fork`. diff --git a/src/datadog/remote_config/remote_config.h b/src/datadog/remote_config/remote_config.h index b63a428be..380dda8dd 100644 --- a/src/datadog/remote_config/remote_config.h +++ b/src/datadog/remote_config/remote_config.h @@ -1,11 +1,11 @@ #pragma once // Remote Configuration is a Datadog capability that allows a user to remotely -// configure and change the behaviour of the tracing library. +// configure and change the behavior of the tracing library. // The current implementation is restricted to Application Performance // Monitoring features. // -// The `RemoteConfigurationManager` class implement the protocol to query, +// The `RemoteConfigurationManager` class implements the protocol to query, // process and verify configuration from a remote source. It is also // responsible for handling configuration updates received from a remote source // and maintains the state of applied configuration. diff --git a/src/datadog/telemetry/telemetry_impl.h b/src/datadog/telemetry/telemetry_impl.h index 59503565e..8e0f4c2ec 100644 --- a/src/datadog/telemetry/telemetry_impl.h +++ b/src/datadog/telemetry/telemetry_impl.h @@ -21,18 +21,16 @@ namespace datadog::telemetry { using MetricSnapshot = std::vector>; -/// The telemetry class is responsible for handling internal telemetry data to -/// track Datadog product usage. It _can_ collect and report logs and metrics. -/// -/// NOTE(@dmehala): The current implementation can lead a significant amount -/// of overhead if the mutext is highly disputed. Unless this is proven to be -/// indeed a bottleneck, I'll embrace KISS principle. However, in a future -/// iteration we could use multiple producer single consumer queue or -/// lock-free queue. +// The `Telemetry` class is responsible for handling internal telemetry data to track Datadog +// product usage. It _can_ collect and report logs and metrics. +// +// The current implementation can lead a significant amount of overhead if the mutex is highly +// disputed. Unless this is proven to be indeed a bottleneck, we embrace KISS principle. However, in +// a future iteration we could use multiple producers - single consumer queue or lock-free queue. class Telemetry final : public std::enable_shared_from_this { - /// Configuration object containing the validated settings for telemetry + // Configuration object containing the validated settings for telemetry FinalizedConfiguration config_; - /// Shared pointer to the user logger instance. + // Shared pointer to the user logger instance std::shared_ptr logger_; std::vector tasks_; tracing::HTTPClient::URL telemetry_endpoint_; @@ -41,23 +39,23 @@ class Telemetry final : public std::enable_shared_from_this { tracing::Clock clock_; std::shared_ptr scheduler_; - /// Counter + // Counter std::mutex counter_mutex_; std::unordered_map, uint64_t> counters_; std::unordered_map, MetricSnapshot> counters_snapshot_; - /// Rate + // Rate std::mutex rate_mutex_; std::unordered_map, uint64_t> rates_; std::unordered_map, MetricSnapshot> rates_snapshot_; - /// Distribution - /// TODO: split distribution in array of N element? + // Distribution + // TODO: split distribution in array of N element? std::mutex distributions_mutex_; std::unordered_map, std::vector> distributions_; - /// Configuration + // Configuration std::vector configuration_snapshot_; std::mutex log_mutex_; @@ -97,15 +95,11 @@ class Telemetry final : public std::enable_shared_from_this { tracing::Clock clock = tracing::default_clock); public: - /// Capture and report internal error message to Datadog. - /// - /// @param message The error message. + // Capture and report internal error message to Datadog. void log_error(std::string message); void log_error(std::string message, std::string stacktrace); - /// capture and report internal warning message to Datadog. - /// - /// @param message The warning message to log. + // Capture and report internal warning message to Datadog. void log_warning(std::string message); void send_configuration_change(); @@ -119,7 +113,7 @@ class Telemetry final : public std::enable_shared_from_this { // After this call the Telemetry object is inert and safe to destroy. void shutdown(); - /// Counter + // Counter void increment_counter(const Counter& counter); void increment_counter(const Counter& counter, const std::vector& tags); @@ -130,12 +124,12 @@ class Telemetry final : public std::enable_shared_from_this { void set_counter(const Counter& counter, const std::vector& tags, uint64_t value); - /// Rate + // Rate void set_rate(const Rate& rate, uint64_t value); void set_rate(const Rate& rate, const std::vector& tags, uint64_t value); - /// Distribution + // Distribution void add_datapoint(const Distribution& distribution, uint64_t value); void add_datapoint(const Distribution& distribution, const std::vector& tags, uint64_t value); From 43d01efe6a3e5ea09e2dce8ca1a64734fefc7b60 Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Mon, 31 Aug 2026 14:42:30 +0200 Subject: [PATCH 2/9] Add 'Thread Safety' documentation --- docs/README.md | 3 ++- docs/thread-safety.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 docs/thread-safety.md diff --git a/docs/README.md b/docs/README.md index f9ff5d69e..3967d2d05 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,7 @@ This directory contains documentation of the Datadog C++ Tracer, including: -- [Design](design.md) - [Conventions](conventions.md) +- [Design](design.md) - [Development Processes](development.md) +- [Thread Safety](thread-safety.md) diff --git a/docs/thread-safety.md b/docs/thread-safety.md new file mode 100644 index 000000000..2048e29af --- /dev/null +++ b/docs/thread-safety.md @@ -0,0 +1 @@ +# Datadog C++ Tracer Thread Safety From dcc7a294de6ade266ba96b820f0149dfe9d4b964 Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Mon, 31 Aug 2026 15:25:58 +0200 Subject: [PATCH 3/9] Add 'Architecture Principles' in 'Thread Safety' documentation --- docs/thread-safety.md | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/thread-safety.md b/docs/thread-safety.md index 2048e29af..362a3c720 100644 --- a/docs/thread-safety.md +++ b/docs/thread-safety.md @@ -1 +1,46 @@ # Datadog C++ Tracer Thread Safety + +## Architecture Principles + +- **Immutability**. The `Tracer` and the objects passed into a `TraceSegment` at construction are +set once and never mutated afterward. +- **Delegation**. All mutable shared state is in an explicit set of classes that each owns a + `std::mutex`: + - `CerrLogger` + - `ConfigManager` + - `Curl` + - `DatadogAgent` + - `SpanSampler::SynchronizedLimiter` + - `Telemetry` + - `ThreadedEventScheduler` + - `TraceSampler` + - `TraceSegment` +- `Span`/`SpanData`/`Tracer` carry no internal lock. `Span` is move-only and non-reassignable. The + contract is: **one `Span`, one owner thread at a time**. Concurrency is meant to happen between + sibling `Span`s of the same `TraceSegment`. +- **Use `std::mutex`, not `std::atomic`**: + - Every synchronized class uses plain `std::mutex` + `lock_guard`/`unique_lock` (no usage of `std::atomic`), for simplicity. + - No logging while holding a process-wide singleton's lock, because of potentially slow custom + `Logger`. +- **The default pluggable interfaces add extra threads.** The default `HTTPClient`, which is `Curl`, and the default `EventScheduler`, which is `ThreadedEventScheduler`, add each one a dedicated thread. +- `Fork` (such as Nginx/Apache pre-fork workers). Because background threads are not automatically + fork-safe, the embedder must construct the `Tracer` (and therefore any default + `Curl`/`ThreadedEventScheduler`) strictly after `fork()`. +- **`Collector` sharing across multiple `Tracer`s**. This is the intended way to fan many + threads/`Tracer`s into one flush pipeline. This is synchronized via `DatadogAgent`'s own mutex. +- **Three process-wide singletons, exceptions to the independence of `Tracer`s**: + - `telemetry::instance()`: the first `Tracer` (of the process) configures the `Telemetry` used by + all `Tracers`. + - `root_session_id::get_or_init()` is meant to be caller-coordinated. Integrations (notably Nginx + and Apache) should set this in the master process before workers fork so all `Tracer`s share the + same root. + - `OtelCtxRegistration` is a mutex-guarded singleton publishing the OpenTelemetry process context. + Each `Tracer` registers/unregisters on construction/destruction. The first `Tracer`'s fields + win. `service_instance_id` is published only while all live `Tracer`s agree on it. +- **Shutdown discipline**. The `DatadogAgent` destructor cancels scheduled tasks and waits for + outstanding HTTP requests. `ThreadedEventScheduler`'s cancellation closure blocks until any + in-flight callback finishes. `Curl`'s destructor joins its thread. This avoids callbacks firing + into a destroyed object. A custom `HTTPClient`/`EventScheduler` must replicate this or accept + trace loss on shutdown. +- **AppSec/WAF-style thread-pool offload**. This Nginx feature stresses the core library's + unsynchronized `Span` (see details below). From 6fd933881d00ac81799080892708dfa2ee88f954 Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Mon, 31 Aug 2026 15:43:58 +0200 Subject: [PATCH 4/9] Add 'Take-aways for Library Users' section in 'Thread Safety' documentation --- docs/thread-safety.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/thread-safety.md b/docs/thread-safety.md index 362a3c720..2a5c9e476 100644 --- a/docs/thread-safety.md +++ b/docs/thread-safety.md @@ -44,3 +44,26 @@ set once and never mutated afterward. trace loss on shutdown. - **AppSec/WAF-style thread-pool offload**. This Nginx feature stresses the core library's unsynchronized `Span` (see details below). + +## Take-aways for Library Users + +- It is safe to share one `Tracer` across many threads. `create_span()`/`extract_span()` can be + called concurrently. This is the primary supported model. +- It is safe to create/finish sibling `Span`s of the same `TraceSegment` concurrently across + threads. +- It is unsafe to use a single `Span` object from two threads at once. Either transfer ownership + completely (moves are supported) or build your own handoff protocol (see, for example, + `nginx-datadog`'s WAF thread-pool integration: atomic release/acquire flag + swapping out the + request's event handlers so the main thread can't touch it mid-flight). +- Never construct a `Tracer` before your process forks. Construct it after, in each child. +- Multiple `Tracer`s in one process share one telemetry pipeline. +- Multiple `Tracer`s in one process will end up with a `root_session_id` decided by whichever + `Tracer` happened to construct first. To avoid this, you can you explicitly set + `TracerConfig::root_session_id` (for example, Nginx and Apache both compute it once, pre-fork, + then pass it explicitly). +- Multiple `Tracer`s in one process publish one shared OpenTelemetry process context. The first + `Tracer`'s fields win. The shared `runtime_id` is published only while every `Tracer` agrees on + it. +- If you supply your own `Collector`/`HTTPClient`/`EventScheduler`/`IDGenerator`/`Logger`, you must + ensure it is safe to be called from multiple threads. The library will call it from whatever + threads its other pluggable pieces run on. From f59d14059489459d61b04b63e869a37546a1ee7b Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Mon, 31 Aug 2026 16:05:01 +0200 Subject: [PATCH 5/9] Add 'Web Reverse Proxies Integrations' section in 'Thread Safety' documentation --- docs/thread-safety.md | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/thread-safety.md b/docs/thread-safety.md index 2a5c9e476..099ec99f8 100644 --- a/docs/thread-safety.md +++ b/docs/thread-safety.md @@ -67,3 +67,63 @@ set once and never mutated afterward. - If you supply your own `Collector`/`HTTPClient`/`EventScheduler`/`IDGenerator`/`Logger`, you must ensure it is safe to be called from multiple threads. The library will call it from whatever threads its other pluggable pieces run on. + +## Web Reverse Proxies Integrations + +The Datadog C++ Tracer was notably designed with the following three integrations in mind. Thus, +they serve both as examples of different threading models and as projects for validating changes to +the Tracer. + +### Datadog Nginx Module + +See [nginx-datadog](https://github.com/DataDog/nginx-datadog). + +Process / Thread Model: + +- Nginx has a **master process** which forks into several **worker processes**. +- Each worker process handles many connections at once in a **single thread** (with one exception: + the optional security/WAF analysis runs in a side thread pool). +- Each worker process creates its `Tracer`. + +It has a custom `NgxEventScheduler`, which runs on the worker's own event loop. + +It has a custom logger locking. + +The `root_session_id` is set explicitly, generated once pre-fork. + +The WAF thread pool mutates a `Span` from a non-owning thread (in `Context::run_waf_start()` , +`Context::run_waf_req_post()` and `Context::do_on_main_log_request()`). It is safe by an ad hoc +protocol: handler swap (`Context::replace_handlers()`), and `std::atomic ran_on_thread_` +release (`Context::handle()`) / acquire (`Context::complete()`). + +### Datadog Apache Httpd Module + +See [httpd-datadog](https://github.com/DataDog/httpd-datadog). + +Process / Thread Model: + +- Apache has a **master process** which forks into several **child processes**. +- Depending on the configuration, the child processes can be single-threaded or **multi-threaded**. +- Each child process creates its `Tracer`, shared by every thread in it. + +The `root_session_id` and `runtime_id` are set explicitly pre-fork. + +It has a custom logger locking. + +The `Span`s are allocated on the heap and tied to the request's Apache Pre-Request (APR) memory pool (and so automatically deleted when the request finishes). + +### Datadog Envoy Extension + +See +[envoyproxy/envoy/source/extensions/tracers/datadog](https://github.com/envoyproxy/envoy/tree/main/source/extensions/tracers/datadog). + +Process / Thread Model: + +- Envoy has a **single process**, with several **worker threads**. +- Each worker thread creates its `Tracer`. + +It uses a custom `AgentHTTPClient` and a custom `EventScheduler`. They are bound to the owning `Dispatcher`, with no extra thread. + +It uses Envoy’s own logging. + +Envoy is the only integration where `OtelCtxRegistration`'s multi-Tracer bookkeeping is actually exercised. From 17d348c2615a4b1794675310be006ab6bf213299 Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Mon, 31 Aug 2026 16:15:35 +0200 Subject: [PATCH 6/9] format code --- src/datadog/limiter.h | 4 ++-- src/datadog/telemetry/telemetry_impl.h | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/datadog/limiter.h b/src/datadog/limiter.h index 7071f90aa..b0cbb95a4 100644 --- a/src/datadog/limiter.h +++ b/src/datadog/limiter.h @@ -3,8 +3,8 @@ // The `Limiter` class is an implementation of the [token // bucket](https://en.wikipedia.org/wiki/Token_bucket) rate limiter. // -// `Limiter` is used by the `TraceSampler` and the `SpanSampler` to enforce their respective -// `max_per_second` configuration parameters. +// `Limiter` is used by the `TraceSampler` and the `SpanSampler` to enforce +// their respective `max_per_second` configuration parameters. #include #include diff --git a/src/datadog/telemetry/telemetry_impl.h b/src/datadog/telemetry/telemetry_impl.h index 8e0f4c2ec..428ab60c7 100644 --- a/src/datadog/telemetry/telemetry_impl.h +++ b/src/datadog/telemetry/telemetry_impl.h @@ -21,12 +21,13 @@ namespace datadog::telemetry { using MetricSnapshot = std::vector>; -// The `Telemetry` class is responsible for handling internal telemetry data to track Datadog -// product usage. It _can_ collect and report logs and metrics. +// The `Telemetry` class is responsible for handling internal telemetry data to +// track Datadog product usage. It _can_ collect and report logs and metrics. // -// The current implementation can lead a significant amount of overhead if the mutex is highly -// disputed. Unless this is proven to be indeed a bottleneck, we embrace KISS principle. However, in -// a future iteration we could use multiple producers - single consumer queue or lock-free queue. +// The current implementation can lead a significant amount of overhead if the +// mutex is highly disputed. Unless this is proven to be indeed a bottleneck, we +// embrace KISS principle. However, in a future iteration we could use multiple +// producers - single consumer queue or lock-free queue. class Telemetry final : public std::enable_shared_from_this { // Configuration object containing the validated settings for telemetry FinalizedConfiguration config_; From 9833ef36af55e045e13e41fa86108fbaced9e40c Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Tue, 1 Sep 2026 17:55:09 +0200 Subject: [PATCH 7/9] Add Clock in the list of suppliable class that is thread sensitive --- docs/thread-safety.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/thread-safety.md b/docs/thread-safety.md index 099ec99f8..0cb32f87f 100644 --- a/docs/thread-safety.md +++ b/docs/thread-safety.md @@ -64,9 +64,9 @@ set once and never mutated afterward. - Multiple `Tracer`s in one process publish one shared OpenTelemetry process context. The first `Tracer`'s fields win. The shared `runtime_id` is published only while every `Tracer` agrees on it. -- If you supply your own `Collector`/`HTTPClient`/`EventScheduler`/`IDGenerator`/`Logger`, you must - ensure it is safe to be called from multiple threads. The library will call it from whatever - threads its other pluggable pieces run on. +- If you supply your own `Clock`/`Collector`/`EventScheduler`/`HTTPClient`/`IDGenerator`/`Logger`, + you must ensure it is safe to be called from multiple threads because the library will call it + from whatever threads its other pluggable pieces run on. ## Web Reverse Proxies Integrations From d60176ae062a262c8defb075537bbbd15aa88924 Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Tue, 1 Sep 2026 18:06:20 +0200 Subject: [PATCH 8/9] tiny: fix markdown file formatting --- docs/thread-safety.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/thread-safety.md b/docs/thread-safety.md index 0cb32f87f..5d7c901c1 100644 --- a/docs/thread-safety.md +++ b/docs/thread-safety.md @@ -19,10 +19,13 @@ set once and never mutated afterward. contract is: **one `Span`, one owner thread at a time**. Concurrency is meant to happen between sibling `Span`s of the same `TraceSegment`. - **Use `std::mutex`, not `std::atomic`**: - - Every synchronized class uses plain `std::mutex` + `lock_guard`/`unique_lock` (no usage of `std::atomic`), for simplicity. + - Every synchronized class uses plain `std::mutex` + `lock_guard`/`unique_lock` (no usage of + `std::atomic`), for simplicity. - No logging while holding a process-wide singleton's lock, because of potentially slow custom `Logger`. -- **The default pluggable interfaces add extra threads.** The default `HTTPClient`, which is `Curl`, and the default `EventScheduler`, which is `ThreadedEventScheduler`, add each one a dedicated thread. +- **The default pluggable interfaces add extra threads.** The default `HTTPClient`, which is `Curl`, + and the default `EventScheduler`, which is `ThreadedEventScheduler`, add each one a dedicated + thread. - `Fork` (such as Nginx/Apache pre-fork workers). Because background threads are not automatically fork-safe, the embedder must construct the `Tracer` (and therefore any default `Curl`/`ThreadedEventScheduler`) strictly after `fork()`. @@ -110,7 +113,8 @@ The `root_session_id` and `runtime_id` are set explicitly pre-fork. It has a custom logger locking. -The `Span`s are allocated on the heap and tied to the request's Apache Pre-Request (APR) memory pool (and so automatically deleted when the request finishes). +The `Span`s are allocated on the heap and tied to the request's Apache Pre-Request (APR) memory pool +(and so automatically deleted when the request finishes). ### Datadog Envoy Extension @@ -122,8 +126,10 @@ Process / Thread Model: - Envoy has a **single process**, with several **worker threads**. - Each worker thread creates its `Tracer`. -It uses a custom `AgentHTTPClient` and a custom `EventScheduler`. They are bound to the owning `Dispatcher`, with no extra thread. +It uses a custom `AgentHTTPClient` and a custom `EventScheduler`. They are bound to the owning +`Dispatcher`, with no extra thread. It uses Envoy’s own logging. -Envoy is the only integration where `OtelCtxRegistration`'s multi-Tracer bookkeeping is actually exercised. +Envoy is the only integration where `OtelCtxRegistration`'s multi-Tracer bookkeeping is actually +exercised. From 26fc882f04e8c64a606090a5e121b931b2191bfe Mon Sep 17 00:00:00 2001 From: Xavier Lamorlette Date: Tue, 1 Sep 2026 18:10:26 +0200 Subject: [PATCH 9/9] Clarify the handling of root_session_id --- docs/thread-safety.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/thread-safety.md b/docs/thread-safety.md index 5d7c901c1..9a3885c70 100644 --- a/docs/thread-safety.md +++ b/docs/thread-safety.md @@ -60,10 +60,11 @@ set once and never mutated afterward. request's event handlers so the main thread can't touch it mid-flight). - Never construct a `Tracer` before your process forks. Construct it after, in each child. - Multiple `Tracer`s in one process share one telemetry pipeline. -- Multiple `Tracer`s in one process will end up with a `root_session_id` decided by whichever - `Tracer` happened to construct first. To avoid this, you can you explicitly set - `TracerConfig::root_session_id` (for example, Nginx and Apache both compute it once, pre-fork, - then pass it explicitly). +- Multiple `Tracer`s in one process end up with a `root_session_id` decided by the first `Tracer` + constructed (later values are silently ignored). Pass the same explicit + `TracerConfig::root_session_id` to every `Tracer` (for example, Nginx and Apache both compute it + once pre-fork, then pass it to each worker's `Tracer`). + - Multiple `Tracer`s in one process publish one shared OpenTelemetry process context. The first `Tracer`'s fields win. The shared `runtime_id` is published only while every `Tracer` agrees on it.