From 543c9b182fab81c82c4229bfad6809714235e9c0 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 14:08:55 +0300 Subject: [PATCH 01/62] feat(feature-flags): add agentless configuration keys to supported-configurations --- .../supported-configurations.yaml | 72 +++++++++++++++++++ .../ConfigurationKeys.FeatureFlags.g.cs | 48 +++++++++++++ .../ConfigurationKeys.FeatureFlags.g.cs | 48 +++++++++++++ .../ConfigurationKeys.FeatureFlags.g.cs | 48 +++++++++++++ .../ConfigurationKeys.FeatureFlags.g.cs | 48 +++++++++++++ 5 files changed, 264 insertions(+) diff --git a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml index cba391e5643a..eef60327d1c8 100644 --- a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml +++ b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml @@ -3157,6 +3157,78 @@ supportedConfigurations: documentation: |- Enables Feature Flags Provider (Experimental). Default value is false (disabled). + DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS: + - implementation: B + scope: managed + type: int + default: '10000' + product: FeatureFlags + const_name: FlaggingProviderInitializationTimeoutMs + documentation: |- + Configuration key for how long, in milliseconds, provider initialization waits for the first + flag configuration to arrive before returning. + Default value is 10000 (10 seconds). + Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations + return the caller's default value, and the provider becomes ready when configuration arrives. + DD_FEATURE_FLAGS_ENABLED: + - implementation: A + scope: managed + type: boolean + default: 'true' + product: FeatureFlags + const_name: FeatureFlagsEnabled + documentation: |- + Configuration key to enable or disable Feature Flags. + Default value is true (enabled). + Feature Flags only contact Datadog once application code initializes the provider, so enabling + this alone does not start requesting flag configuration. + This supersedes . + DD_FEATURE_FLAGS_CONFIGURATION_SOURCE: + - implementation: A + scope: managed + type: string + default: agentless + product: FeatureFlags + const_name: FeatureFlagsConfigurationSource + documentation: |- + Configuration key for selecting where flag configuration is loaded from. + Supported values are agentless (direct HTTP delivery, the default) and + remote_config (delivery through the Datadog Agent's Remote Configuration). + Any other value disables Feature Flags, which fails closed and contacts nothing. + DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL: + - implementation: A + scope: managed + type: string + default: null + product: FeatureFlags + const_name: FeatureFlagsConfigurationSourceAgentlessBaseUrl + documentation: |- + Configuration key for overriding the endpoint used by the agentless configuration source. + When the URL has no path, or a path of /, the standard rules-based server path is appended; + any other path is used verbatim as the exact endpoint. + If unset, the endpoint is derived from . + DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS: + - implementation: A + scope: managed + type: int + default: '30' + product: FeatureFlags + const_name: FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds + documentation: |- + Configuration key for how often, in seconds, the agentless configuration source polls for + flag configuration. + Default value is 30. Values outside (0, 3600] are rejected and the default is used. + DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS: + - implementation: A + scope: managed + type: int + default: '5' + product: FeatureFlags + const_name: FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds + documentation: |- + Configuration key for the request timeout, in seconds, used by the agentless configuration + source. + Default value is 5. Non-positive values are rejected and the default is used. DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED: - implementation: A scope: managed diff --git a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index e3c49c08f659..c0bc6ef9a199 100644 --- a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -23,12 +23,60 @@ internal static class FeatureFlags /// public const string FlaggingProviderEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"; + /// + /// Configuration key for how long, in milliseconds, provider initialization waits for the first + /// flag configuration to arrive before returning. + /// Default value is 10000 (10 seconds). + /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations + /// return the caller's default value, and the provider becomes ready when configuration arrives. + /// + public const string FlaggingProviderInitializationTimeoutMs = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS"; + /// /// Enables APM span enrichment with feature-flag evaluation metadata (Experimental). /// Default value is false (disabled). /// public const string SpanEnrichmentEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED"; + /// + /// Configuration key for selecting where flag configuration is loaded from. + /// Supported values are agentless (direct HTTP delivery, the default) and + /// remote_config (delivery through the Datadog Agent's Remote Configuration). + /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// + public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; + + /// + /// Configuration key for overriding the endpoint used by the agentless configuration source. + /// When the URL has no path, or a path of /, the standard rules-based server path is appended; + /// any other path is used verbatim as the exact endpoint. + /// If unset, the endpoint is derived from . + /// + public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; + + /// + /// Configuration key for how often, in seconds, the agentless configuration source polls for + /// flag configuration. + /// Default value is 30. Values outside (0, 3600] are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS"; + + /// + /// Configuration key for the request timeout, in seconds, used by the agentless configuration + /// source. + /// Default value is 5. Non-positive values are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS"; + + /// + /// Configuration key to enable or disable Feature Flags. + /// Default value is true (enabled). + /// Feature Flags only contact Datadog once application code initializes the provider, so enabling + /// this alone does not start requesting flag configuration. + /// This supersedes . + /// + public const string FeatureFlagsEnabled = "DD_FEATURE_FLAGS_ENABLED"; + /// /// Enables support for collecting and exporting logs generated by the the OpenTelemetry Logs API. /// This feature is available starting with .NET 3.1 when using Microsoft.Extensions.Logging diff --git a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index e3c49c08f659..c0bc6ef9a199 100644 --- a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -23,12 +23,60 @@ internal static class FeatureFlags /// public const string FlaggingProviderEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"; + /// + /// Configuration key for how long, in milliseconds, provider initialization waits for the first + /// flag configuration to arrive before returning. + /// Default value is 10000 (10 seconds). + /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations + /// return the caller's default value, and the provider becomes ready when configuration arrives. + /// + public const string FlaggingProviderInitializationTimeoutMs = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS"; + /// /// Enables APM span enrichment with feature-flag evaluation metadata (Experimental). /// Default value is false (disabled). /// public const string SpanEnrichmentEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED"; + /// + /// Configuration key for selecting where flag configuration is loaded from. + /// Supported values are agentless (direct HTTP delivery, the default) and + /// remote_config (delivery through the Datadog Agent's Remote Configuration). + /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// + public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; + + /// + /// Configuration key for overriding the endpoint used by the agentless configuration source. + /// When the URL has no path, or a path of /, the standard rules-based server path is appended; + /// any other path is used verbatim as the exact endpoint. + /// If unset, the endpoint is derived from . + /// + public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; + + /// + /// Configuration key for how often, in seconds, the agentless configuration source polls for + /// flag configuration. + /// Default value is 30. Values outside (0, 3600] are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS"; + + /// + /// Configuration key for the request timeout, in seconds, used by the agentless configuration + /// source. + /// Default value is 5. Non-positive values are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS"; + + /// + /// Configuration key to enable or disable Feature Flags. + /// Default value is true (enabled). + /// Feature Flags only contact Datadog once application code initializes the provider, so enabling + /// this alone does not start requesting flag configuration. + /// This supersedes . + /// + public const string FeatureFlagsEnabled = "DD_FEATURE_FLAGS_ENABLED"; + /// /// Enables support for collecting and exporting logs generated by the the OpenTelemetry Logs API. /// This feature is available starting with .NET 3.1 when using Microsoft.Extensions.Logging diff --git a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index e3c49c08f659..c0bc6ef9a199 100644 --- a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -23,12 +23,60 @@ internal static class FeatureFlags /// public const string FlaggingProviderEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"; + /// + /// Configuration key for how long, in milliseconds, provider initialization waits for the first + /// flag configuration to arrive before returning. + /// Default value is 10000 (10 seconds). + /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations + /// return the caller's default value, and the provider becomes ready when configuration arrives. + /// + public const string FlaggingProviderInitializationTimeoutMs = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS"; + /// /// Enables APM span enrichment with feature-flag evaluation metadata (Experimental). /// Default value is false (disabled). /// public const string SpanEnrichmentEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED"; + /// + /// Configuration key for selecting where flag configuration is loaded from. + /// Supported values are agentless (direct HTTP delivery, the default) and + /// remote_config (delivery through the Datadog Agent's Remote Configuration). + /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// + public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; + + /// + /// Configuration key for overriding the endpoint used by the agentless configuration source. + /// When the URL has no path, or a path of /, the standard rules-based server path is appended; + /// any other path is used verbatim as the exact endpoint. + /// If unset, the endpoint is derived from . + /// + public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; + + /// + /// Configuration key for how often, in seconds, the agentless configuration source polls for + /// flag configuration. + /// Default value is 30. Values outside (0, 3600] are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS"; + + /// + /// Configuration key for the request timeout, in seconds, used by the agentless configuration + /// source. + /// Default value is 5. Non-positive values are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS"; + + /// + /// Configuration key to enable or disable Feature Flags. + /// Default value is true (enabled). + /// Feature Flags only contact Datadog once application code initializes the provider, so enabling + /// this alone does not start requesting flag configuration. + /// This supersedes . + /// + public const string FeatureFlagsEnabled = "DD_FEATURE_FLAGS_ENABLED"; + /// /// Enables support for collecting and exporting logs generated by the the OpenTelemetry Logs API. /// This feature is available starting with .NET 3.1 when using Microsoft.Extensions.Logging diff --git a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index e3c49c08f659..c0bc6ef9a199 100644 --- a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -23,12 +23,60 @@ internal static class FeatureFlags /// public const string FlaggingProviderEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED"; + /// + /// Configuration key for how long, in milliseconds, provider initialization waits for the first + /// flag configuration to arrive before returning. + /// Default value is 10000 (10 seconds). + /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations + /// return the caller's default value, and the provider becomes ready when configuration arrives. + /// + public const string FlaggingProviderInitializationTimeoutMs = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS"; + /// /// Enables APM span enrichment with feature-flag evaluation metadata (Experimental). /// Default value is false (disabled). /// public const string SpanEnrichmentEnabled = "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED"; + /// + /// Configuration key for selecting where flag configuration is loaded from. + /// Supported values are agentless (direct HTTP delivery, the default) and + /// remote_config (delivery through the Datadog Agent's Remote Configuration). + /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// + public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; + + /// + /// Configuration key for overriding the endpoint used by the agentless configuration source. + /// When the URL has no path, or a path of /, the standard rules-based server path is appended; + /// any other path is used verbatim as the exact endpoint. + /// If unset, the endpoint is derived from . + /// + public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; + + /// + /// Configuration key for how often, in seconds, the agentless configuration source polls for + /// flag configuration. + /// Default value is 30. Values outside (0, 3600] are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS"; + + /// + /// Configuration key for the request timeout, in seconds, used by the agentless configuration + /// source. + /// Default value is 5. Non-positive values are rejected and the default is used. + /// + public const string FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS"; + + /// + /// Configuration key to enable or disable Feature Flags. + /// Default value is true (enabled). + /// Feature Flags only contact Datadog once application code initializes the provider, so enabling + /// this alone does not start requesting flag configuration. + /// This supersedes . + /// + public const string FeatureFlagsEnabled = "DD_FEATURE_FLAGS_ENABLED"; + /// /// Enables support for collecting and exporting logs generated by the the OpenTelemetry Logs API. /// This feature is available starting with .NET 3.1 when using Microsoft.Extensions.Logging From 47530796486e0d93ecf6d2a74a82a5b422bc17e5 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 14:13:45 +0300 Subject: [PATCH 02/62] feat(feature-flags): add FeatureFlagsSettings with source resolution and validation --- .../FeatureFlags/FeatureFlagsSettings.cs | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs new file mode 100644 index 000000000000..71677f39b508 --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -0,0 +1,228 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.Logging; +using Datadog.Trace.Telemetry; +using Datadog.Trace.Util; + +namespace Datadog.Trace.FeatureFlags; + +/// +/// Feature Flags configuration: which delivery source is selected, and how the agentless +/// source is operated. +/// +internal sealed class FeatureFlagsSettings +{ + internal const string AgentlessSourceName = "agentless"; + internal const string RemoteConfigSourceName = "remote_config"; + internal const string OfflineSourceName = "offline"; + + internal const string DefaultSite = "datadoghq.com"; + + internal const int DefaultPollIntervalSeconds = 30; + internal const int DefaultRequestTimeoutSeconds = 5; + internal const int DefaultInitializationTimeoutMs = 10_000; + + // An interval above this is indistinguishable from "never poll" and is more likely a + // misconfiguration (for example milliseconds passed as seconds) than an intent. + private const int MaxPollIntervalSeconds = 3600; + + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(FeatureFlagsSettings)); + + public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry) + { + source ??= NullConfigurationSource.Instance; + var config = new ConfigurationBuilder(source, telemetry); + + // Read as nullable: the precedence rules distinguish "explicitly provided" from "left unset", + // so a default value here would erase the difference the legacy key depends on. + var enabled = config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsEnabled).AsBool(); +#pragma warning disable 618 // superseded, but still honoured so existing adopters keep their source + var legacyEnabled = config.WithKeys(ConfigurationKeys.FeatureFlags.FlaggingProviderEnabled).AsBool(); +#pragma warning restore 618 + var configuredSource = config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource).AsString(); + + if (legacyEnabled is not null) + { +#pragma warning disable 618 + Log.Warning( + "{LegacyKey} is deprecated. Use {EnabledKey} and {SourceKey} instead.", + ConfigurationKeys.FeatureFlags.FlaggingProviderEnabled, + ConfigurationKeys.FeatureFlags.FeatureFlagsEnabled, + ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource); +#pragma warning restore 618 + } + + Source = ResolveSource(enabled, configuredSource, legacyEnabled); + + AgentlessBaseUrl = config + .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessBaseUrl) + .AsString(url => !StringUtil.IsNullOrEmpty(url?.Trim())); + + PollInterval = TimeSpan.FromSeconds( + InRangeOrDefault( + config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds).AsInt32(), + ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds, + DefaultPollIntervalSeconds, + MaxPollIntervalSeconds)); + + RequestTimeout = TimeSpan.FromSeconds( + InRangeOrDefault( + config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds).AsInt32(), + ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds, + DefaultRequestTimeoutSeconds, + maximumSeconds: null)); + + Site = config + .WithKeys(ConfigurationKeys.Site) + .AsString(DefaultSite, site => !StringUtil.IsNullOrEmpty(site?.Trim())); + + Env = config.WithKeys(ConfigurationKeys.Environment).AsString(); + + ApiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString(); + + var initializationTimeoutMs = config + .WithKeys(ConfigurationKeys.FeatureFlags.FlaggingProviderInitializationTimeoutMs) + .AsInt32(DefaultInitializationTimeoutMs, timeout => timeout > 0) + .Value; + InitializationTimeout = TimeSpan.FromMilliseconds(initializationTimeoutMs); + } + + /// + /// Gets the resolved delivery source. means nothing is contacted. + /// + public FeatureFlagsSource Source { get; } + + /// + /// Gets a value indicating whether Feature Flags are enabled at all. + /// + public bool Enabled => Source != FeatureFlagsSource.Disabled; + + /// + /// Gets the configured override for the agentless endpoint, or null to derive it from the site. + /// + public string? AgentlessBaseUrl { get; } + + /// + /// Gets the Datadog site the managed agentless endpoint is derived from. + /// + public string Site { get; } + + /// + /// Gets the configured environment, sent to the agentless endpoint as dd_env. + /// + public string? Env { get; } + + /// + /// Gets the API key, required by the managed agentless endpoint. + /// + public string? ApiKey { get; } + + /// + /// Gets how often the agentless source polls for configuration. + /// + public TimeSpan PollInterval { get; } + + /// + /// Gets the per-request timeout used by the agentless source. + /// + public TimeSpan RequestTimeout { get; } + + /// + /// Gets how long provider initialization waits for the first configuration. + /// + public TimeSpan InitializationTimeout { get; } + + public static FeatureFlagsSettings FromDefaultSource() + => new(GlobalConfigurationSource.Instance, TelemetryFactory.Config); + + /// + /// Resolves the delivery source. Shared across tracers, so the ordering is deliberate: + /// the stable kill switch wins over everything, an explicit source wins over the legacy key + /// (and fails closed when unrecognised), the legacy key grandfathers existing adopters onto + /// Remote Configuration, and everything else defaults to agentless. + /// + internal static FeatureFlagsSource ResolveSource(bool? enabled, string? configuredSource, bool? legacyEnabled) + { + var normalizedSource = NormalizeSource(configuredSource); + + if (enabled == false) + { + return FeatureFlagsSource.Disabled; + } + + if (normalizedSource is not null) + { + switch (normalizedSource) + { + case AgentlessSourceName: + return FeatureFlagsSource.Agentless; + case RemoteConfigSourceName: + return FeatureFlagsSource.RemoteConfig; + case OfflineSourceName: + // Reserved fail-closed sentinel: the provider is intentionally off, so no warning. + return FeatureFlagsSource.Disabled; + default: + Log.Warning( + "Unsupported {SourceKey} value '{Source}'. Feature Flags are disabled.", + ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource, + normalizedSource); + return FeatureFlagsSource.Disabled; + } + } + + // The legacy key only grandfathers adopters who have not migrated: an explicit new-key + // value (true or false) takes precedence, so the legacy key is consulted only when the + // new key was left unset. + if (enabled is null && legacyEnabled is not null) + { + return legacyEnabled.Value ? FeatureFlagsSource.RemoteConfig : FeatureFlagsSource.Disabled; + } + + return FeatureFlagsSource.Agentless; + } + + /// + /// An empty or whitespace-only source is semantically unset, not an unrecognised value. + /// + private static string? NormalizeSource(string? configuredSource) + { + if (configuredSource is null) + { + return null; + } + + var normalized = configuredSource.Trim().ToLowerInvariant(); + return normalized.Length == 0 ? null : normalized; + } + + internal static int InRangeOrDefault(int? configured, string key, int defaultSeconds, int? maximumSeconds) + { + if (configured is null) + { + return defaultSeconds; + } + + var value = configured.Value; + if (value <= 0 || (maximumSeconds is { } maximum && value > maximum)) + { + // A non-positive interval would turn polling into a tight loop against the endpoint, + // so an out-of-range value is rejected rather than honoured. + Log.Warning( + "Invalid value {Key}={Value}. Using {Default} seconds instead.", + key, + value, + defaultSeconds); + return defaultSeconds; + } + + return value; + } +} From 25e168f7fac589e3cff2de605165565579e4dca4 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 14:34:49 +0300 Subject: [PATCH 03/62] feat(feature-flags): add AgentlessEndpoint for CDN URL derivation --- .../Agentless/AgentlessEndpoint.cs | 116 ++++++++++++++++++ .../FeatureFlags/FeatureFlagsSource.cs | 29 +++++ 2 files changed, 145 insertions(+) create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs new file mode 100644 index 000000000000..757691bfa4cf --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -0,0 +1,116 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using Datadog.Trace.Util; + +namespace Datadog.Trace.FeatureFlags.Agentless; + +/// +/// The agentless endpoint, derived from the Datadog site or a custom base URL. +/// +internal readonly struct AgentlessEndpoint +{ + /// + /// Canonical rules-based server path, appended to the managed CDN host and to custom base + /// URLs that only supply an origin. + /// + internal const string DefaultPath = "/api/v2/feature-flagging/config/rules-based/server"; + + /// + /// The prefix prepended to the site to form the managed CDN host. + /// + internal const string ManagedHostPrefix = "ufc-server.ff-cdn."; + + private AgentlessEndpoint(Uri uri, bool isManaged) + { + Uri = uri; + IsManaged = isManaged; + } + + /// + /// Gets the endpoint URI. + /// + public Uri Uri { get; } + + /// + /// Gets a value indicating whether this is the endpoint derived from the site. The API key is + /// only sent there: a custom endpoint reports its own authentication failure rather than + /// having the credential guessed onto it. + /// + public bool IsManaged { get; } + + /// + /// Builds the endpoint. Without a custom the managed Datadog CDN + /// endpoint is derived from the (lowercased) site, so staging and government sites resolve + /// with no allowlist, and dd_env is added only when an environment is configured. + /// A custom base URL that is an origin receives the canonical path; one that carries a path + /// is used verbatim. + /// + /// The Datadog site, for example datadoghq.com. + /// The configured environment, or null. + /// The configured endpoint override, or null. + /// The resulting endpoint. + /// Why the configured base URL was rejected. Never contains the URL, which may carry credentials. + /// true when an endpoint could be built. + public static bool TryCreate(string? site, string? env, string? baseUrl, out AgentlessEndpoint endpoint, out string? error) + { + endpoint = default; + error = null; + + var configured = baseUrl?.Trim(); + if (StringUtil.IsNullOrEmpty(configured)) + { + var trimmedSite = site?.Trim(); + if (StringUtil.IsNullOrEmpty(trimmedSite)) + { + error = "No Datadog site is configured"; + return false; + } + + var managed = new UriBuilder("https", ManagedHostPrefix + trimmedSite!.ToLowerInvariant()) { Path = DefaultPath }; + if (!StringUtil.IsNullOrEmpty(env)) + { + managed.Query = "dd_env=" + Uri.EscapeDataString(env!); + } + + endpoint = new AgentlessEndpoint(managed.Uri, isManaged: true); + return true; + } + + // A URL with internal whitespace is malformed, and Uri parsing is lenient enough to accept it. + foreach (var character in configured!) + { + if (char.IsWhiteSpace(character)) + { + error = "The configured Feature Flags agentless URL is not a valid URL"; + return false; + } + } + + if (!Uri.TryCreate(configured, UriKind.Absolute, out var custom) || StringUtil.IsNullOrEmpty(custom.Host)) + { + error = "The configured Feature Flags agentless URL is not a valid absolute URL"; + return false; + } + + // http is accepted for a custom endpoint only: pointing at one is an operator decision. + if (custom.Scheme != Uri.UriSchemeHttps && custom.Scheme != Uri.UriSchemeHttp) + { + error = "The configured Feature Flags agentless URL must use HTTP or HTTPS"; + return false; + } + + if (custom.AbsolutePath is "" or "/") + { + custom = new UriBuilder(custom) { Path = DefaultPath }.Uri; + } + + endpoint = new AgentlessEndpoint(custom, isManaged: false); + return true; + } +} diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs new file mode 100644 index 000000000000..dd50047ede42 --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs @@ -0,0 +1,29 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.FeatureFlags; + +/// +/// Where flag configuration is loaded from. +/// +internal enum FeatureFlagsSource +{ + /// + /// Feature Flags are disabled: no configuration is loaded, and neither delivery path is contacted. + /// + Disabled, + + /// + /// Configuration is fetched over HTTP, without the Datadog Agent. + /// + Agentless, + + /// + /// Configuration is delivered through the Datadog Agent's Remote Configuration. + /// + RemoteConfig, +} From 9c3a236bfb63e211793ab97eefaa170a26a3c270 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 14:37:12 +0300 Subject: [PATCH 04/62] test(feature-flags): add unit tests for FeatureFlagsSettings and AgentlessEndpoint --- .../FeatureFlags/AgentlessEndpointTests.cs | 116 +++++++++++ .../FeatureFlags/FeatureFlagsSettingsTests.cs | 184 ++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs create mode 100644 tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs new file mode 100644 index 000000000000..1ab87b4c3dec --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -0,0 +1,116 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using Datadog.Trace.FeatureFlags.Agentless; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.FeatureFlags; + +public class AgentlessEndpointTests +{ + private const string DefaultPath = "/api/v2/feature-flagging/config/rules-based/server"; + + [Theory] + [InlineData("datadoghq.com", "https://ufc-server.ff-cdn.datadoghq.com" + DefaultPath)] + [InlineData("DATADOGHQ.COM", "https://ufc-server.ff-cdn.datadoghq.com" + DefaultPath)] // site is lowercased + [InlineData("datad0g.com", "https://ufc-server.ff-cdn.datad0g.com" + DefaultPath)] // staging + [InlineData("ddog-gov.com", "https://ufc-server.ff-cdn.ddog-gov.com" + DefaultPath)] // govcloud + public void DerivesManagedEndpointFromSite(string site, string expected) + { + AgentlessEndpoint.TryCreate(site, env: null, baseUrl: null, out var endpoint, out var error) + .Should().BeTrue(); + error.Should().BeNull(); + endpoint.IsManaged.Should().BeTrue(); + endpoint.Uri.ToString().Should().Be(expected); + } + + [Fact] + public void AddsDdEnvWhenEnvIsConfigured() + { + AgentlessEndpoint.TryCreate("datadoghq.com", env: "production", baseUrl: null, out var endpoint, out _) + .Should().BeTrue(); + endpoint.Uri.Query.Should().Be("?dd_env=production"); + } + + [Fact] + public void DoesNotAddDdEnvWhenEnvIsNull() + { + AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: null, out var endpoint, out _) + .Should().BeTrue(); + endpoint.Uri.Query.Should().BeEmpty(); + } + + [Fact] + public void EscapesDdEnvValue() + { + AgentlessEndpoint.TryCreate("datadoghq.com", env: "my env&test", baseUrl: null, out var endpoint, out _) + .Should().BeTrue(); + endpoint.Uri.Query.Should().Be("?dd_env=my%20env%26test"); + } + + [Theory] + [InlineData("https://flags.example.com", "https://flags.example.com" + DefaultPath)] + [InlineData("https://flags.example.com/", "https://flags.example.com" + DefaultPath)] + [InlineData("https://flags.example.com/ufc", "https://flags.example.com/ufc")] + [InlineData("https://flags.example.com/ufc?custom=query", "https://flags.example.com/ufc?custom=query")] + public void CustomEndpointReceivesCanonicalPathForOriginOnly(string baseUrl, string expected) + { + AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) + .Should().BeTrue(); + error.Should().BeNull(); + endpoint.IsManaged.Should().BeFalse(); + endpoint.Uri.ToString().Should().Be(expected); + } + + [Theory] + [InlineData("http://localhost:8080/ufc")] // http accepted for custom endpoints + public void CustomEndpointAcceptsHttp(string baseUrl) + { + AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) + .Should().BeTrue(); + endpoint.IsManaged.Should().BeFalse(); + } + + [Theory] + [InlineData("ftp://flags.example.com", "The configured Feature Flags agentless URL must use HTTP or HTTPS")] + [InlineData("not a url", "The configured Feature Flags agentless URL is not a valid absolute URL")] + [InlineData(" https://flags.example.com ", "The configured Feature Flags agentless URL is not a valid URL")] // internal whitespace + public void RejectsInvalidBaseUrl(string baseUrl, string expectedError) + { + AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) + .Should().BeFalse(); + error.Should().Be(expectedError); + } + + [Fact] + public void RejectsEmptySiteWithoutBaseUrl() + { + AgentlessEndpoint.TryCreate(site: null, env: null, baseUrl: null, out var endpoint, out var error) + .Should().BeFalse(); + error.Should().Be("No Datadog site is configured"); + } + + [Fact] + public void RejectsWhitespaceOnlySiteWithoutBaseUrl() + { + AgentlessEndpoint.TryCreate(" ", env: null, baseUrl: null, out var endpoint, out var error) + .Should().BeFalse(); + error.Should().Be("No Datadog site is configured"); + } + + [Fact] + public void ErrorNeverContainsUrl() + { + // A URL may carry credentials, so the error must never echo it. + AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: "https://user:pass@flags.example.com bad", out _, out var error) + .Should().BeFalse(); + error.Should().NotContain("user"); + error.Should().NotContain("pass"); + } +} diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs new file mode 100644 index 000000000000..ec8be148a134 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -0,0 +1,184 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Specialized; +using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.FeatureFlags; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.FeatureFlags; + +public class FeatureFlagsSettingsTests +{ + // The source-selection contract is shared across tracers, so these cases mirror the + // system-tests parametric suite (tests/parametric/test_ffe/test_configuration_sources.py). + [Theory] + // Nothing configured: agentless is the default. + [InlineData(null, null, null, FeatureFlagsSource.Agentless)] + // The stable kill switch wins over everything, including a legacy opt-in and an explicit source. + [InlineData("false", null, null, FeatureFlagsSource.Disabled)] + [InlineData("false", null, "true", FeatureFlagsSource.Disabled)] + [InlineData("false", "agentless", null, FeatureFlagsSource.Disabled)] + [InlineData("false", "remote_config", null, FeatureFlagsSource.Disabled)] + // Enabling explicitly does not imply the historical Remote Configuration source. + [InlineData("true", null, null, FeatureFlagsSource.Agentless)] + // An explicit source wins over the legacy key, in both directions. + [InlineData(null, "agentless", "true", FeatureFlagsSource.Agentless)] + [InlineData(null, "remote_config", "false", FeatureFlagsSource.RemoteConfig)] + // The legacy key grandfathers existing adopters, who opted in when RC was the only source. + [InlineData(null, null, "true", FeatureFlagsSource.RemoteConfig)] + [InlineData(null, null, "false", FeatureFlagsSource.Disabled)] + // An explicit new-key value takes precedence over the legacy key, so a stale legacy disable + // does not silently keep Feature Flags off during migration. + [InlineData("true", null, "false", FeatureFlagsSource.Agentless)] + [InlineData("true", null, "true", FeatureFlagsSource.Agentless)] + // An unrecognised source fails closed rather than guessing a billed delivery path. + [InlineData(null, "invalid", null, FeatureFlagsSource.Disabled)] + [InlineData(null, "invalid", "true", FeatureFlagsSource.Disabled)] + // "offline" is a reserved, recognised fail-closed sentinel (not an unrecognised value). + [InlineData(null, "offline", null, FeatureFlagsSource.Disabled)] + [InlineData(null, "offline", "true", FeatureFlagsSource.Disabled)] + public void ResolvesSource(string? enabled, string? source, string? legacyEnabled, object expected) + { + var expectedSource = (FeatureFlagsSource)expected; + var settings = CreateSettings(enabled, source, legacyEnabled); + + settings.Source.Should().Be(expectedSource); + settings.Enabled.Should().Be(expectedSource != FeatureFlagsSource.Disabled); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void TreatsBlankSourceAsUnset(string source) + { + CreateSettings(enabled: null, source: source, legacyEnabled: null) + .Source.Should().Be(FeatureFlagsSource.Agentless); + + // Being semantically unset, a blank source still lets the legacy key grandfather RC. + CreateSettings(enabled: null, source: source, legacyEnabled: "true") + .Source.Should().Be(FeatureFlagsSource.RemoteConfig); + } + + [Theory] + [InlineData("AGENTLESS", FeatureFlagsSource.Agentless)] + [InlineData(" Remote_Config ", FeatureFlagsSource.RemoteConfig)] + public void NormalizesSourceCasingAndWhitespace(string source, object expected) + => CreateSettings(enabled: null, source: source, legacyEnabled: null).Source.Should().Be((FeatureFlagsSource)expected); + + [Fact] + public void UsesDocumentedDefaults() + { + var settings = CreateSettings(null, null, null); + + settings.PollInterval.Should().Be(TimeSpan.FromSeconds(30)); + settings.RequestTimeout.Should().Be(TimeSpan.FromSeconds(5)); + settings.InitializationTimeout.Should().Be(TimeSpan.FromMilliseconds(10_000)); + settings.AgentlessBaseUrl.Should().BeNull(); + } + + [Theory] + [InlineData("60", 60)] + [InlineData("3600", 3600)] + // Out of range values are rejected in favour of the default: a non-positive interval would + // turn polling into a tight loop, and an implausibly large one is a misconfiguration. + [InlineData("0", 30)] + [InlineData("-1", 30)] + [InlineData("3601", 30)] + [InlineData("not-a-number", 30)] + public void ReadsPollInterval(string configured, int expectedSeconds) + { + var settings = CreateSettings( + null, + null, + null, + (ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds, configured)); + + settings.PollInterval.Should().Be(TimeSpan.FromSeconds(expectedSeconds)); + } + + [Theory] + [InlineData("1", 1)] + [InlineData("0", 5)] + [InlineData("-2", 5)] + public void ReadsRequestTimeout(string configured, int expectedSeconds) + { + var settings = CreateSettings( + null, + null, + null, + (ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds, configured)); + + settings.RequestTimeout.Should().Be(TimeSpan.FromSeconds(expectedSeconds)); + } + + [Theory] + [InlineData("1000", 1000)] + [InlineData("0", 10_000)] + [InlineData("-1", 10_000)] + public void ReadsInitializationTimeout(string configured, int expectedMs) + { + var settings = CreateSettings( + null, + null, + null, + (ConfigurationKeys.FeatureFlags.FlaggingProviderInitializationTimeoutMs, configured)); + + settings.InitializationTimeout.Should().Be(TimeSpan.FromMilliseconds(expectedMs)); + } + + [Theory] + [InlineData("https://flags.example.com/ufc", "https://flags.example.com/ufc")] + [InlineData("", null)] + [InlineData(" ", null)] + public void ReadsAgentlessBaseUrl(string configured, string? expected) + { + var settings = CreateSettings( + null, + null, + null, + (ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessBaseUrl, configured)); + + settings.AgentlessBaseUrl.Should().Be(expected); + } + + private static FeatureFlagsSettings CreateSettings( + string? enabled, + string? source, + string? legacyEnabled, + params (string Key, string Value)[] extra) + { + var collection = new NameValueCollection(); + + if (enabled is not null) + { + collection[ConfigurationKeys.FeatureFlags.FeatureFlagsEnabled] = enabled; + } + + if (source is not null) + { + collection[ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource] = source; + } + + if (legacyEnabled is not null) + { +#pragma warning disable 618 // superseded, but still honoured for existing adopters + collection[ConfigurationKeys.FeatureFlags.FlaggingProviderEnabled] = legacyEnabled; +#pragma warning restore 618 + } + + foreach (var (key, value) in extra) + { + collection[key] = value; + } + + return new FeatureFlagsSettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance); + } +} From 95bf50bd8a1db8c4f465cfb4c259d57903bcd2e7 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 15:52:35 +0300 Subject: [PATCH 05/62] fix(tests): correct AgentlessEndpointTests invalid URL expectations --- .../FeatureFlags/AgentlessEndpointTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs index 1ab87b4c3dec..049a6254025d 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -79,8 +79,8 @@ public void CustomEndpointAcceptsHttp(string baseUrl) [Theory] [InlineData("ftp://flags.example.com", "The configured Feature Flags agentless URL must use HTTP or HTTPS")] - [InlineData("not a url", "The configured Feature Flags agentless URL is not a valid absolute URL")] - [InlineData(" https://flags.example.com ", "The configured Feature Flags agentless URL is not a valid URL")] // internal whitespace + [InlineData("notaurl", "The configured Feature Flags agentless URL is not a valid absolute URL")] + [InlineData("https://flags.example.com bad", "The configured Feature Flags agentless URL is not a valid URL")] // internal whitespace public void RejectsInvalidBaseUrl(string baseUrl, string expectedError) { AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) From 4e6232bafed9809aad269594857fc87fb0dbe389 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Aug 2026 14:45:21 +0300 Subject: [PATCH 06/62] fix(feature-flags): redact agentless base URL telemetry and reject malformed sites safely --- .../Agentless/AgentlessEndpoint.cs | 31 +++++++++++++++++-- .../FeatureFlags/FeatureFlagsSettings.cs | 7 +++-- .../FeatureFlags/AgentlessEndpointTests.cs | 11 +++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index 757691bfa4cf..e0f60f1c4e9d 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -72,13 +72,25 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, out Age return false; } - var managed = new UriBuilder("https", ManagedHostPrefix + trimmedSite!.ToLowerInvariant()) { Path = DefaultPath }; + var managedHost = ManagedHostPrefix + trimmedSite!.ToLowerInvariant(); + if (managedHost.Contains("://") || HasWhitespace(managedHost)) + { + error = "The configured Datadog site is not valid"; + return false; + } + + if (!Uri.TryCreate($"https://{managedHost}{DefaultPath}", UriKind.Absolute, out var managedUri)) + { + error = "The configured Datadog site is not valid"; + return false; + } + if (!StringUtil.IsNullOrEmpty(env)) { - managed.Query = "dd_env=" + Uri.EscapeDataString(env!); + managedUri = new UriBuilder(managedUri) { Query = "dd_env=" + Uri.EscapeDataString(env!) }.Uri; } - endpoint = new AgentlessEndpoint(managed.Uri, isManaged: true); + endpoint = new AgentlessEndpoint(managedUri, isManaged: true); return true; } @@ -113,4 +125,17 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, out Age endpoint = new AgentlessEndpoint(custom, isManaged: false); return true; } + + private static bool HasWhitespace(string value) + { + foreach (var c in value) + { + if (char.IsWhiteSpace(c)) + { + return true; + } + } + + return false; + } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 71677f39b508..e25c97fd0dfa 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -62,9 +62,10 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr Source = ResolveSource(enabled, configuredSource, legacyEnabled); - AgentlessBaseUrl = config - .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessBaseUrl) - .AsString(url => !StringUtil.IsNullOrEmpty(url?.Trim())); + var agentlessBaseUrl = config + .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessBaseUrl) + .AsRedactedString(); + AgentlessBaseUrl = !StringUtil.IsNullOrEmpty(agentlessBaseUrl?.Trim()) ? agentlessBaseUrl : null; PollInterval = TimeSpan.FromSeconds( InRangeOrDefault( diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs index 049a6254025d..3a737c8937ff 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -104,6 +104,17 @@ public void RejectsWhitespaceOnlySiteWithoutBaseUrl() error.Should().Be("No Datadog site is configured"); } + [Theory] + [InlineData("https://datadoghq.com")] // user accidentally includes the scheme + [InlineData("data dog hq.com")] // internal spaces + [InlineData("datadoghq.com:99999")] // invalid port + public void RejectsMalformedSiteWithoutThrowing(string site) + { + AgentlessEndpoint.TryCreate(site, env: null, baseUrl: null, out var endpoint, out var error) + .Should().BeFalse(); + error.Should().Be("The configured Datadog site is not valid"); + } + [Fact] public void ErrorNeverContainsUrl() { From 7ee3cf37bbe143311666245edd39ff05b1886c75 Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Aug 2026 14:48:30 +0300 Subject: [PATCH 07/62] fix: remove unnecessary null-forgiving operators in AgentlessEndpoint --- .../Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index e0f60f1c4e9d..657dbfd2d673 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -72,7 +72,7 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, out Age return false; } - var managedHost = ManagedHostPrefix + trimmedSite!.ToLowerInvariant(); + var managedHost = ManagedHostPrefix + trimmedSite.ToLowerInvariant(); if (managedHost.Contains("://") || HasWhitespace(managedHost)) { error = "The configured Datadog site is not valid"; @@ -95,7 +95,7 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, out Age } // A URL with internal whitespace is malformed, and Uri parsing is lenient enough to accept it. - foreach (var character in configured!) + foreach (var character in configured) { if (char.IsWhiteSpace(character)) { From 02a6ac6764e466f7524183447cf7aeb8b0adb95a Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Aug 2026 14:59:00 +0300 Subject: [PATCH 08/62] fix: drop redundant HasWhitespace check, defer to Uri.TryCreate per review --- .../Agentless/AgentlessEndpoint.cs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index 657dbfd2d673..dfa5d032871b 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -73,7 +73,11 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, out Age } var managedHost = ManagedHostPrefix + trimmedSite.ToLowerInvariant(); - if (managedHost.Contains("://") || HasWhitespace(managedHost)) + // A site accidentally set to e.g. "https://datadoghq.com" would produce a host like + // "ufc-server.ff-cdn.https://datadoghq.com" which Uri.TryCreate accepts as valid + // (treating the "//" as a path separator). Catch it explicitly; whitespace and + // invalid ports are already rejected by TryCreate. + if (managedHost.Contains("://")) { error = "The configured Datadog site is not valid"; return false; @@ -125,17 +129,4 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, out Age endpoint = new AgentlessEndpoint(custom, isManaged: false); return true; } - - private static bool HasWhitespace(string value) - { - foreach (var c in value) - { - if (char.IsWhiteSpace(c)) - { - return true; - } - } - - return false; - } } From 7f72e7fe2791740a0143bfb58fe7cb7219af4b61 Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Aug 2026 15:13:13 +0300 Subject: [PATCH 09/62] fix: reword credential comment and remove unnecessary null-forgiving operator on env --- .../Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index dfa5d032871b..ffd17c5d15bf 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -40,7 +40,7 @@ private AgentlessEndpoint(Uri uri, bool isManaged) /// /// Gets a value indicating whether this is the endpoint derived from the site. The API key is /// only sent there: a custom endpoint reports its own authentication failure rather than - /// having the credential guessed onto it. + /// having the credential leaked to it. /// public bool IsManaged { get; } @@ -91,7 +91,7 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, out Age if (!StringUtil.IsNullOrEmpty(env)) { - managedUri = new UriBuilder(managedUri) { Query = "dd_env=" + Uri.EscapeDataString(env!) }.Uri; + managedUri = new UriBuilder(managedUri) { Query = "dd_env=" + Uri.EscapeDataString(env) }.Uri; } endpoint = new AgentlessEndpoint(managedUri, isManaged: true); From fb32e7d19017356abbaa58115984e1558f77d728 Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Aug 2026 15:48:36 +0300 Subject: [PATCH 10/62] refactor(feature-flags): use config framework for source resolution and validation telemetry --- .../FeatureFlags/FeatureFlagsSettings.cs | 122 ++++++++---------- .../FeatureFlags/SourceSelection.cs | 20 +++ 2 files changed, 72 insertions(+), 70 deletions(-) create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index e25c97fd0dfa..cffad6588ee0 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -7,6 +7,7 @@ using System; using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.ConfigurationSources.Telemetry; using Datadog.Trace.Configuration.Telemetry; using Datadog.Trace.Logging; using Datadog.Trace.Telemetry; @@ -47,7 +48,17 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr #pragma warning disable 618 // superseded, but still honoured so existing adopters keep their source var legacyEnabled = config.WithKeys(ConfigurationKeys.FeatureFlags.FlaggingProviderEnabled).AsBool(); #pragma warning restore 618 - var configuredSource = config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource).AsString(); + + // Use GetAsClass so the config framework records both the raw string and the resolved + // value in configuration telemetry. The converter maps recognised values to a + // SourceSelection record, returns Failure for unrecognised values (so the framework + // records the fallback), and returns Failure for null/empty (so "not set" also falls + // back to the default, which is null — meaning "not explicitly set"). + var configuredSource = config + .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource) + .GetAsClass( + validator: null, + converter: ConvertSource); if (legacyEnabled is not null) { @@ -65,25 +76,21 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr var agentlessBaseUrl = config .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessBaseUrl) .AsRedactedString(); - AgentlessBaseUrl = !StringUtil.IsNullOrEmpty(agentlessBaseUrl?.Trim()) ? agentlessBaseUrl : null; + AgentlessBaseUrl = !StringUtil.IsNullOrWhiteSpace(agentlessBaseUrl) ? agentlessBaseUrl : null; PollInterval = TimeSpan.FromSeconds( - InRangeOrDefault( - config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds).AsInt32(), - ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds, - DefaultPollIntervalSeconds, - MaxPollIntervalSeconds)); + config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessPollIntervalSeconds) + .AsInt32(DefaultPollIntervalSeconds, v => v > 0 && v <= MaxPollIntervalSeconds) + .Value); RequestTimeout = TimeSpan.FromSeconds( - InRangeOrDefault( - config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds).AsInt32(), - ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds, - DefaultRequestTimeoutSeconds, - maximumSeconds: null)); + config.WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessRequestTimeoutSeconds) + .AsInt32(DefaultRequestTimeoutSeconds, v => v > 0) + .Value); Site = config .WithKeys(ConfigurationKeys.Site) - .AsString(DefaultSite, site => !StringUtil.IsNullOrEmpty(site?.Trim())); + .AsString(DefaultSite, site => !StringUtil.IsNullOrWhiteSpace(site)); Env = config.WithKeys(ConfigurationKeys.Environment).AsString(); @@ -145,85 +152,60 @@ public static FeatureFlagsSettings FromDefaultSource() => new(GlobalConfigurationSource.Instance, TelemetryFactory.Config); /// - /// Resolves the delivery source. Shared across tracers, so the ordering is deliberate: - /// the stable kill switch wins over everything, an explicit source wins over the legacy key - /// (and fails closed when unrecognised), the legacy key grandfathers existing adopters onto - /// Remote Configuration, and everything else defaults to agentless. + /// Converts a configuration source string to a . + /// Used as the converter for GetAsClass so the + /// config framework records both the raw string and the resolved value in telemetry. /// - internal static FeatureFlagsSource ResolveSource(bool? enabled, string? configuredSource, bool? legacyEnabled) + private static ParsingResult ConvertSource(string? value) { - var normalizedSource = NormalizeSource(configuredSource); - - if (enabled == false) + if (value is null) { - return FeatureFlagsSource.Disabled; + return ParsingResult.Failure(); } - if (normalizedSource is not null) + var normalized = value.Trim().ToLowerInvariant(); + if (normalized.Length == 0) { - switch (normalizedSource) - { - case AgentlessSourceName: - return FeatureFlagsSource.Agentless; - case RemoteConfigSourceName: - return FeatureFlagsSource.RemoteConfig; - case OfflineSourceName: - // Reserved fail-closed sentinel: the provider is intentionally off, so no warning. - return FeatureFlagsSource.Disabled; - default: - Log.Warning( - "Unsupported {SourceKey} value '{Source}'. Feature Flags are disabled.", - ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource, - normalizedSource); - return FeatureFlagsSource.Disabled; - } + return ParsingResult.Failure(); } - // The legacy key only grandfathers adopters who have not migrated: an explicit new-key - // value (true or false) takes precedence, so the legacy key is consulted only when the - // new key was left unset. - if (enabled is null && legacyEnabled is not null) + return normalized switch { - return legacyEnabled.Value ? FeatureFlagsSource.RemoteConfig : FeatureFlagsSource.Disabled; - } - - return FeatureFlagsSource.Agentless; + AgentlessSourceName => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.Agentless, isValid: true)), + RemoteConfigSourceName => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.RemoteConfig, isValid: true)), + OfflineSourceName => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.Disabled, isValid: true)), + _ => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.Disabled, isValid: false)), + }; } /// - /// An empty or whitespace-only source is semantically unset, not an unrecognised value. + /// Resolves the delivery source. Shared across tracers, so the ordering is deliberate: + /// the stable kill switch wins over everything, an explicit source wins over the legacy key + /// (and fails closed when unrecognised), the legacy key grandfathers existing adopters onto + /// Remote Configuration, and everything else defaults to agentless. /// - private static string? NormalizeSource(string? configuredSource) + private static FeatureFlagsSource ResolveSource(bool? enabled, SourceSelection? configuredSource, bool? legacyEnabled) { - if (configuredSource is null) + if (enabled == false) { - return null; + return FeatureFlagsSource.Disabled; } - var normalized = configuredSource.Trim().ToLowerInvariant(); - return normalized.Length == 0 ? null : normalized; - } - - internal static int InRangeOrDefault(int? configured, string key, int defaultSeconds, int? maximumSeconds) - { - if (configured is null) + if (configuredSource is not null) { - return defaultSeconds; + // "offline" is a reserved fail-closed sentinel: the provider is intentionally off. + // An invalid value also fails closed. Both are mapped to Disabled by the converter. + return configuredSource.Source; } - var value = configured.Value; - if (value <= 0 || (maximumSeconds is { } maximum && value > maximum)) + // The legacy key only grandfathers adopters who have not migrated: an explicit new-key + // value (true or false) takes precedence, so the legacy key is consulted only when the + // new key was left unset. + if (enabled is null && legacyEnabled is not null) { - // A non-positive interval would turn polling into a tight loop against the endpoint, - // so an out-of-range value is rejected rather than honoured. - Log.Warning( - "Invalid value {Key}={Value}. Using {Default} seconds instead.", - key, - value, - defaultSeconds); - return defaultSeconds; + return legacyEnabled.Value ? FeatureFlagsSource.RemoteConfig : FeatureFlagsSource.Disabled; } - return value; + return FeatureFlagsSource.Agentless; } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs b/tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs new file mode 100644 index 000000000000..960006d8c009 --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs @@ -0,0 +1,20 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.FeatureFlags; + +/// +/// A wrapper used as the converter result type so the config framework records telemetry, +/// while allowing to distinguish "not set" (null) from +/// "set to an invalid value" (non-null with ). +/// +internal sealed class SourceSelection(FeatureFlagsSource source, bool isValid) +{ + public FeatureFlagsSource Source { get; } = source; + + public bool IsValid { get; } = isValid; +} From 2911d785e826636a40748d04ca379af32b9201cd Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Aug 2026 16:00:34 +0300 Subject: [PATCH 11/62] fix: mark DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL as sensitive in yaml --- .../Datadog.Trace/Configuration/supported-configurations.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml index eef60327d1c8..33a8c0f6a63b 100644 --- a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml +++ b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml @@ -3199,6 +3199,7 @@ supportedConfigurations: - implementation: A scope: managed type: string + sensitive: true default: null product: FeatureFlags const_name: FeatureFlagsConfigurationSourceAgentlessBaseUrl From 7af7bc53e90b7c53fa0be3832eff1ed390a9dd09 Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 17 Aug 2026 16:44:14 +0300 Subject: [PATCH 12/62] refactor: pass site/env/apiKey via constructor instead of re-reading from config --- .../FeatureFlags/FeatureFlagsSettings.cs | 21 +++++++++++-------- .../FeatureFlags/FeatureFlagsSettingsTests.cs | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index cffad6588ee0..488bf1e79f97 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -37,7 +37,7 @@ internal sealed class FeatureFlagsSettings private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(FeatureFlagsSettings)); - public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry) + public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry, string? site, string? env, string? apiKey) { source ??= NullConfigurationSource.Instance; var config = new ConfigurationBuilder(source, telemetry); @@ -88,13 +88,9 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr .AsInt32(DefaultRequestTimeoutSeconds, v => v > 0) .Value); - Site = config - .WithKeys(ConfigurationKeys.Site) - .AsString(DefaultSite, site => !StringUtil.IsNullOrWhiteSpace(site)); - - Env = config.WithKeys(ConfigurationKeys.Environment).AsString(); - - ApiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString(); + Site = StringUtil.IsNullOrWhiteSpace(site) ? DefaultSite : site; + Env = env; + ApiKey = apiKey; var initializationTimeoutMs = config .WithKeys(ConfigurationKeys.FeatureFlags.FlaggingProviderInitializationTimeoutMs) @@ -149,7 +145,14 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr public TimeSpan InitializationTimeout { get; } public static FeatureFlagsSettings FromDefaultSource() - => new(GlobalConfigurationSource.Instance, TelemetryFactory.Config); + { + var source = GlobalConfigurationSource.Instance; + var config = new ConfigurationBuilder(source, TelemetryFactory.Config); + var site = config.WithKeys(ConfigurationKeys.Site).AsString(DefaultSite, s => !StringUtil.IsNullOrWhiteSpace(s)); + var env = config.WithKeys(ConfigurationKeys.Environment).AsString(); + var apiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString(); + return new FeatureFlagsSettings(source, TelemetryFactory.Config, site, env, apiKey); + } /// /// Converts a configuration source string to a . diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index ec8be148a134..ebdeaa734cf9 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -179,6 +179,6 @@ private static FeatureFlagsSettings CreateSettings( collection[key] = value; } - return new FeatureFlagsSettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance); + return new FeatureFlagsSettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance, site: null, env: null, apiKey: null); } } From 8f406d94371dd7657d6d57569a9cfe5f729fda88 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 18 Aug 2026 15:24:46 +0300 Subject: [PATCH 13/62] [FeatureFlags] Resolve the configuration source in a single telemetry-reported read --- .../Configuration/TracerSettings.cs | 11 ++ .../FeatureFlags/FeatureFlagsSettings.cs | 115 ++++++++---------- .../FeatureFlags/SourceSelection.cs | 20 --- .../FeatureFlags/FeatureFlagsSettingsTests.cs | 24 +++- 4 files changed, 88 insertions(+), 82 deletions(-) delete mode 100644 tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index 6e04e7a41e71..1ce04feb0861 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -15,6 +15,7 @@ using Datadog.Trace.Configuration.ConfigurationSources.Telemetry; using Datadog.Trace.Configuration.Telemetry; using Datadog.Trace.DataStreamsMonitoring.TransactionTracking; +using Datadog.Trace.FeatureFlags; using Datadog.Trace.Logging; using Datadog.Trace.Logging.DirectSubmission; using Datadog.Trace.PlatformHelpers; @@ -888,6 +889,11 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = Manager = new(source, this, telemetry, errorLog); + // Created after the manager so the environment can be taken from the initial mutable + // settings, which also honour the "env" entry of DD_TAGS. It is captured once, so a + // later dynamic-configuration change to "env" does not move the agentless endpoint. + FeatureFlags = new FeatureFlagsSettings(source, telemetry, Manager.InitialMutableSettings.Environment); + // OTLP span metrics require OTLP trace export (see TracerManagerFactory.GetAgentWriter). // Force to false otherwise, even if explicitly requested. if (OtelTracesSpanMetricsEnabled && !Manager.InitialExporterSettings.IsOtlpTraceExport) @@ -1500,6 +1506,11 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = /// internal bool IsSpanEnrichmentEnabled { get; } + /// + /// Gets the Feature Flags settings, which select where flag configuration is delivered from. + /// + internal FeatureFlagsSettings FeatureFlags { get; } + /// /// Gets a value indicating whether partial flush is enabled /// diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 488bf1e79f97..f9c804f30364 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -10,7 +10,6 @@ using Datadog.Trace.Configuration.ConfigurationSources.Telemetry; using Datadog.Trace.Configuration.Telemetry; using Datadog.Trace.Logging; -using Datadog.Trace.Telemetry; using Datadog.Trace.Util; namespace Datadog.Trace.FeatureFlags; @@ -37,7 +36,7 @@ internal sealed class FeatureFlagsSettings private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(FeatureFlagsSettings)); - public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry, string? site, string? env, string? apiKey) + public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry, string? env) { source ??= NullConfigurationSource.Instance; var config = new ConfigurationBuilder(source, telemetry); @@ -49,17 +48,6 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr var legacyEnabled = config.WithKeys(ConfigurationKeys.FeatureFlags.FlaggingProviderEnabled).AsBool(); #pragma warning restore 618 - // Use GetAsClass so the config framework records both the raw string and the resolved - // value in configuration telemetry. The converter maps recognised values to a - // SourceSelection record, returns Failure for unrecognised values (so the framework - // records the fallback), and returns Failure for null/empty (so "not set" also falls - // back to the default, which is null — meaning "not explicitly set"). - var configuredSource = config - .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource) - .GetAsClass( - validator: null, - converter: ConvertSource); - if (legacyEnabled is not null) { #pragma warning disable 618 @@ -71,7 +59,26 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr #pragma warning restore 618 } - Source = ResolveSource(enabled, configuredSource, legacyEnabled); + // The source is resolved in a single read so configuration telemetry reports the value we + // actually use. Shared across tracers, so the precedence is deliberate: the stable kill + // switch wins over everything (expressed as a validator that rejects any configured value), + // an explicit source wins over the legacy key, the legacy key grandfathers existing + // adopters onto Remote Configuration, and everything else defaults to agentless. + DefaultResult defaultSource = enabled switch + { + false => new(FeatureFlagsSource.Disabled, OfflineSourceName), + null when legacyEnabled is not null => legacyEnabled.Value + ? new(FeatureFlagsSource.RemoteConfig, RemoteConfigSourceName) + : new(FeatureFlagsSource.Disabled, OfflineSourceName), + _ => new(FeatureFlagsSource.Agentless, AgentlessSourceName), + }; + + Source = config + .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource) + .GetAs( + defaultSource, + validator: enabled == false ? static _ => false : static _ => true, + converter: ConvertSource); var agentlessBaseUrl = config .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessBaseUrl) @@ -88,9 +95,15 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr .AsInt32(DefaultRequestTimeoutSeconds, v => v > 0) .Value); - Site = StringUtil.IsNullOrWhiteSpace(site) ? DefaultSite : site; + // DD_SITE and DD_API_KEY are not extracted on TracerSettings, so they are read here as the + // other product settings do (TelemetrySettings, DirectLogSubmissionSettings). DD_ENV is + // passed in, because TracerSettings also honours the "env" entry of DD_TAGS and re-reading + // the key alone would disagree with the rest of the tracer. + Site = config + .WithKeys(ConfigurationKeys.Site) + .AsString(DefaultSite, static site => !StringUtil.IsNullOrWhiteSpace(site)); Env = env; - ApiKey = apiKey; + ApiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString(); var initializationTimeoutMs = config .WithKeys(ConfigurationKeys.FeatureFlags.FlaggingProviderInitializationTimeoutMs) @@ -144,71 +157,51 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr /// public TimeSpan InitializationTimeout { get; } - public static FeatureFlagsSettings FromDefaultSource() - { - var source = GlobalConfigurationSource.Instance; - var config = new ConfigurationBuilder(source, TelemetryFactory.Config); - var site = config.WithKeys(ConfigurationKeys.Site).AsString(DefaultSite, s => !StringUtil.IsNullOrWhiteSpace(s)); - var env = config.WithKeys(ConfigurationKeys.Environment).AsString(); - var apiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString(); - return new FeatureFlagsSettings(source, TelemetryFactory.Config, site, env, apiKey); - } - /// - /// Converts a configuration source string to a . - /// Used as the converter for GetAsClass so the - /// config framework records both the raw string and the resolved value in telemetry. + /// Converts a configured source name to a . A blank value is + /// treated as unset so the default applies, and an unrecognised one fails closed: it resolves + /// to rather than falling back to a billed delivery + /// path, which is why it is reported as a successful conversion. /// - private static ParsingResult ConvertSource(string? value) + private static ParsingResult ConvertSource(string? value) { - if (value is null) + if (StringUtil.IsNullOrWhiteSpace(value)) { - return ParsingResult.Failure(); + return ParsingResult.Failure(); } - var normalized = value.Trim().ToLowerInvariant(); - if (normalized.Length == 0) + // Compared without allocating. A value with surrounding whitespace is trimmed only once the + // direct comparisons have failed, so the common path stays allocation-free. + if (TryMatch(value, out var source) || TryMatch(value.Trim(), out source)) { - return ParsingResult.Failure(); + return ParsingResult.Success(source); } - return normalized switch - { - AgentlessSourceName => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.Agentless, isValid: true)), - RemoteConfigSourceName => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.RemoteConfig, isValid: true)), - OfflineSourceName => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.Disabled, isValid: true)), - _ => ParsingResult.Success(new SourceSelection(FeatureFlagsSource.Disabled, isValid: false)), - }; + return ParsingResult.Success(FeatureFlagsSource.Disabled); } - /// - /// Resolves the delivery source. Shared across tracers, so the ordering is deliberate: - /// the stable kill switch wins over everything, an explicit source wins over the legacy key - /// (and fails closed when unrecognised), the legacy key grandfathers existing adopters onto - /// Remote Configuration, and everything else defaults to agentless. - /// - private static FeatureFlagsSource ResolveSource(bool? enabled, SourceSelection? configuredSource, bool? legacyEnabled) + private static bool TryMatch(string value, out FeatureFlagsSource source) { - if (enabled == false) + if (string.Equals(value, AgentlessSourceName, StringComparison.OrdinalIgnoreCase)) { - return FeatureFlagsSource.Disabled; + source = FeatureFlagsSource.Agentless; + return true; } - if (configuredSource is not null) + if (string.Equals(value, RemoteConfigSourceName, StringComparison.OrdinalIgnoreCase)) { - // "offline" is a reserved fail-closed sentinel: the provider is intentionally off. - // An invalid value also fails closed. Both are mapped to Disabled by the converter. - return configuredSource.Source; + source = FeatureFlagsSource.RemoteConfig; + return true; } - // The legacy key only grandfathers adopters who have not migrated: an explicit new-key - // value (true or false) takes precedence, so the legacy key is consulted only when the - // new key was left unset. - if (enabled is null && legacyEnabled is not null) + // "offline" is a reserved fail-closed sentinel: the provider is intentionally off. + if (string.Equals(value, OfflineSourceName, StringComparison.OrdinalIgnoreCase)) { - return legacyEnabled.Value ? FeatureFlagsSource.RemoteConfig : FeatureFlagsSource.Disabled; + source = FeatureFlagsSource.Disabled; + return true; } - return FeatureFlagsSource.Agentless; + source = FeatureFlagsSource.Disabled; + return false; } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs b/tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs deleted file mode 100644 index 960006d8c009..000000000000 --- a/tracer/src/Datadog.Trace/FeatureFlags/SourceSelection.cs +++ /dev/null @@ -1,20 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// - -#nullable enable - -namespace Datadog.Trace.FeatureFlags; - -/// -/// A wrapper used as the converter result type so the config framework records telemetry, -/// while allowing to distinguish "not set" (null) from -/// "set to an invalid value" (non-null with ). -/// -internal sealed class SourceSelection(FeatureFlagsSource source, bool isValid) -{ - public FeatureFlagsSource Source { get; } = source; - - public bool IsValid { get; } = isValid; -} diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index ebdeaa734cf9..efd24e3d9419 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -149,6 +149,28 @@ public void ReadsAgentlessBaseUrl(string configured, string? expected) settings.AgentlessBaseUrl.Should().Be(expected); } + [Fact] + public void ReadsSiteAndApiKeyFromTheConfigurationSource() + { + var settings = CreateSettings( + null, + null, + null, + (ConfigurationKeys.Site, "datadoghq.eu"), + (ConfigurationKeys.ApiKey, "an-api-key")); + + settings.Site.Should().Be("datadoghq.eu"); + settings.ApiKey.Should().Be("an-api-key"); + } + + [Theory] + // A blank site is rejected in favour of the default, so the managed endpoint stays resolvable. + [InlineData("")] + [InlineData(" ")] + public void FallsBackToTheDefaultSite(string configured) + => CreateSettings(null, null, null, (ConfigurationKeys.Site, configured)) + .Site.Should().Be(FeatureFlagsSettings.DefaultSite); + private static FeatureFlagsSettings CreateSettings( string? enabled, string? source, @@ -179,6 +201,6 @@ private static FeatureFlagsSettings CreateSettings( collection[key] = value; } - return new FeatureFlagsSettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance, site: null, env: null, apiKey: null); + return new FeatureFlagsSettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance, env: null); } } From ada921dbd0ecf91fd29888427c623f5209ae11cc Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 18 Aug 2026 17:00:18 +0300 Subject: [PATCH 14/62] [FeatureFlags] Make AgentlessEndpoint a class so a failed TryCreate yields null --- .../Agentless/AgentlessEndpoint.cs | 13 +++-- .../FeatureFlags/AgentlessEndpointTests.cs | 51 +++++++++---------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index ffd17c5d15bf..2ebc7b03b3b3 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -6,14 +6,17 @@ #nullable enable using System; +using System.Diagnostics.CodeAnalysis; using Datadog.Trace.Util; namespace Datadog.Trace.FeatureFlags.Agentless; /// -/// The agentless endpoint, derived from the Datadog site or a custom base URL. +/// The agentless endpoint, derived from the Datadog site or a custom base URL. A class rather +/// than a struct so that "no endpoint" is null instead of a default instance whose +/// non-nullable is null; it is built once per process, so the allocation is free. /// -internal readonly struct AgentlessEndpoint +internal sealed class AgentlessEndpoint { /// /// Canonical rules-based server path, appended to the managed CDN host and to custom base @@ -54,12 +57,12 @@ private AgentlessEndpoint(Uri uri, bool isManaged) /// The Datadog site, for example datadoghq.com. /// The configured environment, or null. /// The configured endpoint override, or null. - /// The resulting endpoint. + /// The resulting endpoint, or null when none could be built. /// Why the configured base URL was rejected. Never contains the URL, which may carry credentials. /// true when an endpoint could be built. - public static bool TryCreate(string? site, string? env, string? baseUrl, out AgentlessEndpoint endpoint, out string? error) + public static bool TryCreate(string? site, string? env, string? baseUrl, [NotNullWhen(true)] out AgentlessEndpoint? endpoint, out string? error) { - endpoint = default; + endpoint = null; error = null; var configured = baseUrl?.Trim(); diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs index 3a737c8937ff..2fbda55d3836 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -23,36 +23,23 @@ public class AgentlessEndpointTests [InlineData("ddog-gov.com", "https://ufc-server.ff-cdn.ddog-gov.com" + DefaultPath)] // govcloud public void DerivesManagedEndpointFromSite(string site, string expected) { - AgentlessEndpoint.TryCreate(site, env: null, baseUrl: null, out var endpoint, out var error) - .Should().BeTrue(); - error.Should().BeNull(); + var endpoint = Create(site); + endpoint.IsManaged.Should().BeTrue(); endpoint.Uri.ToString().Should().Be(expected); } [Fact] public void AddsDdEnvWhenEnvIsConfigured() - { - AgentlessEndpoint.TryCreate("datadoghq.com", env: "production", baseUrl: null, out var endpoint, out _) - .Should().BeTrue(); - endpoint.Uri.Query.Should().Be("?dd_env=production"); - } + => Create("datadoghq.com", env: "production").Uri.Query.Should().Be("?dd_env=production"); [Fact] public void DoesNotAddDdEnvWhenEnvIsNull() - { - AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: null, out var endpoint, out _) - .Should().BeTrue(); - endpoint.Uri.Query.Should().BeEmpty(); - } + => Create("datadoghq.com").Uri.Query.Should().BeEmpty(); [Fact] public void EscapesDdEnvValue() - { - AgentlessEndpoint.TryCreate("datadoghq.com", env: "my env&test", baseUrl: null, out var endpoint, out _) - .Should().BeTrue(); - endpoint.Uri.Query.Should().Be("?dd_env=my%20env%26test"); - } + => Create("datadoghq.com", env: "my env&test").Uri.Query.Should().Be("?dd_env=my%20env%26test"); [Theory] [InlineData("https://flags.example.com", "https://flags.example.com" + DefaultPath)] @@ -61,9 +48,8 @@ public void EscapesDdEnvValue() [InlineData("https://flags.example.com/ufc?custom=query", "https://flags.example.com/ufc?custom=query")] public void CustomEndpointReceivesCanonicalPathForOriginOnly(string baseUrl, string expected) { - AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) - .Should().BeTrue(); - error.Should().BeNull(); + var endpoint = Create("datadoghq.com", baseUrl: baseUrl); + endpoint.IsManaged.Should().BeFalse(); endpoint.Uri.ToString().Should().Be(expected); } @@ -71,11 +57,7 @@ public void CustomEndpointReceivesCanonicalPathForOriginOnly(string baseUrl, str [Theory] [InlineData("http://localhost:8080/ufc")] // http accepted for custom endpoints public void CustomEndpointAcceptsHttp(string baseUrl) - { - AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) - .Should().BeTrue(); - endpoint.IsManaged.Should().BeFalse(); - } + => Create("datadoghq.com", baseUrl: baseUrl).IsManaged.Should().BeFalse(); [Theory] [InlineData("ftp://flags.example.com", "The configured Feature Flags agentless URL must use HTTP or HTTPS")] @@ -86,6 +68,7 @@ public void RejectsInvalidBaseUrl(string baseUrl, string expectedError) AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be(expectedError); + endpoint.Should().BeNull(); } [Fact] @@ -94,6 +77,7 @@ public void RejectsEmptySiteWithoutBaseUrl() AgentlessEndpoint.TryCreate(site: null, env: null, baseUrl: null, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be("No Datadog site is configured"); + endpoint.Should().BeNull(); } [Fact] @@ -102,6 +86,7 @@ public void RejectsWhitespaceOnlySiteWithoutBaseUrl() AgentlessEndpoint.TryCreate(" ", env: null, baseUrl: null, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be("No Datadog site is configured"); + endpoint.Should().BeNull(); } [Theory] @@ -113,6 +98,7 @@ public void RejectsMalformedSiteWithoutThrowing(string site) AgentlessEndpoint.TryCreate(site, env: null, baseUrl: null, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be("The configured Datadog site is not valid"); + endpoint.Should().BeNull(); } [Fact] @@ -124,4 +110,17 @@ public void ErrorNeverContainsUrl() error.Should().NotContain("user"); error.Should().NotContain("pass"); } + + // Builds an endpoint that is expected to be valid, and returns it as non-nullable so the + // assertions can read it directly. Throwing rather than asserting keeps the compiler's nullable + // analysis satisfied without a null-forgiving operator, which would let an assertion be + // silently skipped if the endpoint were ever null. + private static AgentlessEndpoint Create(string? site, string? env = null, string? baseUrl = null) + { + AgentlessEndpoint.TryCreate(site, env, baseUrl, out var endpoint, out var error) + .Should().BeTrue(); + error.Should().BeNull(); + + return endpoint ?? throw new InvalidOperationException("TryCreate reported success without producing an endpoint."); + } } From c224e2d20c552327a8223fcb7a8530189b1137ba Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 14:53:55 +0300 Subject: [PATCH 15/62] feat(feature-flags): add UfcConfigurationParser for JSON:API envelope validation --- .../Agentless/UfcConfigurationParser.cs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs new file mode 100644 index 000000000000..58bbfe2691e9 --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -0,0 +1,90 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using Datadog.Trace.Vendors.Newtonsoft.Json; +using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; + +namespace Datadog.Trace.FeatureFlags.Agentless; + +/// +/// Reads the JSON:API envelope returned by the agentless endpoint. +/// +internal static class UfcConfigurationParser +{ + private const string ResourceType = "universal-flag-configuration"; + + /// + /// Validates a JSON:API Universal Flag Configuration response and returns data.attributes, + /// which is the document the evaluator consumes. A raw UFC document is rejected, including from + /// a custom endpoint, so that every source agrees on one wire format. + /// + /// The response body. + /// The parsed configuration. + /// Why the payload was rejected. + /// true when the payload matches the contract. + public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfiguration? configuration, out string? error) + { + configuration = null; + error = null; + + JToken payload; + try + { + using var stringReader = new StringReader(body ?? string.Empty); + + // Timestamps stay strings: the model carries createdAt verbatim, and letting Newtonsoft + // turn it into a date would also make the type check below fail. + using var jsonReader = new JsonTextReader(stringReader) { DateParseHandling = DateParseHandling.None }; + payload = JToken.ReadFrom(jsonReader); + } + catch (Exception) + { + error = "Malformed UFC payload"; + return false; + } + + if (payload is not JObject + || payload["data"] is not JObject data + || data["type"]?.Value() != ResourceType) + { + error = "Expected a JSON:API Universal Flag Configuration resource"; + return false; + } + + if (data["attributes"] is not JObject attributes + || attributes["format"]?.Type != JTokenType.String + || attributes["createdAt"]?.Type != JTokenType.String + || attributes["environment"] is not JObject environment + || environment["name"]?.Type != JTokenType.String + || attributes["flags"] is not JObject) + { + error = "Expected a Universal Flag Configuration v1 object"; + return false; + } + + try + { + configuration = attributes.ToObject(); + } + catch (Exception) + { + configuration = null; + } + + if (configuration is null) + { + error = "Expected a Universal Flag Configuration v1 object"; + return false; + } + + return true; + } +} From 6a5f49a9091ac032d4919b3983e5302ad5094691 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 15:01:50 +0300 Subject: [PATCH 16/62] test(feature-flags): add UfcConfigurationParserTests for JSON:API envelope validation --- .../Agentless/AgentlessConfigurationSource.cs | 399 ++++++++++++++++++ .../UfcConfigurationParserTests.cs | 108 +++++ 2 files changed, 507 insertions(+) create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs create mode 100644 tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs new file mode 100644 index 000000000000..6d23a7045c31 --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -0,0 +1,399 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Agent.Transports; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using Datadog.Trace.Headers; +using Datadog.Trace.Logging; +using Datadog.Trace.Telemetry; +using Datadog.Trace.Util; + +namespace Datadog.Trace.FeatureFlags.Agentless; + +/// +/// Polls the agentless endpoint for flag configuration. Polling is billable, so it is only +/// started once application code has activated the provider. +/// +internal sealed class AgentlessConfigurationSource : IDisposable +{ + private const int MaxAttempts = 3; + private const double RetryJitter = 0.2; + + private static readonly TimeSpan FirstRetryMin = TimeSpan.FromSeconds(2); + private static readonly TimeSpan FirstRetryMax = TimeSpan.FromSeconds(10); + private static readonly TimeSpan SecondRetryMin = TimeSpan.FromSeconds(5); + private static readonly TimeSpan SecondRetryMax = TimeSpan.FromSeconds(30); + + // A jittered retry delay never drops below this, so a short poll interval cannot turn + // retries into a burst against the endpoint. + private static readonly TimeSpan MinRetryDelay = TimeSpan.FromSeconds(1); + + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AgentlessConfigurationSource)); + + private readonly IApiRequestFactory _requestFactory; + private readonly Uri _endpoint; + private readonly TimeSpan _pollInterval; + private readonly Func _applyConfiguration; + private readonly Func _waitAsync; + private readonly CancellationTokenSource _shutdown = new(); + private readonly Random _random = new(); + + // Only ever touched from the poll loop. + private readonly HashSet _loggedFailureCategories = new(); + private bool _malformedPayloadLogged; + private bool _applyFailureLogged; + private string? _etag; + + private int _started; + + internal AgentlessConfigurationSource( + Uri endpoint, + IApiRequestFactory requestFactory, + TimeSpan pollInterval, + Func applyConfiguration, + Func? waitAsync = null) + { + _endpoint = endpoint; + _requestFactory = requestFactory; + _pollInterval = pollInterval; + _applyConfiguration = applyConfiguration; + _waitAsync = waitAsync ?? Task.Delay; + } + + /// + /// Creates the source, or returns null when it cannot be operated: a base URL that is + /// not a URL, or the managed endpoint without an API key. Polling anyway would only produce + /// failures every interval. + /// + public static AgentlessConfigurationSource? Create(FeatureFlagsSettings settings, Func applyConfiguration) + { + if (!AgentlessEndpoint.TryCreate(settings.Site, settings.Env, settings.AgentlessBaseUrl, out var endpoint, out var error)) + { + Log.Error("Feature Flags agentless source is unavailable: {Error}", error); + return null; + } + + if (endpoint.IsManaged && StringUtil.IsNullOrEmpty(settings.ApiKey)) + { + Log.Error("Feature Flags agentless source requires an API key. Set DD_API_KEY, or point DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL at an endpoint of your own."); + return null; + } + + return new AgentlessConfigurationSource( + endpoint.Uri, + CreateRequestFactory(endpoint, settings), + settings.PollInterval, + applyConfiguration); + } + + /// + /// Starts polling. Idempotent. + /// + public void Start() + { + if (Interlocked.CompareExchange(ref _started, 1, 0) != 0) + { + return; + } + + // Deliberately not wrapped in Task.Run: this is called from provider initialization, which + // is waiting for the first configuration, so the first request should go out on the calling + // thread rather than queue behind whatever else is on the thread pool. + _ = RunAsync().ContinueWith(t => Log.Error(t.Exception, "Feature Flags agentless poll loop failed"), TaskContinuationOptions.OnlyOnFaulted); + } + + /// + /// Runs a single poll, including its in-tick retries. + /// + internal async Task PollAsync() + { + var result = default(PollResult); + + for (var attempt = 1; attempt <= MaxAttempts; attempt++) + { + result = await RequestAsync().ConfigureAwait(false); + + if (_shutdown.IsCancellationRequested) + { + // A shutdown mid-poll leaves the response unusable for state transitions: keep + // last-known-good and the current ETag. + return; + } + + if (!IsRetryable(result)) + { + break; + } + + if (attempt == MaxAttempts) + { + // Every attempt failed in a retryable way. Last-known-good stays in place. + WarnFailure(result, MaxAttempts); + return; + } + + await WaitAsync(RetryDelay(attempt)).ConfigureAwait(false); + + if (_shutdown.IsCancellationRequested) + { + return; + } + } + + if (_shutdown.IsCancellationRequested) + { + // A shutdown during the final attempt leaves the response unusable for state + // transitions: keep last-known-good and the current ETag. + return; + } + + await ApplyAsync(result).ConfigureAwait(false); + } + + public void Dispose() + { + // The request in flight is bounded by the request timeout, and the loop is never joined, + // so a shutdown does not wait for it. A poll that completes after disposal is prevented + // from applying its result by the shutdown check in PollAsync. + try + { + _shutdown.Cancel(); + } + catch (Exception ex) + { + Log.Debug(ex, "Error cancelling the Feature Flags agentless poll loop"); + } + } + + // The concrete type is returned rather than the interface because CA1859 asks for it on a + // private member, which is also why the signature varies by target framework. +#if NETCOREAPP + private static HttpClientRequestFactory CreateRequestFactory(AgentlessEndpoint endpoint, FeatureFlagsSettings settings) +#else + private static ApiWebRequestFactory CreateRequestFactory(AgentlessEndpoint endpoint, FeatureFlagsSettings settings) +#endif + { + var headers = new List> + { + // The endpoint serves gzip, and neither transport decompresses for us. + new("Accept-Encoding", "gzip"), + new(TelemetryConstants.ClientLibraryLanguageHeader, TracerConstants.Language), + new(TelemetryConstants.ClientLibraryVersionHeader, TracerConstants.ThreePartVersion), + + // Without this the poll is itself instrumented, producing a span per poll and letting + // auto-instrumentation recurse through the poller's own client. + new(HttpHeaderNames.TracingEnabled, "false"), + }; + + if (endpoint.IsManaged) + { + // A custom endpoint is left to report its own authentication failure rather than + // having the Datadog credential sent to it. + headers.Add(new(TelemetryConstants.ApiKeyHeader, settings.ApiKey!)); + } + +#if NETCOREAPP + return new HttpClientRequestFactory(endpoint.Uri, headers.ToArray(), timeout: settings.RequestTimeout); +#else + return new ApiWebRequestFactory(endpoint.Uri, headers.ToArray(), timeout: settings.RequestTimeout); +#endif + } + + private static bool IsRetryable(in PollResult result) + => result.StatusCode is not { } status || status is 408 or 429 or (>= 500 and <= 599); + + private async Task RunAsync() + { + Log.Debug("AgentlessConfigurationSource::RunAsync -> Enter"); + + while (!_shutdown.IsCancellationRequested) + { + try + { + await PollAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + Log.Debug(ex, "Feature Flags agentless poll failed unexpectedly"); + } + + // Fixed delay after completion, so polls never overlap. + await WaitAsync(_pollInterval).ConfigureAwait(false); + } + + Log.Debug("AgentlessConfigurationSource::RunAsync -> Exit"); + } + + private async Task WaitAsync(TimeSpan delay) + { + try + { + await _waitAsync(delay, _shutdown.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Shutting down + } + } + + private TimeSpan RetryDelay(int attempt) + { + var seconds = attempt == 1 + ? Clamp(_pollInterval.TotalSeconds / 6, FirstRetryMin, FirstRetryMax) + : Clamp(_pollInterval.TotalSeconds / 3, SecondRetryMin, SecondRetryMax); + + double jitter; + lock (_random) + { + jitter = 1 - RetryJitter + (_random.NextDouble() * RetryJitter * 2); + } + + return TimeSpan.FromSeconds(Math.Max(MinRetryDelay.TotalSeconds, seconds * jitter)); + + static double Clamp(double value, TimeSpan minimum, TimeSpan maximum) + => Math.Max(minimum.TotalSeconds, Math.Min(maximum.TotalSeconds, value)); + } + + private async Task RequestAsync() + { + try + { + var request = _requestFactory.Create(_endpoint); + if (_etag is { } etag) + { + request.AddHeader("If-None-Match", etag); + } + + using var response = await request.GetAsync().ConfigureAwait(false); + + // Only a 200 carries configuration; other bodies are never decoded as one. + var body = response.StatusCode == 200 ? await ReadBodyAsync(response).ConfigureAwait(false) : null; + return new PollResult(response.StatusCode, response.GetHeader("ETag"), body, error: null); + } + catch (Exception ex) + { + return new PollResult(statusCode: null, etag: null, body: null, error: ex); + } + } + + private async Task ReadBodyAsync(IApiResponse response) + { + var stream = await response.GetStreamAsync().ConfigureAwait(false); + GZipStream? decompressed = null; + + try + { + if (response.GetContentEncodingType() == ContentEncodingType.GZip) + { + decompressed = new GZipStream(stream, CompressionMode.Decompress); + } + + using var reader = new StreamReader(decompressed ?? stream, response.GetCharsetEncoding()); + return await reader.ReadToEndAsync().ConfigureAwait(false); + } + finally + { + decompressed?.Dispose(); + } + } + + private Task ApplyAsync(PollResult result) + { + switch (result.StatusCode) + { + case 304: + // Nothing changed, and the ETag stays as it is. + return Task.CompletedTask; + case 401 or 403: + WarnFailure(result, attempts: 1); + return Task.CompletedTask; + case not 200: + WarnFailure(result, attempts: 1); + return Task.CompletedTask; + } + + if (!UfcConfigurationParser.TryParse(result.Body, out var configuration, out var error)) + { + if (!_malformedPayloadLogged) + { + _malformedPayloadLogged = true; + Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", error); + } + + return Task.CompletedTask; + } + + if (!_applyConfiguration(configuration)) + { + if (!_applyFailureLogged) + { + _applyFailureLogged = true; + Log.Warning("Feature Flags agentless configuration could not be applied"); + } + + return Task.CompletedTask; + } + + // The ETag advances only once parsing and applying have both succeeded. Advancing on + // receipt would acknowledge a payload that was never applied, and every later poll would + // answer 304, pinning the process to stale configuration with no way back. + var newEtag = result.ETag?.Trim(); + _etag = StringUtil.IsNullOrEmpty(newEtag) ? null : newEtag; + + return Task.CompletedTask; + } + + /// + /// Warns once per failure category. A dead endpoint would otherwise produce a warning every + /// poll interval, indefinitely. + /// + private void WarnFailure(in PollResult result, int attempts) + { + var category = result.StatusCode switch + { + 401 or 403 => "authentication", + not null => "http", + _ => "request", + }; + + if (!_loggedFailureCategories.Add(category)) + { + return; + } + + switch (result.StatusCode) + { + case 401 or 403: + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); + break; + case not null: + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); + break; + default: + Log.Warning(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); + break; + } + } + + internal readonly struct PollResult(int? statusCode, string? etag, string? body, Exception? error) + { + public int? StatusCode { get; } = statusCode; + + public string? ETag { get; } = etag; + + public string? Body { get; } = body; + + public Exception? Error { get; } = error; + } +} diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs new file mode 100644 index 000000000000..14fc51ab85a7 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs @@ -0,0 +1,108 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System.Collections.Generic; +using Datadog.Trace.FeatureFlags.Agentless; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.FeatureFlags; + +public class UfcConfigurationParserTests +{ + private const string ValidEnvelope = """ + { "data": { "type": "universal-flag-configuration", + "attributes": { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, "flags": {} } } } + """; + + private const string Attributes = """ + { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, "flags": {} } + """; + + [Fact] + public void ParsesValidEnvelope() + { + UfcConfigurationParser.TryParse(ValidEnvelope, out var configuration, out var error) + .Should().BeTrue(); + + error.Should().BeNull(); + configuration.Should().NotBeNull(); + configuration!.Environment!.Name.Should().Be("production"); + configuration.Flags.Should().BeEmpty(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("not json")] + [InlineData("{ \"data\": ")] + public void RejectsMalformedJson(string? body) + { + UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + + configuration.Should().BeNull(); + error.Should().Be("Malformed UFC payload"); + } + + [Theory] + // A raw UFC document is rejected too, so every source agrees on one wire format. + [InlineData(Attributes)] + // Wrong resource type + [InlineData("""{ "data": { "type": "wrong-type", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] + // Missing data + [InlineData("""{ "meta": {} }""")] + // data is not an object + [InlineData("""{ "data": "string" }""")] + public void RejectsInvalidEnvelope(string body) + { + UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + + configuration.Should().BeNull(); + error.Should().Be("Expected a JSON:API Universal Flag Configuration resource"); + } + + [Theory] + // Missing format + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] + // Missing createdAt + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "environment": { "name": "prod" }, "flags": {} } } }""")] + // Missing environment + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "flags": {} } } }""")] + // environment.name is not a string + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": 123 }, "flags": {} } } }""")] + // Missing flags + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" } } } }""")] + // flags is not an object + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": [] } } }""")] + public void RejectsInvalidAttributes(string body) + { + UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + + configuration.Should().BeNull(); + error.Should().Be("Expected a Universal Flag Configuration v1 object"); + } + + [Fact] + public void ParsesFlagsFromEnvelope() + { + var body = """ + { "data": { "type": "universal-flag-configuration", + "attributes": { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, + "flags": { "test-flag": { "key": "test-flag", "enabled": true, "variationType": "BOOLEAN" } } } } } + """; + + UfcConfigurationParser.TryParse(body, out var configuration, out _).Should().BeTrue(); + + configuration!.Flags.Should().NotBeNull(); + configuration!.Flags!.Should().ContainKey("test-flag"); + configuration!.Flags!["test-flag"].Enabled.Should().BeTrue(); + } +} From 388ccfab524997d03fba21e44638764894688f5e Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 15:47:50 +0300 Subject: [PATCH 17/62] test(feature-flags): add AgentlessConfigurationSourceTests for poller retry, ETag, gzip, and shutdown --- .../AgentlessConfigurationSourceTests.cs | 284 ++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs new file mode 100644 index 000000000000..935c778b96f4 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -0,0 +1,284 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.FeatureFlags.Agentless; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using Datadog.Trace.TestHelpers.TransportHelpers; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.FeatureFlags; + +public class AgentlessConfigurationSourceTests +{ + private const string Body = """ + { "data": { "type": "universal-flag-configuration", + "attributes": { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, "flags": {} } } } + """; + + private static readonly Uri Endpoint = new("https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server"); + + [Fact] + public async Task AppliesConfigurationFromA200() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new TestApiRequest(uri, responseContent: Body)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().ContainSingle(); + applied[0].Environment!.Name.Should().Be("production"); + factory.RequestsSent.Should().ContainSingle(); + } + + [Fact] + public async Task SendsTheEtagOfTheLastAppliedConfiguration() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, responseContent: Body, responseHeaders: new() { { "ETag", "\"ufc-v1\"" } })); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + applied.Should().ContainSingle(); + + // Second poll should send If-None-Match + await source.PollAsync(); + factory.RequestsSent.Should().HaveCount(2); + factory.RequestsSent[1].ExtraHeaders.Should().ContainKey("If-None-Match"); + factory.RequestsSent[1].ExtraHeaders["If-None-Match"].Should().Be("\"ufc-v1\""); + } + + [Fact] + public async Task DoesNotApplyOn304() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 304, responseContent: "{}")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyOn401() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 401, responseContent: "Unauthorized")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyOnMalformedPayload() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 200, responseContent: "not json")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyAfterDisposal() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new TestApiRequest(uri, responseContent: Body)); + var source = CreateSource(factory, applied); + + // A shutdown mid-poll leaves the response unusable for a state transition. + source.Dispose(); + await source.PollAsync(); + + factory.RequestsSent.Should().ContainSingle(); + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyWhenDisposedAfterRequestSucceeds() + { + var applied = new List(); + AgentlessConfigurationSource? sourceRef = null; + var factory = new TestRequestFactory(uri => + { + var request = new DisposingApiRequest(uri, Body); + request.Source = sourceRef; + return request; + }); + using var source = CreateSource(factory, applied); + sourceRef = source; + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task RetriesOn500ThenAppliesOnSuccess() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error"), + uri => new TestApiRequest(uri, responseContent: Body)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().ContainSingle(); + factory.RequestsSent.Should().HaveCount(2); + } + + [Fact] + public async Task RetriesUpToMaxAttemptsOn500() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error"), + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error"), + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + factory.RequestsSent.Should().HaveCount(3); + } + + [Fact] + public async Task DoesNotRetryOn400() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 400, responseContent: "bad request")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + factory.RequestsSent.Should().ContainSingle(); + } + + [Fact] + public async Task HandlesGzipResponse() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new GzipApiRequest(uri, Body)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().ContainSingle(); + applied[0].Environment!.Name.Should().Be("production"); + } + + [Fact] + public async Task HandlesNetworkError() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new ThrowingApiRequest(uri), + uri => new ThrowingApiRequest(uri), + uri => new ThrowingApiRequest(uri)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + factory.RequestsSent.Should().HaveCount(3); + } + + private static AgentlessConfigurationSource CreateSource(TestRequestFactory factory, List applied) + => new( + Endpoint, + factory, + TimeSpan.FromSeconds(30), + configuration => + { + applied.Add(configuration); + return true; + }, + NoWait); + + private static Task NoWait(TimeSpan delay, CancellationToken cancellationToken) => Task.CompletedTask; + + private class ThrowingApiRequest(Uri endpoint) : TestApiRequest(endpoint) + { + public override Task GetAsync() => throw new IOException("The connection was refused"); + } + + private class GzipApiRequest(Uri endpoint, string body) : TestApiRequest(endpoint) + { + public override Task GetAsync() => Task.FromResult(new GzipApiResponse(body)); + } + + private class GzipApiResponse(string body) : IApiResponse + { + public int StatusCode => 200; + + public long ContentLength => -1; + + public string? ContentTypeHeader => "application/json"; + + public string? ContentEncodingHeader => "gzip"; + + public void Dispose() + { + } + + public string? GetHeader(string headerName) => null; + + public Encoding GetCharsetEncoding() => Encoding.UTF8; + + public ContentEncodingType GetContentEncodingType() => ContentEncodingType.GZip; + + public Task GetStreamAsync() + { + var compressed = new MemoryStream(); + using (var gzip = new GZipStream(compressed, CompressionMode.Compress, leaveOpen: true)) + { + var bytes = Encoding.UTF8.GetBytes(body); + gzip.Write(bytes, 0, bytes.Length); + } + + compressed.Position = 0; + return Task.FromResult(compressed); + } + } + + private class DisposingApiRequest(Uri endpoint, string body) : TestApiRequest(endpoint, responseContent: body) + { + public AgentlessConfigurationSource? Source { get; set; } + + public override Task GetAsync() + { + var response = base.GetAsync(); + // Simulate a shutdown arriving after the request completes but before ApplyAsync. + Source?.Dispose(); + return response; + } + } +} From 376b0ad357c49b63c002da739059ea6aa1c21df8 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Aug 2026 15:01:41 +0300 Subject: [PATCH 18/62] fix(feature-flags): reject non-string data.type, log failures as errors --- .../Agentless/AgentlessConfigurationSource.cs | 8 ++++---- .../FeatureFlags/Agentless/UfcConfigurationParser.cs | 1 + .../FeatureFlags/UfcConfigurationParserTests.cs | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 6d23a7045c31..7ec342a4fc6d 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -225,7 +225,7 @@ private async Task RunAsync() } catch (Exception ex) { - Log.Debug(ex, "Feature Flags agentless poll failed unexpectedly"); + Log.Error(ex, "Feature Flags agentless poll failed unexpectedly"); } // Fixed delay after completion, so polls never overlap. @@ -375,13 +375,13 @@ private void WarnFailure(in PollResult result, int attempts) switch (result.StatusCode) { case 401 or 403: - Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); + Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); break; case not null: - Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); + Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); break; default: - Log.Warning(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); + Log.Error(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); break; } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs index 58bbfe2691e9..a078f5f4374d 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -53,6 +53,7 @@ public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfigur if (payload is not JObject || payload["data"] is not JObject data + || data["type"]?.Type != JTokenType.String || data["type"]?.Value() != ResourceType) { error = "Expected a JSON:API Universal Flag Configuration resource"; diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs index 14fc51ab85a7..b57c9568ebcd 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs @@ -60,6 +60,10 @@ public void RejectsMalformedJson(string? body) [InlineData("""{ "meta": {} }""")] // data is not an object [InlineData("""{ "data": "string" }""")] + // data.type is not a string (object) + [InlineData("""{ "data": { "type": { "nested": true }, "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] + // data.type is not a string (array) + [InlineData("""{ "data": { "type": [1, 2], "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] public void RejectsInvalidEnvelope(string body) { UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); From 2f2682f4a9ef82311bc51048f061335d206d1a79 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Aug 2026 15:15:13 +0300 Subject: [PATCH 19/62] fix(feature-flags): add explicit request timeout race for net461 async WebRequest --- .../Agentless/AgentlessConfigurationSource.cs | 25 +++++++++++++++++++ .../AgentlessConfigurationSourceTests.cs | 1 + 2 files changed, 26 insertions(+) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 7ec342a4fc6d..4f2658fda111 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -44,6 +44,7 @@ internal sealed class AgentlessConfigurationSource : IDisposable private readonly IApiRequestFactory _requestFactory; private readonly Uri _endpoint; private readonly TimeSpan _pollInterval; + private readonly TimeSpan _requestTimeout; private readonly Func _applyConfiguration; private readonly Func _waitAsync; private readonly CancellationTokenSource _shutdown = new(); @@ -61,12 +62,14 @@ internal AgentlessConfigurationSource( Uri endpoint, IApiRequestFactory requestFactory, TimeSpan pollInterval, + TimeSpan requestTimeout, Func applyConfiguration, Func? waitAsync = null) { _endpoint = endpoint; _requestFactory = requestFactory; _pollInterval = pollInterval; + _requestTimeout = requestTimeout; _applyConfiguration = applyConfiguration; _waitAsync = waitAsync ?? Task.Delay; } @@ -94,6 +97,7 @@ internal AgentlessConfigurationSource( endpoint.Uri, CreateRequestFactory(endpoint, settings), settings.PollInterval, + settings.RequestTimeout, applyConfiguration); } @@ -275,7 +279,28 @@ private async Task RequestAsync() request.AddHeader("If-None-Match", etag); } +#if NETCOREAPP + // HttpClient.Timeout applies to async calls, so no explicit race is needed. using var response = await request.GetAsync().ConfigureAwait(false); +#else + // HttpWebRequest.Timeout does not apply to async calls (GetResponseAsync), so we race + // the request against an explicit delay to bound the wait on net461/netstandard2.0. + var getTask = request.GetAsync(); + var timeoutTask = Task.Delay(_requestTimeout); + + if (await Task.WhenAny(getTask, timeoutTask).ConfigureAwait(false) == timeoutTask) + { + // The request is still in flight. Dispose the response when it eventually completes + // (success or fault) so the underlying connection is released. + _ = getTask.ContinueWith( + t => { try { using var r = t.Result; } catch { } }, + TaskContinuationOptions.None); + + return new PollResult(statusCode: null, etag: null, body: null, error: new TimeoutException($"Feature Flags agentless request timed out after {_requestTimeout.TotalSeconds}s")); + } + + using var response = await getTask.ConfigureAwait(false); +#endif // Only a 200 carries configuration; other bodies are never decoded as one. var body = response.StatusCode == 200 ? await ReadBodyAsync(response).ConfigureAwait(false) : null; diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index 935c778b96f4..e2e39a15acde 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -216,6 +216,7 @@ private static AgentlessConfigurationSource CreateSource(TestRequestFactory fact Endpoint, factory, TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), configuration => { applied.Add(configuration); From 154ef588eb8840e5a832ffed0ea8ec6e95a9bfad Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Aug 2026 17:34:26 +0300 Subject: [PATCH 20/62] [FeatureFlags] Apply dd_env per request instead of baking it into the endpoint --- .../Configuration/TracerSettings.cs | 8 +- .../Agentless/AgentlessEndpoint.cs | 89 ++++++++++++++++--- .../FeatureFlags/FeatureFlagsSettings.cs | 14 +-- .../FeatureFlags/AgentlessEndpointTests.cs | 60 ++++++++++--- .../FeatureFlags/FeatureFlagsSettingsTests.cs | 2 +- 5 files changed, 132 insertions(+), 41 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index 1ce04feb0861..6051c80536d2 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -889,10 +889,10 @@ not null when string.Equals(value, "otlp", StringComparison.OrdinalIgnoreCase) = Manager = new(source, this, telemetry, errorLog); - // Created after the manager so the environment can be taken from the initial mutable - // settings, which also honour the "env" entry of DD_TAGS. It is captured once, so a - // later dynamic-configuration change to "env" does not move the agentless endpoint. - FeatureFlags = new FeatureFlagsSettings(source, telemetry, Manager.InitialMutableSettings.Environment); + // The environment is deliberately not passed in: it can be changed in code after + // startup, so the delivery source subscribes to the manager and applies the current + // value per request instead of capturing one here. + FeatureFlags = new FeatureFlagsSettings(source, telemetry); // OTLP span metrics require OTLP trace export (see TracerManagerFactory.GetAgentWriter). // Force to false otherwise, even if explicitly requested. diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index 2ebc7b03b3b3..a779d909a731 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -29,14 +29,25 @@ internal sealed class AgentlessEndpoint /// internal const string ManagedHostPrefix = "ufc-server.ff-cdn."; - private AgentlessEndpoint(Uri uri, bool isManaged) + /// + /// The query parameter carrying the environment to request configuration for. + /// + internal const string EnvParameterName = "dd_env"; + + // Set when the configured base URL already carries dd_env. The operator chose that value + // deliberately, so it is left alone rather than duplicated or overwritten. + private readonly bool _pinsEnv; + + private AgentlessEndpoint(Uri uri, bool isManaged, bool pinsEnv) { Uri = uri; IsManaged = isManaged; + _pinsEnv = pinsEnv; } /// - /// Gets the endpoint URI. + /// Gets the endpoint URI, without the environment. Use to get the + /// URI to request. /// public Uri Uri { get; } @@ -50,17 +61,19 @@ private AgentlessEndpoint(Uri uri, bool isManaged) /// /// Builds the endpoint. Without a custom the managed Datadog CDN /// endpoint is derived from the (lowercased) site, so staging and government sites resolve - /// with no allowlist, and dd_env is added only when an environment is configured. - /// A custom base URL that is an origin receives the canonical path; one that carries a path - /// is used verbatim. + /// with no allowlist. A custom base URL that is an origin receives the canonical path; one that + /// carries a path is used verbatim. + /// + /// The environment is not part of the endpoint: it can be changed in code while the application + /// runs, so it is applied per request by instead. + /// /// /// The Datadog site, for example datadoghq.com. - /// The configured environment, or null. /// The configured endpoint override, or null. /// The resulting endpoint, or null when none could be built. /// Why the configured base URL was rejected. Never contains the URL, which may carry credentials. /// true when an endpoint could be built. - public static bool TryCreate(string? site, string? env, string? baseUrl, [NotNullWhen(true)] out AgentlessEndpoint? endpoint, out string? error) + public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] out AgentlessEndpoint? endpoint, out string? error) { endpoint = null; error = null; @@ -92,12 +105,7 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, [NotNul return false; } - if (!StringUtil.IsNullOrEmpty(env)) - { - managedUri = new UriBuilder(managedUri) { Query = "dd_env=" + Uri.EscapeDataString(env) }.Uri; - } - - endpoint = new AgentlessEndpoint(managedUri, isManaged: true); + endpoint = new AgentlessEndpoint(managedUri, isManaged: true, pinsEnv: false); return true; } @@ -129,7 +137,60 @@ public static bool TryCreate(string? site, string? env, string? baseUrl, [NotNul custom = new UriBuilder(custom) { Path = DefaultPath }.Uri; } - endpoint = new AgentlessEndpoint(custom, isManaged: false); + endpoint = new AgentlessEndpoint(custom, isManaged: false, pinsEnv: HasEnvParameter(custom.Query)); return true; } + + /// + /// Returns the URI to request configuration for . The environment is + /// added as a query parameter rather than baked into the endpoint, because it can change while + /// the application runs. + /// + /// Any query the configured base URL already carries is kept: it may hold credentials or routing + /// the operator needs. An endpoint that already pins dd_env is returned unchanged. + /// + /// + /// The current environment, or null when none is configured. + /// The URI to request. + public Uri BuildRequestUri(string? env) + { + if (_pinsEnv || StringUtil.IsNullOrEmpty(env)) + { + return Uri; + } + + var parameter = EnvParameterName + "=" + Uri.EscapeDataString(env); + var builder = new UriBuilder(Uri); + + // The getter returns the query with its leading "?", and the setter keeps one that is + // already there, so an existing query can be extended without trimming it first. A URL + // ending in a bare "?" reports an empty query, which the length check treats as no query. + builder.Query = builder.Query.Length > 1 ? builder.Query + "&" + parameter : parameter; + return builder.Uri; + } + + /// + /// Reports whether a query string already carries a dd_env parameter. Matching the name + /// alone would also hit a value such as ?next=dd_env, so the surrounding delimiters are + /// checked too. + /// + private static bool HasEnvParameter(string query) + { + var index = query.IndexOf(EnvParameterName, StringComparison.OrdinalIgnoreCase); + while (index >= 0) + { + var preceding = index == 0 ? '?' : query[index - 1]; + var followingIndex = index + EnvParameterName.Length; + var following = followingIndex < query.Length ? query[followingIndex] : '\0'; + + if (preceding is '?' or '&' && following is '=' or '&' or '\0') + { + return true; + } + + index = query.IndexOf(EnvParameterName, index + 1, StringComparison.OrdinalIgnoreCase); + } + + return false; + } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index f9c804f30364..3ebd6419e7e6 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -36,7 +36,7 @@ internal sealed class FeatureFlagsSettings private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(FeatureFlagsSettings)); - public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry, string? env) + public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetry telemetry) { source ??= NullConfigurationSource.Instance; var config = new ConfigurationBuilder(source, telemetry); @@ -96,13 +96,12 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr .Value); // DD_SITE and DD_API_KEY are not extracted on TracerSettings, so they are read here as the - // other product settings do (TelemetrySettings, DirectLogSubmissionSettings). DD_ENV is - // passed in, because TracerSettings also honours the "env" entry of DD_TAGS and re-reading - // the key alone would disagree with the rest of the tracer. + // other product settings do (TelemetrySettings, DirectLogSubmissionSettings). DD_ENV is not + // read at all: it can be changed in code while the application runs, so the delivery source + // takes the current value per request rather than capturing one here. Site = config .WithKeys(ConfigurationKeys.Site) .AsString(DefaultSite, static site => !StringUtil.IsNullOrWhiteSpace(site)); - Env = env; ApiKey = config.WithKeys(ConfigurationKeys.ApiKey).AsRedactedString(); var initializationTimeoutMs = config @@ -132,11 +131,6 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr /// public string Site { get; } - /// - /// Gets the configured environment, sent to the agentless endpoint as dd_env. - /// - public string? Env { get; } - /// /// Gets the API key, required by the managed agentless endpoint. /// diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs index 2fbda55d3836..0edc9f96bdcb 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -30,16 +30,52 @@ public void DerivesManagedEndpointFromSite(string site, string expected) } [Fact] - public void AddsDdEnvWhenEnvIsConfigured() - => Create("datadoghq.com", env: "production").Uri.Query.Should().Be("?dd_env=production"); + public void EndpointItselfCarriesNoEnvironment() + => Create("datadoghq.com").Uri.Query.Should().BeEmpty(); [Fact] - public void DoesNotAddDdEnvWhenEnvIsNull() - => Create("datadoghq.com").Uri.Query.Should().BeEmpty(); + public void AddsDdEnvWhenEnvIsConfigured() + => Create("datadoghq.com").BuildRequestUri("production").Query.Should().Be("?dd_env=production"); + + [Theory] + [InlineData(null)] + [InlineData("")] + public void DoesNotAddDdEnvWhenEnvIsNotConfigured(string? env) + => Create("datadoghq.com").BuildRequestUri(env).Query.Should().BeEmpty(); [Fact] public void EscapesDdEnvValue() - => Create("datadoghq.com", env: "my env&test").Uri.Query.Should().Be("?dd_env=my%20env%26test"); + => Create("datadoghq.com").BuildRequestUri("my env&test").Query.Should().Be("?dd_env=my%20env%26test"); + + [Fact] + public void KeepsTheQueryConfiguredOnACustomEndpoint() + { + // The query may carry credentials or routing the operator needs, so dd_env is appended to it + // rather than replacing it. + var endpoint = Create("datadoghq.com", baseUrl: "https://flags.example.com/ufc?token=abc"); + + endpoint.BuildRequestUri("production").Query.Should().Be("?token=abc&dd_env=production"); + } + + [Theory] + [InlineData("https://flags.example.com/ufc?dd_env=staging")] + [InlineData("https://flags.example.com/ufc?token=abc&dd_env=staging")] + public void LeavesACustomEndpointThatAlreadyPinsDdEnvAlone(string baseUrl) + { + var endpoint = Create("datadoghq.com", baseUrl: baseUrl); + + endpoint.BuildRequestUri("production").Should().Be(endpoint.Uri); + } + + [Fact] + public void AddsDdEnvToACustomEndpointWithoutAQuery() + => Create("datadoghq.com", baseUrl: "https://flags.example.com/ufc") + .BuildRequestUri("production").Query.Should().Be("?dd_env=production"); + + [Fact] + public void DoesNotMistakeAQueryValueForAPinnedDdEnv() + => Create("datadoghq.com", baseUrl: "https://flags.example.com/ufc?next=dd_env") + .BuildRequestUri("production").Query.Should().Be("?next=dd_env&dd_env=production"); [Theory] [InlineData("https://flags.example.com", "https://flags.example.com" + DefaultPath)] @@ -65,7 +101,7 @@ public void CustomEndpointAcceptsHttp(string baseUrl) [InlineData("https://flags.example.com bad", "The configured Feature Flags agentless URL is not a valid URL")] // internal whitespace public void RejectsInvalidBaseUrl(string baseUrl, string expectedError) { - AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: baseUrl, out var endpoint, out var error) + AgentlessEndpoint.TryCreate("datadoghq.com", baseUrl: baseUrl, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be(expectedError); endpoint.Should().BeNull(); @@ -74,7 +110,7 @@ public void RejectsInvalidBaseUrl(string baseUrl, string expectedError) [Fact] public void RejectsEmptySiteWithoutBaseUrl() { - AgentlessEndpoint.TryCreate(site: null, env: null, baseUrl: null, out var endpoint, out var error) + AgentlessEndpoint.TryCreate(site: null, baseUrl: null, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be("No Datadog site is configured"); endpoint.Should().BeNull(); @@ -83,7 +119,7 @@ public void RejectsEmptySiteWithoutBaseUrl() [Fact] public void RejectsWhitespaceOnlySiteWithoutBaseUrl() { - AgentlessEndpoint.TryCreate(" ", env: null, baseUrl: null, out var endpoint, out var error) + AgentlessEndpoint.TryCreate(" ", baseUrl: null, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be("No Datadog site is configured"); endpoint.Should().BeNull(); @@ -95,7 +131,7 @@ public void RejectsWhitespaceOnlySiteWithoutBaseUrl() [InlineData("datadoghq.com:99999")] // invalid port public void RejectsMalformedSiteWithoutThrowing(string site) { - AgentlessEndpoint.TryCreate(site, env: null, baseUrl: null, out var endpoint, out var error) + AgentlessEndpoint.TryCreate(site, baseUrl: null, out var endpoint, out var error) .Should().BeFalse(); error.Should().Be("The configured Datadog site is not valid"); endpoint.Should().BeNull(); @@ -105,7 +141,7 @@ public void RejectsMalformedSiteWithoutThrowing(string site) public void ErrorNeverContainsUrl() { // A URL may carry credentials, so the error must never echo it. - AgentlessEndpoint.TryCreate("datadoghq.com", env: null, baseUrl: "https://user:pass@flags.example.com bad", out _, out var error) + AgentlessEndpoint.TryCreate("datadoghq.com", baseUrl: "https://user:pass@flags.example.com bad", out _, out var error) .Should().BeFalse(); error.Should().NotContain("user"); error.Should().NotContain("pass"); @@ -115,9 +151,9 @@ public void ErrorNeverContainsUrl() // assertions can read it directly. Throwing rather than asserting keeps the compiler's nullable // analysis satisfied without a null-forgiving operator, which would let an assertion be // silently skipped if the endpoint were ever null. - private static AgentlessEndpoint Create(string? site, string? env = null, string? baseUrl = null) + private static AgentlessEndpoint Create(string? site, string? baseUrl = null) { - AgentlessEndpoint.TryCreate(site, env, baseUrl, out var endpoint, out var error) + AgentlessEndpoint.TryCreate(site, baseUrl, out var endpoint, out var error) .Should().BeTrue(); error.Should().BeNull(); diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index efd24e3d9419..a55e21c5e765 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -201,6 +201,6 @@ private static FeatureFlagsSettings CreateSettings( collection[key] = value; } - return new FeatureFlagsSettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance, env: null); + return new FeatureFlagsSettings(new NameValueConfigurationSource(collection), NullConfigurationTelemetry.Instance); } } From dbae9dc71ce81c5abd011dfa15b23a70c704f84f Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Aug 2026 17:41:41 +0300 Subject: [PATCH 21/62] [FeatureFlags] Normalize dd_env the same way span tags are normalized --- .../Agentless/AgentlessEndpoint.cs | 15 +++++++++++-- .../FeatureFlags/AgentlessEndpointTests.cs | 22 ++++++++++++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index a779d909a731..5fa9e07a7892 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -7,6 +7,7 @@ using System; using System.Diagnostics.CodeAnalysis; +using Datadog.Trace.Processors; using Datadog.Trace.Util; namespace Datadog.Trace.FeatureFlags.Agentless; @@ -154,12 +155,22 @@ public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] /// The URI to request. public Uri BuildRequestUri(string? env) { - if (_pinsEnv || StringUtil.IsNullOrEmpty(env)) + if (_pinsEnv) { return Uri; } - var parameter = EnvParameterName + "=" + Uri.EscapeDataString(env); + // Normalized the same way the tracer normalizes it before tagging spans, so that flag + // targeting and span tags agree on what the environment is. It also bounds the value at 200 + // characters, which keeps a misconfigured environment from producing an unusable URL. + // A value that normalizes to nothing is treated as no environment at all. + var normalized = TraceUtil.NormalizeTag(env); + if (StringUtil.IsNullOrEmpty(normalized)) + { + return Uri; + } + + var parameter = EnvParameterName + "=" + Uri.EscapeDataString(normalized); var builder = new UriBuilder(Uri); // The getter returns the query with its leading "?", and the setter keeps one that is diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs index 0edc9f96bdcb..2f53914f4b7c 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -40,12 +40,32 @@ public void AddsDdEnvWhenEnvIsConfigured() [Theory] [InlineData(null)] [InlineData("")] + [InlineData(" ")] + // The tracer's tag normalization requires a leading letter, so a value that is entirely digits + // normalizes away. Spans are tagged the same way, so no environment is reported either. + [InlineData("2024")] public void DoesNotAddDdEnvWhenEnvIsNotConfigured(string? env) => Create("datadoghq.com").BuildRequestUri(env).Query.Should().BeEmpty(); + [Theory] + // Normalized exactly as the tracer normalizes the environment before tagging spans: lowercased, + // with runs of unsupported characters collapsed to a single underscore. + [InlineData("Production", "production")] + [InlineData("Prod EU", "prod_eu")] + [InlineData(" staging ", "staging")] + [InlineData("my env&test", "my_env_test")] + public void NormalizesDdEnvValue(string env, string expected) + => Create("datadoghq.com").BuildRequestUri(env).Query.Should().Be("?dd_env=" + expected); + [Fact] public void EscapesDdEnvValue() - => Create("datadoghq.com").BuildRequestUri("my env&test").Query.Should().Be("?dd_env=my%20env%26test"); + // "/" survives normalization but cannot be carried unescaped in a query value. + => Create("datadoghq.com").BuildRequestUri("team/a").Query.Should().Be("?dd_env=team%2Fa"); + + [Fact] + public void TruncatesAnOverlongDdEnvValue() + => Create("datadoghq.com").BuildRequestUri(new string('a', 500)) + .Query.Should().Be("?dd_env=" + new string('a', 200)); [Fact] public void KeepsTheQueryConfiguredOnACustomEndpoint() From 7c7d7e6f6214d17fa9c7e9cdea20fe222299aeb6 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Aug 2026 17:48:16 +0300 Subject: [PATCH 22/62] [FeatureFlags] Report an unrecognised configuration source as a parsing failure --- .../Configuration/supported-configurations.yaml | 7 ++++--- .../Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs | 8 ++++---- .../ConfigurationKeys.FeatureFlags.g.cs | 7 ++++--- .../ConfigurationKeys.FeatureFlags.g.cs | 7 ++++--- .../ConfigurationKeys.FeatureFlags.g.cs | 7 ++++--- .../ConfigurationKeys.FeatureFlags.g.cs | 7 ++++--- .../FeatureFlags/FeatureFlagsSettingsTests.cs | 7 ++++--- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml index 33a8c0f6a63b..3523c5a3c676 100644 --- a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml +++ b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml @@ -3192,9 +3192,10 @@ supportedConfigurations: const_name: FeatureFlagsConfigurationSource documentation: |- Configuration key for selecting where flag configuration is loaded from. - Supported values are agentless (direct HTTP delivery, the default) and - remote_config (delivery through the Datadog Agent's Remote Configuration). - Any other value disables Feature Flags, which fails closed and contacts nothing. + Supported values are agentless (direct HTTP delivery, the default), + remote_config (delivery through the Datadog Agent's Remote Configuration) + and offline, which disables Feature Flags and contacts nothing. + Any other value is rejected and the default applies. DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL: - implementation: A scope: managed diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 3ebd6419e7e6..426bd6780484 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -153,9 +153,9 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr /// /// Converts a configured source name to a . A blank value is - /// treated as unset so the default applies, and an unrecognised one fails closed: it resolves - /// to rather than falling back to a billed delivery - /// path, which is why it is reported as a successful conversion. + /// treated as unset, and an unrecognised one is reported as a parsing failure so that it shows + /// up as rejected in configuration telemetry rather than as a value nobody configured. Both + /// fall back to the default, which is what an unset key would have selected anyway. /// private static ParsingResult ConvertSource(string? value) { @@ -171,7 +171,7 @@ private static ParsingResult ConvertSource(string? value) return ParsingResult.Success(source); } - return ParsingResult.Success(FeatureFlagsSource.Disabled); + return ParsingResult.Failure(); } private static bool TryMatch(string value, out FeatureFlagsSource source) diff --git a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index c0bc6ef9a199..6d3efb137e4e 100644 --- a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -40,9 +40,10 @@ internal static class FeatureFlags /// /// Configuration key for selecting where flag configuration is loaded from. - /// Supported values are agentless (direct HTTP delivery, the default) and - /// remote_config (delivery through the Datadog Agent's Remote Configuration). - /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// Supported values are agentless (direct HTTP delivery, the default), + /// remote_config (delivery through the Datadog Agent's Remote Configuration) + /// and offline, which disables Feature Flags and contacts nothing. + /// Any other value is rejected and the default applies. /// public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; diff --git a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index c0bc6ef9a199..6d3efb137e4e 100644 --- a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -40,9 +40,10 @@ internal static class FeatureFlags /// /// Configuration key for selecting where flag configuration is loaded from. - /// Supported values are agentless (direct HTTP delivery, the default) and - /// remote_config (delivery through the Datadog Agent's Remote Configuration). - /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// Supported values are agentless (direct HTTP delivery, the default), + /// remote_config (delivery through the Datadog Agent's Remote Configuration) + /// and offline, which disables Feature Flags and contacts nothing. + /// Any other value is rejected and the default applies. /// public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; diff --git a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index c0bc6ef9a199..6d3efb137e4e 100644 --- a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -40,9 +40,10 @@ internal static class FeatureFlags /// /// Configuration key for selecting where flag configuration is loaded from. - /// Supported values are agentless (direct HTTP delivery, the default) and - /// remote_config (delivery through the Datadog Agent's Remote Configuration). - /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// Supported values are agentless (direct HTTP delivery, the default), + /// remote_config (delivery through the Datadog Agent's Remote Configuration) + /// and offline, which disables Feature Flags and contacts nothing. + /// Any other value is rejected and the default applies. /// public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; diff --git a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index c0bc6ef9a199..6d3efb137e4e 100644 --- a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -40,9 +40,10 @@ internal static class FeatureFlags /// /// Configuration key for selecting where flag configuration is loaded from. - /// Supported values are agentless (direct HTTP delivery, the default) and - /// remote_config (delivery through the Datadog Agent's Remote Configuration). - /// Any other value disables Feature Flags, which fails closed and contacts nothing. + /// Supported values are agentless (direct HTTP delivery, the default), + /// remote_config (delivery through the Datadog Agent's Remote Configuration) + /// and offline, which disables Feature Flags and contacts nothing. + /// Any other value is rejected and the default applies. /// public const string FeatureFlagsConfigurationSource = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"; diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index a55e21c5e765..3dafd5457de1 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -39,9 +39,10 @@ public class FeatureFlagsSettingsTests // does not silently keep Feature Flags off during migration. [InlineData("true", null, "false", FeatureFlagsSource.Agentless)] [InlineData("true", null, "true", FeatureFlagsSource.Agentless)] - // An unrecognised source fails closed rather than guessing a billed delivery path. - [InlineData(null, "invalid", null, FeatureFlagsSource.Disabled)] - [InlineData(null, "invalid", "true", FeatureFlagsSource.Disabled)] + // An unrecognised source is a parsing failure, reported as such in configuration telemetry, so + // the key behaves as if it were unset rather than resolving to a value nobody configured. + [InlineData(null, "invalid", null, FeatureFlagsSource.Agentless)] + [InlineData(null, "invalid", "true", FeatureFlagsSource.RemoteConfig)] // "offline" is a reserved, recognised fail-closed sentinel (not an unrecognised value). [InlineData(null, "offline", null, FeatureFlagsSource.Disabled)] [InlineData(null, "offline", "true", FeatureFlagsSource.Disabled)] From 89357807915bc73adeb855092cac899780954d12 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Aug 2026 18:18:37 +0300 Subject: [PATCH 23/62] [FeatureFlags] Poll for the current environment instead of the one captured at startup --- .../Agentless/AgentlessConfigurationSource.cs | 66 ++++++++++++++++--- .../AgentlessConfigurationSourceTests.cs | 47 ++++++++++++- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 4f2658fda111..a0bdd1279537 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -13,6 +13,7 @@ using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.Agent.Transports; +using Datadog.Trace.Configuration; using Datadog.Trace.FeatureFlags.Rcm.Model; using Datadog.Trace.Headers; using Datadog.Trace.Logging; @@ -42,7 +43,7 @@ internal sealed class AgentlessConfigurationSource : IDisposable private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AgentlessConfigurationSource)); private readonly IApiRequestFactory _requestFactory; - private readonly Uri _endpoint; + private readonly AgentlessEndpoint _endpoint; private readonly TimeSpan _pollInterval; private readonly TimeSpan _requestTimeout; private readonly Func _applyConfiguration; @@ -52,18 +53,26 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); + + // Written by the settings-change callback, read by the poll loop. + private string? _environment; + private IDisposable? _environmentSubscription; + + // Only ever touched from the poll loop. private bool _malformedPayloadLogged; private bool _applyFailureLogged; private string? _etag; + private Uri? _etagUri; private int _started; internal AgentlessConfigurationSource( - Uri endpoint, + AgentlessEndpoint endpoint, IApiRequestFactory requestFactory, TimeSpan pollInterval, TimeSpan requestTimeout, Func applyConfiguration, + string? environment = null, Func? waitAsync = null) { _endpoint = endpoint; @@ -71,6 +80,7 @@ internal AgentlessConfigurationSource( _pollInterval = pollInterval; _requestTimeout = requestTimeout; _applyConfiguration = applyConfiguration; + _environment = environment; _waitAsync = waitAsync ?? Task.Delay; } @@ -79,9 +89,12 @@ internal AgentlessConfigurationSource( /// not a URL, or the managed endpoint without an API key. Polling anyway would only produce /// failures every interval. /// - public static AgentlessConfigurationSource? Create(FeatureFlagsSettings settings, Func applyConfiguration) + public static AgentlessConfigurationSource? Create( + FeatureFlagsSettings settings, + TracerSettings.SettingsManager manager, + Func applyConfiguration) { - if (!AgentlessEndpoint.TryCreate(settings.Site, settings.Env, settings.AgentlessBaseUrl, out var endpoint, out var error)) + if (!AgentlessEndpoint.TryCreate(settings.Site, settings.AgentlessBaseUrl, out var endpoint, out var error)) { Log.Error("Feature Flags agentless source is unavailable: {Error}", error); return null; @@ -93,14 +106,34 @@ internal AgentlessConfigurationSource( return null; } - return new AgentlessConfigurationSource( - endpoint.Uri, + var source = new AgentlessConfigurationSource( + endpoint, CreateRequestFactory(endpoint, settings), settings.PollInterval, settings.RequestTimeout, - applyConfiguration); + applyConfiguration, + manager.InitialMutableSettings.Environment); + + // The environment is tracked rather than captured: customers can change it in code while + // the application runs, and flags are targeted per environment. Only the value is stored + // here, so that the poll loop stays the only thing that touches the request state. + source._environmentSubscription = manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + source.UpdateEnvironment(mutable.Environment); + } + }); + + return source; } + /// + /// Records the environment to request configuration for. Applied by the poll loop on its next + /// request, so a change never disturbs a request already in flight. + /// + internal void UpdateEnvironment(string? environment) => Volatile.Write(ref _environment, environment); + /// /// Starts polling. Idempotent. /// @@ -172,6 +205,7 @@ public void Dispose() // from applying its result by the shutdown check in PollAsync. try { + _environmentSubscription?.Dispose(); _shutdown.Cancel(); } catch (Exception ex) @@ -207,6 +241,8 @@ private static ApiWebRequestFactory CreateRequestFactory(AgentlessEndpoint endpo headers.Add(new(TelemetryConstants.ApiKeyHeader, settings.ApiKey!)); } + // The endpoint is only the factory's default: both transports honour the URI passed to + // Create, which is what carries the current environment. #if NETCOREAPP return new HttpClientRequestFactory(endpoint.Uri, headers.ToArray(), timeout: settings.RequestTimeout); #else @@ -273,7 +309,21 @@ private async Task RequestAsync() { try { - var request = _requestFactory.Create(_endpoint); + // The environment is applied per request rather than baked into the endpoint, because + // it can be changed in code after startup. + var uri = _endpoint.BuildRequestUri(Volatile.Read(ref _environment)); + + // An ETag only identifies the configuration served for the URI it came from. Sending it + // against a different environment would earn a 304 and pin the process to the previous + // environment's flags, with no way back. + if (_etagUri is not null && uri != _etagUri) + { + _etag = null; + } + + _etagUri = uri; + + var request = _requestFactory.Create(uri); if (_etag is { } etag) { request.AddHeader("If-None-Match", etag); diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index e2e39a15acde..a807554c0cc7 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -30,7 +30,7 @@ public class AgentlessConfigurationSourceTests "environment": { "name": "production" }, "flags": {} } } } """; - private static readonly Uri Endpoint = new("https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server"); + private const string EndpointUrl = "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server"; [Fact] public async Task AppliesConfigurationFromA200() @@ -64,6 +64,37 @@ public async Task SendsTheEtagOfTheLastAppliedConfiguration() factory.RequestsSent[1].ExtraHeaders["If-None-Match"].Should().Be("\"ufc-v1\""); } + [Fact] + public async Task RequestsTheConfiguredEnvironment() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new TestApiRequest(uri, responseContent: Body)); + using var source = CreateSource(factory, applied, environment: "production"); + + await source.PollAsync(); + + factory.RequestsSent[0].Endpoint.Should().Be(new Uri(EndpointUrl + "?dd_env=production")); + } + + [Fact] + public async Task DropsTheEtagWhenTheEnvironmentChanges() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, responseContent: Body, responseHeaders: new() { { "ETag", "\"ufc-v1\"" } })); + using var source = CreateSource(factory, applied, environment: "production"); + + await source.PollAsync(); + + // The ETag identifies production's configuration, so it must not be sent against staging: + // a 304 would pin the process to production's flags with no way back. + source.UpdateEnvironment("staging"); + await source.PollAsync(); + + factory.RequestsSent[1].Endpoint.Should().Be(new Uri(EndpointUrl + "?dd_env=staging")); + factory.RequestsSent[1].ExtraHeaders.Should().NotContainKey("If-None-Match"); + } + [Fact] public async Task DoesNotApplyOn304() { @@ -211,9 +242,12 @@ public async Task HandlesNetworkError() factory.RequestsSent.Should().HaveCount(3); } - private static AgentlessConfigurationSource CreateSource(TestRequestFactory factory, List applied) + private static AgentlessConfigurationSource CreateSource( + TestRequestFactory factory, + List applied, + string? environment = null) => new( - Endpoint, + CreateEndpoint(), factory, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(5), @@ -222,8 +256,15 @@ private static AgentlessConfigurationSource CreateSource(TestRequestFactory fact applied.Add(configuration); return true; }, + environment, NoWait); + private static AgentlessEndpoint CreateEndpoint() + { + AgentlessEndpoint.TryCreate("datadoghq.com", baseUrl: null, out var endpoint, out _).Should().BeTrue(); + return endpoint ?? throw new InvalidOperationException("TryCreate reported success without producing an endpoint."); + } + private static Task NoWait(TimeSpan delay, CancellationToken cancellationToken) => Task.CompletedTask; private class ThrowingApiRequest(Uri endpoint) : TestApiRequest(endpoint) From 2317ea8fff30107dbe1fda02dca5c36828c99ee2 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 25 Aug 2026 15:39:15 +0300 Subject: [PATCH 24/62] [FeatureFlags] Fix double question mark in agentless request URI on .NET Framework --- .../FeatureFlags/Agentless/AgentlessEndpoint.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index 5fa9e07a7892..11e258035245 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -173,10 +173,11 @@ public Uri BuildRequestUri(string? env) var parameter = EnvParameterName + "=" + Uri.EscapeDataString(normalized); var builder = new UriBuilder(Uri); - // The getter returns the query with its leading "?", and the setter keeps one that is - // already there, so an existing query can be extended without trimming it first. A URL - // ending in a bare "?" reports an empty query, which the length check treats as no query. - builder.Query = builder.Query.Length > 1 ? builder.Query + "&" + parameter : parameter; + // The getter returns the query with its leading "?", while the setter prepends one of its + // own on .NET Framework, so the existing query is trimmed before it is extended. A URL + // ending in a bare "?" reports a query of "?", which the length check treats as no query. + var existing = builder.Query; + builder.Query = existing.Length > 1 ? existing.TrimStart('?') + "&" + parameter : parameter; return builder.Uri; } From d868f5ace16f2899cb9764e689dc7a6e33dfa6a2 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 15:53:54 +0300 Subject: [PATCH 25/62] [FeatureFlags] Align the provider initialization timeout default with the other tracers --- .../Configuration/supported-configurations.yaml | 6 +++--- .../src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs | 4 +++- .../ConfigurationKeys.FeatureFlags.g.cs | 2 +- .../FeatureFlags/FeatureFlagsSettingsTests.cs | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml index 3523c5a3c676..7347d243d4d2 100644 --- a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml +++ b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml @@ -3158,16 +3158,16 @@ supportedConfigurations: Enables Feature Flags Provider (Experimental). Default value is false (disabled). DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS: - - implementation: B + - implementation: A scope: managed type: int - default: '10000' + default: '30000' product: FeatureFlags const_name: FlaggingProviderInitializationTimeoutMs documentation: |- Configuration key for how long, in milliseconds, provider initialization waits for the first flag configuration to arrive before returning. - Default value is 10000 (10 seconds). + Default value is 30000 (30 seconds), matching the other tracers. Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations return the caller's default value, and the provider becomes ready when configuration arrives. DD_FEATURE_FLAGS_ENABLED: diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 426bd6780484..654148504d05 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -28,7 +28,9 @@ internal sealed class FeatureFlagsSettings internal const int DefaultPollIntervalSeconds = 30; internal const int DefaultRequestTimeoutSeconds = 5; - internal const int DefaultInitializationTimeoutMs = 10_000; + // Matches the Go, Java and Node tracers, so the same slow first configuration does not give + // one language the caller's default value while the others still return a real one. + internal const int DefaultInitializationTimeoutMs = 30_000; // An interval above this is indistinguishable from "never poll" and is more likely a // misconfiguration (for example milliseconds passed as seconds) than an intent. diff --git a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index 6d3efb137e4e..fc6c32d2b92b 100644 --- a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -26,7 +26,7 @@ internal static class FeatureFlags /// /// Configuration key for how long, in milliseconds, provider initialization waits for the first /// flag configuration to arrive before returning. - /// Default value is 10000 (10 seconds). + /// Default value is 30000 (30 seconds), matching the other tracers. /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations /// return the caller's default value, and the provider becomes ready when configuration arrives. /// diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index 3dafd5457de1..e9548b61c6a3 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -81,7 +81,7 @@ public void UsesDocumentedDefaults() settings.PollInterval.Should().Be(TimeSpan.FromSeconds(30)); settings.RequestTimeout.Should().Be(TimeSpan.FromSeconds(5)); - settings.InitializationTimeout.Should().Be(TimeSpan.FromMilliseconds(10_000)); + settings.InitializationTimeout.Should().Be(TimeSpan.FromMilliseconds(30_000)); settings.AgentlessBaseUrl.Should().BeNull(); } @@ -122,8 +122,8 @@ public void ReadsRequestTimeout(string configured, int expectedSeconds) [Theory] [InlineData("1000", 1000)] - [InlineData("0", 10_000)] - [InlineData("-1", 10_000)] + [InlineData("0", 30_000)] + [InlineData("-1", 30_000)] public void ReadsInitializationTimeout(string configured, int expectedMs) { var settings = CreateSettings( From b2f012a9f7a3bdcc5c7a18824ba000cb89f66da1 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 15:56:28 +0300 Subject: [PATCH 26/62] Fix comments --- .../ConfigurationKeys.FeatureFlags.g.cs | 2 +- .../ConfigurationKeys.FeatureFlags.g.cs | 2 +- .../ConfigurationKeys.FeatureFlags.g.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index 6d3efb137e4e..fc6c32d2b92b 100644 --- a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -26,7 +26,7 @@ internal static class FeatureFlags /// /// Configuration key for how long, in milliseconds, provider initialization waits for the first /// flag configuration to arrive before returning. - /// Default value is 10000 (10 seconds). + /// Default value is 30000 (30 seconds), matching the other tracers. /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations /// return the caller's default value, and the provider becomes ready when configuration arrives. /// diff --git a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index 6d3efb137e4e..fc6c32d2b92b 100644 --- a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -26,7 +26,7 @@ internal static class FeatureFlags /// /// Configuration key for how long, in milliseconds, provider initialization waits for the first /// flag configuration to arrive before returning. - /// Default value is 10000 (10 seconds). + /// Default value is 30000 (30 seconds), matching the other tracers. /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations /// return the caller's default value, and the provider becomes ready when configuration arrives. /// diff --git a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index 6d3efb137e4e..fc6c32d2b92b 100644 --- a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -26,7 +26,7 @@ internal static class FeatureFlags /// /// Configuration key for how long, in milliseconds, provider initialization waits for the first /// flag configuration to arrive before returning. - /// Default value is 10000 (10 seconds). + /// Default value is 30000 (30 seconds), matching the other tracers. /// Initialization does not fail when the timeout expires: the provider stays not-ready, evaluations /// return the caller's default value, and the provider becomes ready when configuration arrives. /// From 5654f49e6fdab9206411dc0c68182f19f01ccf77 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:30:42 +0300 Subject: [PATCH 27/62] [FeatureFlags] Signal agentless poller shutdown with a TaskCompletionSource --- .../Agentless/AgentlessConfigurationSource.cs | 35 +++++++++---------- .../AgentlessConfigurationSourceTests.cs | 3 +- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index a0bdd1279537..7058d0d01ed4 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -47,8 +47,11 @@ internal sealed class AgentlessConfigurationSource : IDisposable private readonly TimeSpan _pollInterval; private readonly TimeSpan _requestTimeout; private readonly Func _applyConfiguration; - private readonly Func _waitAsync; - private readonly CancellationTokenSource _shutdown = new(); + private readonly Func _waitAsync; + + // Not a CancellationTokenSource: cancellation throws, and an exception on the shutdown path can + // crash the runtime, so shutdown is signalled by completing a task instead. + private readonly TaskCompletionSource _shutdown = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly Random _random = new(); // Only ever touched from the poll loop. @@ -73,7 +76,7 @@ internal AgentlessConfigurationSource( TimeSpan requestTimeout, Func applyConfiguration, string? environment = null, - Func? waitAsync = null) + Func? waitAsync = null) { _endpoint = endpoint; _requestFactory = requestFactory; @@ -161,7 +164,7 @@ internal async Task PollAsync() { result = await RequestAsync().ConfigureAwait(false); - if (_shutdown.IsCancellationRequested) + if (_shutdown.Task.IsCompleted) { // A shutdown mid-poll leaves the response unusable for state transitions: keep // last-known-good and the current ETag. @@ -182,13 +185,13 @@ internal async Task PollAsync() await WaitAsync(RetryDelay(attempt)).ConfigureAwait(false); - if (_shutdown.IsCancellationRequested) + if (_shutdown.Task.IsCompleted) { return; } } - if (_shutdown.IsCancellationRequested) + if (_shutdown.Task.IsCompleted) { // A shutdown during the final attempt leaves the response unusable for state // transitions: keep last-known-good and the current ETag. @@ -203,14 +206,15 @@ public void Dispose() // The request in flight is bounded by the request timeout, and the loop is never joined, // so a shutdown does not wait for it. A poll that completes after disposal is prevented // from applying its result by the shutdown check in PollAsync. + _shutdown.TrySetResult(true); + try { _environmentSubscription?.Dispose(); - _shutdown.Cancel(); } catch (Exception ex) { - Log.Debug(ex, "Error cancelling the Feature Flags agentless poll loop"); + Log.Debug(ex, "Error unsubscribing the Feature Flags agentless poll loop from settings changes"); } } @@ -257,7 +261,7 @@ private async Task RunAsync() { Log.Debug("AgentlessConfigurationSource::RunAsync -> Enter"); - while (!_shutdown.IsCancellationRequested) + while (!_shutdown.Task.IsCompleted) { try { @@ -275,17 +279,10 @@ private async Task RunAsync() Log.Debug("AgentlessConfigurationSource::RunAsync -> Exit"); } + // A shutdown ends the wait early. The delay itself is left to expire on its own: it holds no + // thread, and the loop has already exited by the time it does. private async Task WaitAsync(TimeSpan delay) - { - try - { - await _waitAsync(delay, _shutdown.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - // Shutting down - } - } + => await Task.WhenAny(_waitAsync(delay), _shutdown.Task).ConfigureAwait(false); private TimeSpan RetryDelay(int attempt) { diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index a807554c0cc7..276f28b37251 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -11,7 +11,6 @@ using System.IO.Compression; using System.Linq; using System.Text; -using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.FeatureFlags.Agentless; @@ -265,7 +264,7 @@ private static AgentlessEndpoint CreateEndpoint() return endpoint ?? throw new InvalidOperationException("TryCreate reported success without producing an endpoint."); } - private static Task NoWait(TimeSpan delay, CancellationToken cancellationToken) => Task.CompletedTask; + private static Task NoWait(TimeSpan delay) => Task.CompletedTask; private class ThrowingApiRequest(Uri endpoint) : TestApiRequest(endpoint) { From 0eadda95b7c2c683e81871690ebae03f55479048 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:39:32 +0300 Subject: [PATCH 28/62] [FeatureFlags] Use ThreadSafeRandom.Shared for agentless retry jitter --- .../FeatureFlags/Agentless/AgentlessConfigurationSource.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 7058d0d01ed4..b2e5f47bd662 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -52,7 +52,6 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Not a CancellationTokenSource: cancellation throws, and an exception on the shutdown path can // crash the runtime, so shutdown is signalled by completing a task instead. private readonly TaskCompletionSource _shutdown = new(TaskCreationOptions.RunContinuationsAsynchronously); - private readonly Random _random = new(); // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); @@ -290,11 +289,7 @@ private TimeSpan RetryDelay(int attempt) ? Clamp(_pollInterval.TotalSeconds / 6, FirstRetryMin, FirstRetryMax) : Clamp(_pollInterval.TotalSeconds / 3, SecondRetryMin, SecondRetryMax); - double jitter; - lock (_random) - { - jitter = 1 - RetryJitter + (_random.NextDouble() * RetryJitter * 2); - } + var jitter = 1 - RetryJitter + (ThreadSafeRandom.Shared.NextDouble() * RetryJitter * 2); return TimeSpan.FromSeconds(Math.Max(MinRetryDelay.TotalSeconds, seconds * jitter)); From e2a6f9a6a9fd69f2e1717e142039d5972c219910 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:42:08 +0300 Subject: [PATCH 29/62] [FeatureFlags] Log agentless delivery failures as warnings --- .../FeatureFlags/Agentless/AgentlessConfigurationSource.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index b2e5f47bd662..bcd0d5dafddf 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -442,13 +442,13 @@ private void WarnFailure(in PollResult result, int attempts) switch (result.StatusCode) { case 401 or 403: - Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); break; case not null: - Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); break; default: - Log.Error(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); + Log.Warning(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); break; } } From 4f892ab9286cd70ed89f66b1979d43b82abc23e5 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:48:41 +0300 Subject: [PATCH 30/62] [FeatureFlags] Fix stream disposal when reading the agentless response body --- .../Agentless/AgentlessConfigurationSource.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index bcd0d5dafddf..68e358202fa5 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -356,23 +356,23 @@ private async Task RequestAsync() private async Task ReadBodyAsync(IApiResponse response) { - var stream = await response.GetStreamAsync().ConfigureAwait(false); - GZipStream? decompressed = null; - - try - { - if (response.GetContentEncodingType() == ContentEncodingType.GZip) - { - decompressed = new GZipStream(stream, CompressionMode.Decompress); - } - - using var reader = new StreamReader(decompressed ?? stream, response.GetCharsetEncoding()); - return await reader.ReadToEndAsync().ConfigureAwait(false); - } - finally - { - decompressed?.Dispose(); - } + using var stream = await response.GetStreamAsync().ConfigureAwait(false); + + using var decompressed = + response.GetContentEncodingType() == ContentEncodingType.GZip + ? new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true) + : null; + + // Every parameter has to be given to reach leaveOpen. A byte order mark is not expected, and + // letting one be detected would override the encoding the response declared. + using var reader = new StreamReader( + decompressed ?? stream, + response.GetCharsetEncoding(), + detectEncodingFromByteOrderMarks: false, + bufferSize: 1024, // the default + leaveOpen: true); + + return await reader.ReadToEndAsync().ConfigureAwait(false); } private Task ApplyAsync(PollResult result) From a522bfb49d5880c388297395c9b499abf0daf557 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:52:31 +0300 Subject: [PATCH 31/62] [FeatureFlags] Make the agentless apply step synchronous and pass PollResult by in --- .../Agentless/AgentlessConfigurationSource.cs | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 68e358202fa5..1cffba761f54 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -170,7 +170,7 @@ internal async Task PollAsync() return; } - if (!IsRetryable(result)) + if (!IsRetryable(in result)) { break; } @@ -178,7 +178,7 @@ internal async Task PollAsync() if (attempt == MaxAttempts) { // Every attempt failed in a retryable way. Last-known-good stays in place. - WarnFailure(result, MaxAttempts); + WarnFailure(in result, MaxAttempts); return; } @@ -197,7 +197,7 @@ internal async Task PollAsync() return; } - await ApplyAsync(result).ConfigureAwait(false); + Apply(in result); } public void Dispose() @@ -375,19 +375,19 @@ private async Task ReadBodyAsync(IApiResponse response) return await reader.ReadToEndAsync().ConfigureAwait(false); } - private Task ApplyAsync(PollResult result) + private void Apply(in PollResult result) { switch (result.StatusCode) { case 304: // Nothing changed, and the ETag stays as it is. - return Task.CompletedTask; + return; case 401 or 403: - WarnFailure(result, attempts: 1); - return Task.CompletedTask; + WarnFailure(in result, attempts: 1); + return; case not 200: - WarnFailure(result, attempts: 1); - return Task.CompletedTask; + WarnFailure(in result, attempts: 1); + return; } if (!UfcConfigurationParser.TryParse(result.Body, out var configuration, out var error)) @@ -398,7 +398,7 @@ private Task ApplyAsync(PollResult result) Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", error); } - return Task.CompletedTask; + return; } if (!_applyConfiguration(configuration)) @@ -409,7 +409,7 @@ private Task ApplyAsync(PollResult result) Log.Warning("Feature Flags agentless configuration could not be applied"); } - return Task.CompletedTask; + return; } // The ETag advances only once parsing and applying have both succeeded. Advancing on @@ -417,8 +417,6 @@ private Task ApplyAsync(PollResult result) // answer 304, pinning the process to stale configuration with no way back. var newEtag = result.ETag?.Trim(); _etag = StringUtil.IsNullOrEmpty(newEtag) ? null : newEtag; - - return Task.CompletedTask; } /// From ffcfc15e2c89da28c4a3d9a6b72a62228be6acff Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:53:54 +0300 Subject: [PATCH 32/62] [FeatureFlags] Start the agentless poll loop with Task.Run --- .../FeatureFlags/Agentless/AgentlessConfigurationSource.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 1cffba761f54..cf3b18ee6040 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -146,10 +146,9 @@ public void Start() return; } - // Deliberately not wrapped in Task.Run: this is called from provider initialization, which - // is waiting for the first configuration, so the first request should go out on the calling - // thread rather than queue behind whatever else is on the thread pool. - _ = RunAsync().ContinueWith(t => Log.Error(t.Exception, "Feature Flags agentless poll loop failed"), TaskContinuationOptions.OnlyOnFaulted); + // The loop runs on the thread pool, so nothing of it happens on the caller's thread. Nothing + // ever awaits it either, so a fault is observed here or not at all. + _ = Task.Run(RunAsync).ContinueWith(t => Log.Error(t.Exception, "Feature Flags agentless poll loop failed"), TaskContinuationOptions.OnlyOnFaulted); } /// From da215dbfb715d922bc94f0fba943ac2c9b500fef Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 17:01:55 +0300 Subject: [PATCH 33/62] [FeatureFlags] Rebuild the agentless request URI only when the environment changes --- .../Agentless/AgentlessConfigurationSource.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index cf3b18ee6040..f8c0ced5886b 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -56,8 +56,9 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); - // Written by the settings-change callback, read by the poll loop. - private string? _environment; + // Written by the settings-change callback, read by the poll loop. The URI is stored rather than + // the environment it carries, so it is only built when the environment changes. + private Uri _requestUri; private IDisposable? _environmentSubscription; // Only ever touched from the poll loop. @@ -82,7 +83,7 @@ internal AgentlessConfigurationSource( _pollInterval = pollInterval; _requestTimeout = requestTimeout; _applyConfiguration = applyConfiguration; - _environment = environment; + _requestUri = endpoint.BuildRequestUri(environment); _waitAsync = waitAsync ?? Task.Delay; } @@ -117,8 +118,8 @@ internal AgentlessConfigurationSource( manager.InitialMutableSettings.Environment); // The environment is tracked rather than captured: customers can change it in code while - // the application runs, and flags are targeted per environment. Only the value is stored - // here, so that the poll loop stays the only thing that touches the request state. + // the application runs, and flags are targeted per environment. Only the request URI is + // stored here, so that the poll loop stays the only thing that touches the request state. source._environmentSubscription = manager.SubscribeToChanges(changes => { if (changes.UpdatedMutable is { } mutable) @@ -131,10 +132,11 @@ internal AgentlessConfigurationSource( } /// - /// Records the environment to request configuration for. Applied by the poll loop on its next - /// request, so a change never disturbs a request already in flight. + /// Records the environment to request configuration for, as the URI that carries it. Picked up + /// by the poll loop on its next request, so a change never disturbs a request already in flight. /// - internal void UpdateEnvironment(string? environment) => Volatile.Write(ref _environment, environment); + internal void UpdateEnvironment(string? environment) + => Volatile.Write(ref _requestUri, _endpoint.BuildRequestUri(environment)); /// /// Starts polling. Idempotent. @@ -300,9 +302,9 @@ private async Task RequestAsync() { try { - // The environment is applied per request rather than baked into the endpoint, because - // it can be changed in code after startup. - var uri = _endpoint.BuildRequestUri(Volatile.Read(ref _environment)); + // Read per request rather than captured, because the environment it carries can be + // changed in code after startup. + var uri = Volatile.Read(ref _requestUri); // An ETag only identifies the configuration served for the URI it came from. Sending it // against a different environment would earn a 304 and pin the process to the previous From 0ad52cacd408ca646779477659b25b494954c2cd Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 17:13:44 +0300 Subject: [PATCH 34/62] [FeatureFlags] Parse the agentless payload straight from the response stream --- .../Agentless/AgentlessConfigurationSource.cs | 69 +++++++++++-------- .../Agentless/UfcConfigurationParser.cs | 11 ++- .../UfcConfigurationParserTests.cs | 18 +++-- 3 files changed, 57 insertions(+), 41 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index f8c0ced5886b..058d06e9d98f 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -339,43 +339,50 @@ private async Task RequestAsync() t => { try { using var r = t.Result; } catch { } }, TaskContinuationOptions.None); - return new PollResult(statusCode: null, etag: null, body: null, error: new TimeoutException($"Feature Flags agentless request timed out after {_requestTimeout.TotalSeconds}s")); + return new PollResult(statusCode: null, etag: null, configuration: null, parseError: null, error: new TimeoutException($"Feature Flags agentless request timed out after {_requestTimeout.TotalSeconds}s")); } using var response = await getTask.ConfigureAwait(false); #endif - // Only a 200 carries configuration; other bodies are never decoded as one. - var body = response.StatusCode == 200 ? await ReadBodyAsync(response).ConfigureAwait(false) : null; - return new PollResult(response.StatusCode, response.GetHeader("ETag"), body, error: null); + // Only a 200 carries configuration; other bodies are never decoded as one. The payload + // is parsed here, while the response is still open, so it never has to be held as a + // string. A payload that does not parse is reported, not thrown: it is not retryable. + ServerConfiguration? configuration = null; + string? parseError = null; + + if (response.StatusCode == 200) + { + using var stream = await response.GetStreamAsync().ConfigureAwait(false); + + using var decompressed = + response.GetContentEncodingType() == ContentEncodingType.GZip + ? new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true) + : null; + + // Every parameter has to be given to reach leaveOpen. A byte order mark is not + // expected, and letting one be detected would override the declared encoding. + using var reader = new StreamReader( + decompressed ?? stream, + response.GetCharsetEncoding(), + detectEncodingFromByteOrderMarks: false, + bufferSize: 1024, // the default + leaveOpen: true); + + if (UfcConfigurationParser.TryParse(reader, out var parsed, out parseError)) + { + configuration = parsed; + } + } + + return new PollResult(response.StatusCode, response.GetHeader("ETag"), configuration, parseError, error: null); } catch (Exception ex) { - return new PollResult(statusCode: null, etag: null, body: null, error: ex); + return new PollResult(statusCode: null, etag: null, configuration: null, parseError: null, error: ex); } } - private async Task ReadBodyAsync(IApiResponse response) - { - using var stream = await response.GetStreamAsync().ConfigureAwait(false); - - using var decompressed = - response.GetContentEncodingType() == ContentEncodingType.GZip - ? new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true) - : null; - - // Every parameter has to be given to reach leaveOpen. A byte order mark is not expected, and - // letting one be detected would override the encoding the response declared. - using var reader = new StreamReader( - decompressed ?? stream, - response.GetCharsetEncoding(), - detectEncodingFromByteOrderMarks: false, - bufferSize: 1024, // the default - leaveOpen: true); - - return await reader.ReadToEndAsync().ConfigureAwait(false); - } - private void Apply(in PollResult result) { switch (result.StatusCode) @@ -391,12 +398,12 @@ private void Apply(in PollResult result) return; } - if (!UfcConfigurationParser.TryParse(result.Body, out var configuration, out var error)) + if (result.Configuration is not { } configuration) { if (!_malformedPayloadLogged) { _malformedPayloadLogged = true; - Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", error); + Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", result.ParseError); } return; @@ -452,13 +459,15 @@ private void WarnFailure(in PollResult result, int attempts) } } - internal readonly struct PollResult(int? statusCode, string? etag, string? body, Exception? error) + internal readonly struct PollResult(int? statusCode, string? etag, ServerConfiguration? configuration, string? parseError, Exception? error) { public int? StatusCode { get; } = statusCode; public string? ETag { get; } = etag; - public string? Body { get; } = body; + public ServerConfiguration? Configuration { get; } = configuration; + + public string? ParseError { get; } = parseError; public Exception? Error { get; } = error; } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs index a078f5f4374d..50e1fd6fa565 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -26,11 +26,11 @@ internal static class UfcConfigurationParser /// which is the document the evaluator consumes. A raw UFC document is rejected, including from /// a custom endpoint, so that every source agrees on one wire format. /// - /// The response body. + /// The response body. Read straight from the response, so it never has to be held as a string. /// The parsed configuration. /// Why the payload was rejected. /// true when the payload matches the contract. - public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfiguration? configuration, out string? error) + public static bool TryParse(TextReader body, [NotNullWhen(true)] out ServerConfiguration? configuration, out string? error) { configuration = null; error = null; @@ -38,11 +38,10 @@ public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfigur JToken payload; try { - using var stringReader = new StringReader(body ?? string.Empty); - // Timestamps stay strings: the model carries createdAt verbatim, and letting Newtonsoft - // turn it into a date would also make the type check below fail. - using var jsonReader = new JsonTextReader(stringReader) { DateParseHandling = DateParseHandling.None }; + // turn it into a date would also make the type check below fail. The reader belongs to + // the caller, which owns the response it came from. + using var jsonReader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false }; payload = JToken.ReadFrom(jsonReader); } catch (Exception) diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs index b57c9568ebcd..cfdf5f85adf9 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs @@ -6,6 +6,7 @@ #nullable enable using System.Collections.Generic; +using System.IO; using Datadog.Trace.FeatureFlags.Agentless; using Datadog.Trace.FeatureFlags.Rcm.Model; using FluentAssertions; @@ -29,7 +30,7 @@ public class UfcConfigurationParserTests [Fact] public void ParsesValidEnvelope() { - UfcConfigurationParser.TryParse(ValidEnvelope, out var configuration, out var error) + Parse(ValidEnvelope, out var configuration, out var error) .Should().BeTrue(); error.Should().BeNull(); @@ -45,7 +46,7 @@ public void ParsesValidEnvelope() [InlineData("{ \"data\": ")] public void RejectsMalformedJson(string? body) { - UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + Parse(body, out var configuration, out var error).Should().BeFalse(); configuration.Should().BeNull(); error.Should().Be("Malformed UFC payload"); @@ -66,7 +67,7 @@ public void RejectsMalformedJson(string? body) [InlineData("""{ "data": { "type": [1, 2], "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] public void RejectsInvalidEnvelope(string body) { - UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + Parse(body, out var configuration, out var error).Should().BeFalse(); configuration.Should().BeNull(); error.Should().Be("Expected a JSON:API Universal Flag Configuration resource"); @@ -87,7 +88,7 @@ public void RejectsInvalidEnvelope(string body) [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": [] } } }""")] public void RejectsInvalidAttributes(string body) { - UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + Parse(body, out var configuration, out var error).Should().BeFalse(); configuration.Should().BeNull(); error.Should().Be("Expected a Universal Flag Configuration v1 object"); @@ -103,10 +104,17 @@ public void ParsesFlagsFromEnvelope() "flags": { "test-flag": { "key": "test-flag", "enabled": true, "variationType": "BOOLEAN" } } } } } """; - UfcConfigurationParser.TryParse(body, out var configuration, out _).Should().BeTrue(); + Parse(body, out var configuration, out _).Should().BeTrue(); configuration!.Flags.Should().NotBeNull(); configuration!.Flags!.Should().ContainKey("test-flag"); configuration!.Flags!["test-flag"].Enabled.Should().BeTrue(); } + + // The parser reads the response stream directly, so a body under test is handed to it as a reader. + private static bool Parse(string? body, out ServerConfiguration? configuration, out string? error) + { + using var reader = new StringReader(body ?? string.Empty); + return UfcConfigurationParser.TryParse(reader, out configuration, out error); + } } From e9936a2667e7b527ea706e7f86586eb347eaedfc Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 17:27:24 +0300 Subject: [PATCH 35/62] [FeatureFlags] Deserialize the UFC envelope without building a JSON tree --- .../Agentless/UfcConfigurationParser.cs | 149 ++++++++++++++---- .../Rcm/Model/FlagCollectionJsonConverter.cs | 9 ++ .../UfcConfigurationParserTests.cs | 16 +- 3 files changed, 140 insertions(+), 34 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs index 50e1fd6fa565..b03f17141196 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -10,7 +10,6 @@ using System.IO; using Datadog.Trace.FeatureFlags.Rcm.Model; using Datadog.Trace.Vendors.Newtonsoft.Json; -using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; namespace Datadog.Trace.FeatureFlags.Agentless; @@ -21,10 +20,19 @@ internal static class UfcConfigurationParser { private const string ResourceType = "universal-flag-configuration"; + private const string MalformedError = "Malformed UFC payload"; + private const string ResourceError = "Expected a JSON:API Universal Flag Configuration resource"; + private const string AttributesError = "Expected a Universal Flag Configuration v1 object"; + /// /// Validates a JSON:API Universal Flag Configuration response and returns data.attributes, /// which is the document the evaluator consumes. A raw UFC document is rejected, including from /// a custom endpoint, so that every source agrees on one wire format. + /// + /// The envelope is walked with the reader rather than loaded into a JSON tree, and + /// data.attributes is deserialized in place, so the payload is read exactly once and no + /// copy of it is ever held. + /// /// /// The response body. Read straight from the response, so it never has to be held as a string. /// The parsed configuration. @@ -35,56 +43,133 @@ public static bool TryParse(TextReader body, [NotNullWhen(true)] out ServerConfi configuration = null; error = null; - JToken payload; + var sawData = false; + string? resourceType = null; + ServerConfiguration? attributes = null; + try { // Timestamps stay strings: the model carries createdAt verbatim, and letting Newtonsoft // turn it into a date would also make the type check below fail. The reader belongs to // the caller, which owns the response it came from. - using var jsonReader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false }; - payload = JToken.ReadFrom(jsonReader); + using var reader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false }; + var serializer = new JsonSerializer { DateParseHandling = DateParseHandling.None }; + + if (!reader.Read()) + { + // Nothing at all, so there is no document to judge against the contract. + error = MalformedError; + return false; + } + + if (reader.TokenType != JsonToken.StartObject) + { + error = ResourceError; + return false; + } + + while (reader.Read() && reader.TokenType == JsonToken.PropertyName) + { + if ((string?)reader.Value != "data") + { + reader.Skip(); + continue; + } + + sawData = true; + + if (!reader.Read()) + { + // The document ended where the resource should have been. + error = MalformedError; + return false; + } + + if (reader.TokenType != JsonToken.StartObject) + { + error = ResourceError; + return false; + } + + while (reader.Read() && reader.TokenType == JsonToken.PropertyName) + { + switch ((string?)reader.Value) + { + case "type": + if (!reader.Read()) + { + error = MalformedError; + return false; + } + + // A type that is not a string cannot identify the resource. Checked on + // the token, because a number would otherwise be read as its digits. + if (reader.TokenType != JsonToken.String) + { + error = ResourceError; + return false; + } + + resourceType = (string?)reader.Value; + break; + + case "attributes": + if (!reader.Read()) + { + error = MalformedError; + return false; + } + + if (reader.TokenType != JsonToken.StartObject) + { + error = AttributesError; + return false; + } + + attributes = serializer.Deserialize(reader); + break; + + default: + reader.Skip(); + break; + } + } + } + + // A document that ends before the root object closes was truncated in transit, whatever + // was found in it up to that point. + if (reader.TokenType != JsonToken.EndObject) + { + error = MalformedError; + return false; + } } catch (Exception) { - error = "Malformed UFC payload"; + error = MalformedError; return false; } - if (payload is not JObject - || payload["data"] is not JObject data - || data["type"]?.Type != JTokenType.String - || data["type"]?.Value() != ResourceType) + if (!sawData || resourceType != ResourceType) { - error = "Expected a JSON:API Universal Flag Configuration resource"; + error = ResourceError; return false; } - if (data["attributes"] is not JObject attributes - || attributes["format"]?.Type != JTokenType.String - || attributes["createdAt"]?.Type != JTokenType.String - || attributes["environment"] is not JObject environment - || environment["name"]?.Type != JTokenType.String - || attributes["flags"] is not JObject) - { - error = "Expected a Universal Flag Configuration v1 object"; - return false; - } - - try - { - configuration = attributes.ToObject(); - } - catch (Exception) - { - configuration = null; - } - - if (configuration is null) + // Every member of the v1 contract has to be there. A member of the wrong shape arrives as + // null, because the flag collection rejects anything that is not an object and Newtonsoft + // leaves a member it cannot convert unset. + if (attributes is null + || attributes.Format is null + || attributes.CreatedAt is null + || attributes.Environment?.Name is null + || attributes.Flags is null) { - error = "Expected a Universal Flag Configuration v1 object"; + error = AttributesError; return false; } + configuration = attributes; return true; } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs b/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs index 87a4a8937986..101818093e56 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs @@ -25,6 +25,15 @@ internal sealed class FlagCollectionJsonConverter : JsonConverter Date: Wed, 26 Aug 2026 17:31:53 +0300 Subject: [PATCH 36/62] [FeatureFlags] Subscribe to settings changes in the agentless constructor --- .../Agentless/AgentlessConfigurationSource.cs | 40 ++++++++++--------- .../AgentlessConfigurationSourceTests.cs | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 058d06e9d98f..35f966af1c4e 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -56,10 +56,11 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); + private readonly IDisposable? _environmentSubscription; + // Written by the settings-change callback, read by the poll loop. The URI is stored rather than // the environment it carries, so it is only built when the environment changes. private Uri _requestUri; - private IDisposable? _environmentSubscription; // Only ever touched from the poll loop. private bool _malformedPayloadLogged; @@ -76,6 +77,7 @@ internal AgentlessConfigurationSource( TimeSpan requestTimeout, Func applyConfiguration, string? environment = null, + TracerSettings.SettingsManager? settingsManager = null, Func? waitAsync = null) { _endpoint = endpoint; @@ -85,6 +87,17 @@ internal AgentlessConfigurationSource( _applyConfiguration = applyConfiguration; _requestUri = endpoint.BuildRequestUri(environment); _waitAsync = waitAsync ?? Task.Delay; + + // Subscribed last, so every field the callback touches is already set. The environment is + // tracked rather than captured: customers can change it in code while the application runs, + // and flags are targeted per environment. + _environmentSubscription = settingsManager?.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + UpdateEnvironment(mutable.Environment); + } + }); } /// @@ -109,26 +122,14 @@ internal AgentlessConfigurationSource( return null; } - var source = new AgentlessConfigurationSource( + return new AgentlessConfigurationSource( endpoint, CreateRequestFactory(endpoint, settings), settings.PollInterval, settings.RequestTimeout, applyConfiguration, - manager.InitialMutableSettings.Environment); - - // The environment is tracked rather than captured: customers can change it in code while - // the application runs, and flags are targeted per environment. Only the request URI is - // stored here, so that the poll loop stays the only thing that touches the request state. - source._environmentSubscription = manager.SubscribeToChanges(changes => - { - if (changes.UpdatedMutable is { } mutable) - { - source.UpdateEnvironment(mutable.Environment); - } - }); - - return source; + manager.InitialMutableSettings.Environment, + manager); } /// @@ -199,6 +200,10 @@ internal async Task PollAsync() } Apply(in result); + + // A failure with no status code never reached the endpoint, so it is worth another attempt. + static bool IsRetryable(in PollResult result) + => result.StatusCode is not { } status || status is 408 or 429 or (>= 500 and <= 599); } public void Dispose() @@ -254,9 +259,6 @@ private static ApiWebRequestFactory CreateRequestFactory(AgentlessEndpoint endpo #endif } - private static bool IsRetryable(in PollResult result) - => result.StatusCode is not { } status || status is 408 or 429 or (>= 500 and <= 599); - private async Task RunAsync() { Log.Debug("AgentlessConfigurationSource::RunAsync -> Enter"); diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index 276f28b37251..b87daf911085 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -256,7 +256,7 @@ private static AgentlessConfigurationSource CreateSource( return true; }, environment, - NoWait); + waitAsync: NoWait); private static AgentlessEndpoint CreateEndpoint() { From 97fdde3202cb4193ebe1e1fda3f1a211671e284f Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 18:00:02 +0300 Subject: [PATCH 37/62] [FeatureFlags] Add dd_env to the managed agentless endpoint only --- .../Agentless/AgentlessEndpoint.cs | 50 ++++--------------- .../FeatureFlags/AgentlessEndpointTests.cs | 29 +++-------- 2 files changed, 18 insertions(+), 61 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index 11e258035245..a92e60db653f 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -31,19 +31,15 @@ internal sealed class AgentlessEndpoint internal const string ManagedHostPrefix = "ufc-server.ff-cdn."; /// - /// The query parameter carrying the environment to request configuration for. + /// The query parameter carrying the environment to request configuration for. Added to the + /// managed endpoint only. /// internal const string EnvParameterName = "dd_env"; - // Set when the configured base URL already carries dd_env. The operator chose that value - // deliberately, so it is left alone rather than duplicated or overwritten. - private readonly bool _pinsEnv; - - private AgentlessEndpoint(Uri uri, bool isManaged, bool pinsEnv) + private AgentlessEndpoint(Uri uri, bool isManaged) { Uri = uri; IsManaged = isManaged; - _pinsEnv = pinsEnv; } /// @@ -62,8 +58,8 @@ private AgentlessEndpoint(Uri uri, bool isManaged, bool pinsEnv) /// /// Builds the endpoint. Without a custom the managed Datadog CDN /// endpoint is derived from the (lowercased) site, so staging and government sites resolve - /// with no allowlist. A custom base URL that is an origin receives the canonical path; one that - /// carries a path is used verbatim. + /// with no allowlist, and is the only endpoint dd_env is added to. A custom base URL that + /// is an origin receives the canonical path; one that carries a path is used verbatim. /// /// The environment is not part of the endpoint: it can be changed in code while the application /// runs, so it is applied per request by instead. @@ -106,7 +102,7 @@ public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] return false; } - endpoint = new AgentlessEndpoint(managedUri, isManaged: true, pinsEnv: false); + endpoint = new AgentlessEndpoint(managedUri, isManaged: true); return true; } @@ -138,7 +134,7 @@ public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] custom = new UriBuilder(custom) { Path = DefaultPath }.Uri; } - endpoint = new AgentlessEndpoint(custom, isManaged: false, pinsEnv: HasEnvParameter(custom.Query)); + endpoint = new AgentlessEndpoint(custom, isManaged: false); return true; } @@ -147,15 +143,16 @@ public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] /// added as a query parameter rather than baked into the endpoint, because it can change while /// the application runs. /// - /// Any query the configured base URL already carries is kept: it may hold credentials or routing - /// the operator needs. An endpoint that already pins dd_env is returned unchanged. + /// A configured base URL is opaque: it is requested exactly as the operator wrote it, since it + /// may hold credentials, routing, or a scope of its own. Only the managed endpoint carries + /// dd_env, which matches the other tracers. /// /// /// The current environment, or null when none is configured. /// The URI to request. public Uri BuildRequestUri(string? env) { - if (_pinsEnv) + if (!IsManaged) { return Uri; } @@ -180,29 +177,4 @@ public Uri BuildRequestUri(string? env) builder.Query = existing.Length > 1 ? existing.TrimStart('?') + "&" + parameter : parameter; return builder.Uri; } - - /// - /// Reports whether a query string already carries a dd_env parameter. Matching the name - /// alone would also hit a value such as ?next=dd_env, so the surrounding delimiters are - /// checked too. - /// - private static bool HasEnvParameter(string query) - { - var index = query.IndexOf(EnvParameterName, StringComparison.OrdinalIgnoreCase); - while (index >= 0) - { - var preceding = index == 0 ? '?' : query[index - 1]; - var followingIndex = index + EnvParameterName.Length; - var following = followingIndex < query.Length ? query[followingIndex] : '\0'; - - if (preceding is '?' or '&' && following is '=' or '&' or '\0') - { - return true; - } - - index = query.IndexOf(EnvParameterName, index + 1, StringComparison.OrdinalIgnoreCase); - } - - return false; - } } diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs index 2f53914f4b7c..00f67195754e 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -67,36 +67,21 @@ public void TruncatesAnOverlongDdEnvValue() => Create("datadoghq.com").BuildRequestUri(new string('a', 500)) .Query.Should().Be("?dd_env=" + new string('a', 200)); - [Fact] - public void KeepsTheQueryConfiguredOnACustomEndpoint() - { - // The query may carry credentials or routing the operator needs, so dd_env is appended to it - // rather than replacing it. - var endpoint = Create("datadoghq.com", baseUrl: "https://flags.example.com/ufc?token=abc"); - - endpoint.BuildRequestUri("production").Query.Should().Be("?token=abc&dd_env=production"); - } - [Theory] + // A configured base URL is opaque: the operator may have scoped it themselves, and it may carry + // credentials or routing, so it is requested exactly as written. Java, JS, Go and Python all + // treat a custom endpoint the same way. + [InlineData("https://flags.example.com/ufc")] + [InlineData("https://flags.example.com/ufc?token=abc")] [InlineData("https://flags.example.com/ufc?dd_env=staging")] [InlineData("https://flags.example.com/ufc?token=abc&dd_env=staging")] - public void LeavesACustomEndpointThatAlreadyPinsDdEnvAlone(string baseUrl) + public void DoesNotAddDdEnvToACustomEndpoint(string baseUrl) { - var endpoint = Create("datadoghq.com", baseUrl: baseUrl); + var endpoint = Create("datadoghq.com", baseUrl); endpoint.BuildRequestUri("production").Should().Be(endpoint.Uri); } - [Fact] - public void AddsDdEnvToACustomEndpointWithoutAQuery() - => Create("datadoghq.com", baseUrl: "https://flags.example.com/ufc") - .BuildRequestUri("production").Query.Should().Be("?dd_env=production"); - - [Fact] - public void DoesNotMistakeAQueryValueForAPinnedDdEnv() - => Create("datadoghq.com", baseUrl: "https://flags.example.com/ufc?next=dd_env") - .BuildRequestUri("production").Query.Should().Be("?next=dd_env&dd_env=production"); - [Theory] [InlineData("https://flags.example.com", "https://flags.example.com" + DefaultPath)] [InlineData("https://flags.example.com/", "https://flags.example.com" + DefaultPath)] From eba74ae8c3fd432ff5e830c0760ac655877d7b59 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 18:08:36 +0300 Subject: [PATCH 38/62] [FeatureFlags] Name the offline source after the value that selects it --- .../FeatureFlags/FeatureFlagsSettings.cs | 16 ++++++++-------- .../FeatureFlags/FeatureFlagsSource.cs | 6 ++++-- .../FeatureFlags/FeatureFlagsSettingsTests.cs | 16 ++++++++-------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 654148504d05..193f3db804b5 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -32,8 +32,8 @@ internal sealed class FeatureFlagsSettings // one language the caller's default value while the others still return a real one. internal const int DefaultInitializationTimeoutMs = 30_000; - // An interval above this is indistinguishable from "never poll" and is more likely a - // misconfiguration (for example milliseconds passed as seconds) than an intent. + // One hour. An interval above this is more likely a misconfiguration (for example milliseconds + // passed as seconds) than an intent, and the other tracers cap it at the same value. private const int MaxPollIntervalSeconds = 3600; private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(FeatureFlagsSettings)); @@ -68,10 +68,10 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr // adopters onto Remote Configuration, and everything else defaults to agentless. DefaultResult defaultSource = enabled switch { - false => new(FeatureFlagsSource.Disabled, OfflineSourceName), + false => new(FeatureFlagsSource.Offline, OfflineSourceName), null when legacyEnabled is not null => legacyEnabled.Value ? new(FeatureFlagsSource.RemoteConfig, RemoteConfigSourceName) - : new(FeatureFlagsSource.Disabled, OfflineSourceName), + : new(FeatureFlagsSource.Offline, OfflineSourceName), _ => new(FeatureFlagsSource.Agentless, AgentlessSourceName), }; @@ -114,14 +114,14 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr } /// - /// Gets the resolved delivery source. means nothing is contacted. + /// Gets the resolved delivery source. means nothing is contacted. /// public FeatureFlagsSource Source { get; } /// /// Gets a value indicating whether Feature Flags are enabled at all. /// - public bool Enabled => Source != FeatureFlagsSource.Disabled; + public bool Enabled => Source != FeatureFlagsSource.Offline; /// /// Gets the configured override for the agentless endpoint, or null to derive it from the site. @@ -193,11 +193,11 @@ private static bool TryMatch(string value, out FeatureFlagsSource source) // "offline" is a reserved fail-closed sentinel: the provider is intentionally off. if (string.Equals(value, OfflineSourceName, StringComparison.OrdinalIgnoreCase)) { - source = FeatureFlagsSource.Disabled; + source = FeatureFlagsSource.Offline; return true; } - source = FeatureFlagsSource.Disabled; + source = FeatureFlagsSource.Offline; return false; } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs index dd50047ede42..558a28900adf 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs @@ -13,9 +13,11 @@ namespace Datadog.Trace.FeatureFlags; internal enum FeatureFlagsSource { /// - /// Feature Flags are disabled: no configuration is loaded, and neither delivery path is contacted. + /// Nothing is contacted and no configuration is loaded, so evaluations return the caller's + /// default value. Named after the offline value that selects it: it is a source rather + /// than an off switch, because it is reserved for serving configuration supplied at startup. /// - Disabled, + Offline, /// /// Configuration is fetched over HTTP, without the Datadog Agent. diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index e9548b61c6a3..b83c204da4d9 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -23,10 +23,10 @@ public class FeatureFlagsSettingsTests // Nothing configured: agentless is the default. [InlineData(null, null, null, FeatureFlagsSource.Agentless)] // The stable kill switch wins over everything, including a legacy opt-in and an explicit source. - [InlineData("false", null, null, FeatureFlagsSource.Disabled)] - [InlineData("false", null, "true", FeatureFlagsSource.Disabled)] - [InlineData("false", "agentless", null, FeatureFlagsSource.Disabled)] - [InlineData("false", "remote_config", null, FeatureFlagsSource.Disabled)] + [InlineData("false", null, null, FeatureFlagsSource.Offline)] + [InlineData("false", null, "true", FeatureFlagsSource.Offline)] + [InlineData("false", "agentless", null, FeatureFlagsSource.Offline)] + [InlineData("false", "remote_config", null, FeatureFlagsSource.Offline)] // Enabling explicitly does not imply the historical Remote Configuration source. [InlineData("true", null, null, FeatureFlagsSource.Agentless)] // An explicit source wins over the legacy key, in both directions. @@ -34,7 +34,7 @@ public class FeatureFlagsSettingsTests [InlineData(null, "remote_config", "false", FeatureFlagsSource.RemoteConfig)] // The legacy key grandfathers existing adopters, who opted in when RC was the only source. [InlineData(null, null, "true", FeatureFlagsSource.RemoteConfig)] - [InlineData(null, null, "false", FeatureFlagsSource.Disabled)] + [InlineData(null, null, "false", FeatureFlagsSource.Offline)] // An explicit new-key value takes precedence over the legacy key, so a stale legacy disable // does not silently keep Feature Flags off during migration. [InlineData("true", null, "false", FeatureFlagsSource.Agentless)] @@ -44,15 +44,15 @@ public class FeatureFlagsSettingsTests [InlineData(null, "invalid", null, FeatureFlagsSource.Agentless)] [InlineData(null, "invalid", "true", FeatureFlagsSource.RemoteConfig)] // "offline" is a reserved, recognised fail-closed sentinel (not an unrecognised value). - [InlineData(null, "offline", null, FeatureFlagsSource.Disabled)] - [InlineData(null, "offline", "true", FeatureFlagsSource.Disabled)] + [InlineData(null, "offline", null, FeatureFlagsSource.Offline)] + [InlineData(null, "offline", "true", FeatureFlagsSource.Offline)] public void ResolvesSource(string? enabled, string? source, string? legacyEnabled, object expected) { var expectedSource = (FeatureFlagsSource)expected; var settings = CreateSettings(enabled, source, legacyEnabled); settings.Source.Should().Be(expectedSource); - settings.Enabled.Should().Be(expectedSource != FeatureFlagsSource.Disabled); + settings.Enabled.Should().Be(expectedSource != FeatureFlagsSource.Offline); } [Theory] From 1c063c33d8e0817ceb2a7384c2ba85d49d59e0fc Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 18:16:13 +0300 Subject: [PATCH 39/62] [FeatureFlags] Address review comments on source resolution and endpoint validation --- .../Agentless/AgentlessEndpoint.cs | 14 ++++++++------ .../FeatureFlags/FeatureFlagsSettings.cs | 18 ++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index a92e60db653f..2780dafddcf7 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -85,17 +85,18 @@ public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] return false; } - var managedHost = ManagedHostPrefix + trimmedSite.ToLowerInvariant(); // A site accidentally set to e.g. "https://datadoghq.com" would produce a host like - // "ufc-server.ff-cdn.https://datadoghq.com" which Uri.TryCreate accepts as valid - // (treating the "//" as a path separator). Catch it explicitly; whitespace and - // invalid ports are already rejected by TryCreate. - if (managedHost.Contains("://")) + // "ufc-server.ff-cdn.https://datadoghq.com", which Uri.TryCreate accepts as valid + // (treating the "//" as a path separator), so it is caught here. Whitespace and invalid + // ports need no check: Uri.TryCreate does reject those inside a host. + if (trimmedSite.Contains("://")) { error = "The configured Datadog site is not valid"; return false; } + var managedHost = ManagedHostPrefix + trimmedSite.ToLowerInvariant(); + if (!Uri.TryCreate($"https://{managedHost}{DefaultPath}", UriKind.Absolute, out var managedUri)) { error = "The configured Datadog site is not valid"; @@ -106,7 +107,8 @@ public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] return true; } - // A URL with internal whitespace is malformed, and Uri parsing is lenient enough to accept it. + // Uri parsing rejects whitespace inside a host, but accepts it in a path or query, so a + // URL carrying it there is malformed and has to be caught explicitly. foreach (var character in configured) { if (char.IsWhiteSpace(character)) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 193f3db804b5..0c22fb787e33 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -61,17 +61,17 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr #pragma warning restore 618 } - // The source is resolved in a single read so configuration telemetry reports the value we - // actually use. Shared across tracers, so the precedence is deliberate: the stable kill + // The source key is read once, with every other outcome expressed as its default, so + // configuration telemetry reports the one value we act on rather than one entry per + // candidate key. Shared across tracers, so the precedence is deliberate: the stable kill // switch wins over everything (expressed as a validator that rejects any configured value), // an explicit source wins over the legacy key, the legacy key grandfathers existing // adopters onto Remote Configuration, and everything else defaults to agentless. - DefaultResult defaultSource = enabled switch + DefaultResult defaultSource = (enabled, legacyEnabled) switch { - false => new(FeatureFlagsSource.Offline, OfflineSourceName), - null when legacyEnabled is not null => legacyEnabled.Value - ? new(FeatureFlagsSource.RemoteConfig, RemoteConfigSourceName) - : new(FeatureFlagsSource.Offline, OfflineSourceName), + (false, _) => new(FeatureFlagsSource.Offline, OfflineSourceName), + (null, true) => new(FeatureFlagsSource.RemoteConfig, RemoteConfigSourceName), + (null, false) => new(FeatureFlagsSource.Offline, OfflineSourceName), _ => new(FeatureFlagsSource.Agentless, AgentlessSourceName), }; @@ -166,9 +166,7 @@ private static ParsingResult ConvertSource(string? value) return ParsingResult.Failure(); } - // Compared without allocating. A value with surrounding whitespace is trimmed only once the - // direct comparisons have failed, so the common path stays allocation-free. - if (TryMatch(value, out var source) || TryMatch(value.Trim(), out source)) + if (TryMatch(value.Trim(), out var source)) { return ParsingResult.Success(source); } From 85eacdb937edbbde64a1766c86087ee1d2d1e50e Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 18:30:51 +0300 Subject: [PATCH 40/62] FeatureFlags] Document the agentless base URL contract --- .../Configuration/supported-configurations.yaml | 11 ++++++++--- .../FeatureFlags/FeatureFlagsSettings.cs | 11 +++++++---- .../ConfigurationKeys.FeatureFlags.g.cs | 11 ++++++++--- .../ConfigurationKeys.FeatureFlags.g.cs | 11 ++++++++--- .../ConfigurationKeys.FeatureFlags.g.cs | 11 ++++++++--- .../ConfigurationKeys.FeatureFlags.g.cs | 11 ++++++++--- 6 files changed, 47 insertions(+), 19 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml index 7347d243d4d2..de6472239a3b 100644 --- a/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml +++ b/tracer/src/Datadog.Trace/Configuration/supported-configurations.yaml @@ -3206,9 +3206,14 @@ supportedConfigurations: const_name: FeatureFlagsConfigurationSourceAgentlessBaseUrl documentation: |- Configuration key for overriding the endpoint used by the agentless configuration source. - When the URL has no path, or a path of /, the standard rules-based server path is appended; - any other path is used verbatim as the exact endpoint. - If unset, the endpoint is derived from . + If unset, the endpoint is derived from and Datadog hosts it. + A configured URL is treated as an endpoint of your own, which changes three things: + the Datadog API key is not sent to it, so it is responsible for its own authentication; + it is requested exactly as written, so dd_env is not added and any environment or tenant + scope has to be part of the URL you configure; + and only its path is completed, when it has none or a path of /, with the standard + rules-based server path. Any other path is used verbatim as the exact endpoint. + The value may carry credentials, so it is never written to logs or telemetry. DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS: - implementation: A scope: managed diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 0c22fb787e33..766612537db8 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -67,11 +67,14 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr // switch wins over everything (expressed as a validator that rejects any configured value), // an explicit source wins over the legacy key, the legacy key grandfathers existing // adopters onto Remote Configuration, and everything else defaults to agentless. - DefaultResult defaultSource = (enabled, legacyEnabled) switch + // Not a tuple pattern: net461 has no System.ValueTuple, so (enabled, legacyEnabled) switch + // does not compile there. + DefaultResult defaultSource = enabled switch { - (false, _) => new(FeatureFlagsSource.Offline, OfflineSourceName), - (null, true) => new(FeatureFlagsSource.RemoteConfig, RemoteConfigSourceName), - (null, false) => new(FeatureFlagsSource.Offline, OfflineSourceName), + false => new(FeatureFlagsSource.Offline, OfflineSourceName), + null when legacyEnabled is not null => legacyEnabled.Value + ? new(FeatureFlagsSource.RemoteConfig, RemoteConfigSourceName) + : new(FeatureFlagsSource.Offline, OfflineSourceName), _ => new(FeatureFlagsSource.Agentless, AgentlessSourceName), }; diff --git a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index fc6c32d2b92b..288cd8f848eb 100644 --- a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -49,9 +49,14 @@ internal static class FeatureFlags /// /// Configuration key for overriding the endpoint used by the agentless configuration source. - /// When the URL has no path, or a path of /, the standard rules-based server path is appended; - /// any other path is used verbatim as the exact endpoint. - /// If unset, the endpoint is derived from . + /// If unset, the endpoint is derived from and Datadog hosts it. + /// A configured URL is treated as an endpoint of your own, which changes three things: + /// the Datadog API key is not sent to it, so it is responsible for its own authentication; + /// it is requested exactly as written, so dd_env is not added and any environment or tenant + /// scope has to be part of the URL you configure; + /// and only its path is completed, when it has none or a path of /, with the standard + /// rules-based server path. Any other path is used verbatim as the exact endpoint. + /// The value may carry credentials, so it is never written to logs or telemetry. /// public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; diff --git a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index fc6c32d2b92b..288cd8f848eb 100644 --- a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -49,9 +49,14 @@ internal static class FeatureFlags /// /// Configuration key for overriding the endpoint used by the agentless configuration source. - /// When the URL has no path, or a path of /, the standard rules-based server path is appended; - /// any other path is used verbatim as the exact endpoint. - /// If unset, the endpoint is derived from . + /// If unset, the endpoint is derived from and Datadog hosts it. + /// A configured URL is treated as an endpoint of your own, which changes three things: + /// the Datadog API key is not sent to it, so it is responsible for its own authentication; + /// it is requested exactly as written, so dd_env is not added and any environment or tenant + /// scope has to be part of the URL you configure; + /// and only its path is completed, when it has none or a path of /, with the standard + /// rules-based server path. Any other path is used verbatim as the exact endpoint. + /// The value may carry credentials, so it is never written to logs or telemetry. /// public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; diff --git a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index fc6c32d2b92b..288cd8f848eb 100644 --- a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -49,9 +49,14 @@ internal static class FeatureFlags /// /// Configuration key for overriding the endpoint used by the agentless configuration source. - /// When the URL has no path, or a path of /, the standard rules-based server path is appended; - /// any other path is used verbatim as the exact endpoint. - /// If unset, the endpoint is derived from . + /// If unset, the endpoint is derived from and Datadog hosts it. + /// A configured URL is treated as an endpoint of your own, which changes three things: + /// the Datadog API key is not sent to it, so it is responsible for its own authentication; + /// it is requested exactly as written, so dd_env is not added and any environment or tenant + /// scope has to be part of the URL you configure; + /// and only its path is completed, when it has none or a path of /, with the standard + /// rules-based server path. Any other path is used verbatim as the exact endpoint. + /// The value may carry credentials, so it is never written to logs or telemetry. /// public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; diff --git a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs index fc6c32d2b92b..288cd8f848eb 100644 --- a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/ConfigurationKeysGenerator/ConfigurationKeys.FeatureFlags.g.cs @@ -49,9 +49,14 @@ internal static class FeatureFlags /// /// Configuration key for overriding the endpoint used by the agentless configuration source. - /// When the URL has no path, or a path of /, the standard rules-based server path is appended; - /// any other path is used verbatim as the exact endpoint. - /// If unset, the endpoint is derived from . + /// If unset, the endpoint is derived from and Datadog hosts it. + /// A configured URL is treated as an endpoint of your own, which changes three things: + /// the Datadog API key is not sent to it, so it is responsible for its own authentication; + /// it is requested exactly as written, so dd_env is not added and any environment or tenant + /// scope has to be part of the URL you configure; + /// and only its path is completed, when it has none or a path of /, with the standard + /// rules-based server path. Any other path is used verbatim as the exact endpoint. + /// The value may carry credentials, so it is never written to logs or telemetry. /// public const string FeatureFlagsConfigurationSourceAgentlessBaseUrl = "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; From 4149d7b0d2cb86b8e24ebc7b4f121a19ee9ce985 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 18:49:27 +0300 Subject: [PATCH 41/62] [FeatureFlags] Fail closed on an unrecognised configuration source --- .../FeatureFlags/FeatureFlagsSettings.cs | 19 +++++++++++++------ .../FeatureFlags/FeatureFlagsSettingsTests.cs | 9 +++++---- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 766612537db8..42140bd6f726 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -65,8 +65,9 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr // configuration telemetry reports the one value we act on rather than one entry per // candidate key. Shared across tracers, so the precedence is deliberate: the stable kill // switch wins over everything (expressed as a validator that rejects any configured value), - // an explicit source wins over the legacy key, the legacy key grandfathers existing - // adopters onto Remote Configuration, and everything else defaults to agentless. + // an explicit source wins over the legacy key (an unrecognised one fails closed, so a typo + // never starts billed delivery), the legacy key grandfathers existing adopters onto Remote + // Configuration, and everything else defaults to agentless. // Not a tuple pattern: net461 has no System.ValueTuple, so (enabled, legacyEnabled) switch // does not compile there. DefaultResult defaultSource = enabled switch @@ -158,9 +159,8 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr /// /// Converts a configured source name to a . A blank value is - /// treated as unset, and an unrecognised one is reported as a parsing failure so that it shows - /// up as rejected in configuration telemetry rather than as a value nobody configured. Both - /// fall back to the default, which is what an unset key would have selected anyway. + /// treated as unset and falls back to the default, which is what an absent key would have + /// selected anyway. An unrecognised value fails closed instead: nothing is contacted. /// private static ParsingResult ConvertSource(string? value) { @@ -174,7 +174,14 @@ private static ParsingResult ConvertSource(string? value) return ParsingResult.Success(source); } - return ParsingResult.Failure(); + // A value nobody recognises fails closed rather than falling back to agentless: guessing a + // billed delivery path from a typo is worse than delivering nothing. Shared across tracers, + // and asserted by the system-tests parametric suite. + Log.Warning( + "Unsupported Feature Flags configuration source {Source}. No configuration will be delivered.", + value); + + return ParsingResult.Success(FeatureFlagsSource.Offline); } private static bool TryMatch(string value, out FeatureFlagsSource source) diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index b83c204da4d9..96055611a967 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -39,10 +39,11 @@ public class FeatureFlagsSettingsTests // does not silently keep Feature Flags off during migration. [InlineData("true", null, "false", FeatureFlagsSource.Agentless)] [InlineData("true", null, "true", FeatureFlagsSource.Agentless)] - // An unrecognised source is a parsing failure, reported as such in configuration telemetry, so - // the key behaves as if it were unset rather than resolving to a value nobody configured. - [InlineData(null, "invalid", null, FeatureFlagsSource.Agentless)] - [InlineData(null, "invalid", "true", FeatureFlagsSource.RemoteConfig)] + // An unrecognised source fails closed, and does so before the legacy key is considered: starting + // billed delivery off a typo is worse than delivering nothing. Java and JS resolve it the same + // way, and the system-tests parametric suite asserts no request is made. + [InlineData(null, "invalid", null, FeatureFlagsSource.Offline)] + [InlineData(null, "invalid", "true", FeatureFlagsSource.Offline)] // "offline" is a reserved, recognised fail-closed sentinel (not an unrecognised value). [InlineData(null, "offline", null, FeatureFlagsSource.Offline)] [InlineData(null, "offline", "true", FeatureFlagsSource.Offline)] From 3bfd025a59ca657587d9ccf632eafa4c551f98ae Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 18:57:54 +0300 Subject: [PATCH 42/62] [FeatureFlags] Reject a Datadog site that can redirect the managed endpoint --- .../Agentless/AgentlessEndpoint.cs | 18 +++++++++++------- .../FeatureFlags/AgentlessEndpointTests.cs | 6 ++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs index 2780dafddcf7..08868b22573d 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessEndpoint.cs @@ -85,14 +85,18 @@ public static bool TryCreate(string? site, string? baseUrl, [NotNullWhen(true)] return false; } - // A site accidentally set to e.g. "https://datadoghq.com" would produce a host like - // "ufc-server.ff-cdn.https://datadoghq.com", which Uri.TryCreate accepts as valid - // (treating the "//" as a path separator), so it is caught here. Whitespace and invalid - // ports need no check: Uri.TryCreate does reject those inside a host. - if (trimmedSite.Contains("://")) + // The site is concatenated into a host, so every character that can change what a URL means + // has to be rejected before that happens. "@" is the dangerous one: it would make the rest of + // the value the real host, and the API key would be sent there. "/", "?" and "#" would start a + // path, query or fragment, and ":" a port or a scheme. Uri.TryCreate accepts several of these, + // so it cannot be relied on to catch them. The other tracers reject the same set. + foreach (var character in trimmedSite) { - error = "The configured Datadog site is not valid"; - return false; + if (char.IsWhiteSpace(character) || character is '/' or '?' or '#' or '@' or ':') + { + error = "The configured Datadog site is not valid"; + return false; + } } var managedHost = ManagedHostPrefix + trimmedSite.ToLowerInvariant(); diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs index 00f67195754e..6e024bfe7456 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessEndpointTests.cs @@ -134,6 +134,12 @@ public void RejectsWhitespaceOnlySiteWithoutBaseUrl() [InlineData("https://datadoghq.com")] // user accidentally includes the scheme [InlineData("data dog hq.com")] // internal spaces [InlineData("datadoghq.com:99999")] // invalid port + // "@" would end the userinfo and make the remainder the real host, so the request, and with it + // the API key, would go to a host the operator never named. + [InlineData("datadoghq.com@attacker.example")] + [InlineData("datadoghq.com/../evil")] // a path escapes the host + [InlineData("datadoghq.com?x=1")] // a query escapes the host + [InlineData("datadoghq.com#f")] // a fragment escapes the host public void RejectsMalformedSiteWithoutThrowing(string site) { AgentlessEndpoint.TryCreate(site, baseUrl: null, out var endpoint, out var error) From d8e1083f6ca35507e628ca44220761d9af90b8fd Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 20:19:46 +0300 Subject: [PATCH 43/62] [FeatureFlags] Reject a Datadog site that can redirect the managed endpoint --- .../FeatureFlags/FeatureFlagsSettings.cs | 45 +++++++------------ .../FeatureFlags/FeatureFlagsSource.cs | 5 +-- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 42140bd6f726..023c93103631 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -68,8 +68,7 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr // an explicit source wins over the legacy key (an unrecognised one fails closed, so a typo // never starts billed delivery), the legacy key grandfathers existing adopters onto Remote // Configuration, and everything else defaults to agentless. - // Not a tuple pattern: net461 has no System.ValueTuple, so (enabled, legacyEnabled) switch - // does not compile there. + // net461 has no System.ValueTuple, so a tuple pattern over both values does not compile. DefaultResult defaultSource = enabled switch { false => new(FeatureFlagsSource.Offline, OfflineSourceName), @@ -169,9 +168,22 @@ private static ParsingResult ConvertSource(string? value) return ParsingResult.Failure(); } - if (TryMatch(value.Trim(), out var source)) + var trimmed = value.Trim(); + + if (string.Equals(trimmed, AgentlessSourceName, StringComparison.OrdinalIgnoreCase)) + { + return ParsingResult.Success(FeatureFlagsSource.Agentless); + } + + if (string.Equals(trimmed, RemoteConfigSourceName, StringComparison.OrdinalIgnoreCase)) + { + return ParsingResult.Success(FeatureFlagsSource.RemoteConfig); + } + + // "offline" is a reserved sentinel: the provider is intentionally off. + if (string.Equals(trimmed, OfflineSourceName, StringComparison.OrdinalIgnoreCase)) { - return ParsingResult.Success(source); + return ParsingResult.Success(FeatureFlagsSource.Offline); } // A value nobody recognises fails closed rather than falling back to agentless: guessing a @@ -183,29 +195,4 @@ private static ParsingResult ConvertSource(string? value) return ParsingResult.Success(FeatureFlagsSource.Offline); } - - private static bool TryMatch(string value, out FeatureFlagsSource source) - { - if (string.Equals(value, AgentlessSourceName, StringComparison.OrdinalIgnoreCase)) - { - source = FeatureFlagsSource.Agentless; - return true; - } - - if (string.Equals(value, RemoteConfigSourceName, StringComparison.OrdinalIgnoreCase)) - { - source = FeatureFlagsSource.RemoteConfig; - return true; - } - - // "offline" is a reserved fail-closed sentinel: the provider is intentionally off. - if (string.Equals(value, OfflineSourceName, StringComparison.OrdinalIgnoreCase)) - { - source = FeatureFlagsSource.Offline; - return true; - } - - source = FeatureFlagsSource.Offline; - return false; - } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs index 558a28900adf..3c85ae61433f 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSource.cs @@ -13,9 +13,8 @@ namespace Datadog.Trace.FeatureFlags; internal enum FeatureFlagsSource { /// - /// Nothing is contacted and no configuration is loaded, so evaluations return the caller's - /// default value. Named after the offline value that selects it: it is a source rather - /// than an off switch, because it is reserved for serving configuration supplied at startup. + /// Selected by offline. Nothing is contacted and no configuration is loaded, so + /// evaluations return the caller's default value. /// Offline, From aa8fc5be7168090b51ec7d5a093cd1ff8579f849 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 27 Aug 2026 09:33:49 +0300 Subject: [PATCH 44/62] [FeatureFlags] Separate whether Feature Flags run from where configuration comes from --- .../FeatureFlags/FeatureFlagsSettings.cs | 26 ++++++----- .../FeatureFlags/FeatureFlagsSettingsTests.cs | 43 ++++++++++--------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs index 023c93103631..9bbb0aa7234b 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/FeatureFlagsSettings.cs @@ -61,17 +61,17 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr #pragma warning restore 618 } + // Where configuration comes from is a separate question from whether the product runs, and + // the two are answered separately below, as the other tracers answer them. + // // The source key is read once, with every other outcome expressed as its default, so // configuration telemetry reports the one value we act on rather than one entry per - // candidate key. Shared across tracers, so the precedence is deliberate: the stable kill - // switch wins over everything (expressed as a validator that rejects any configured value), - // an explicit source wins over the legacy key (an unrecognised one fails closed, so a typo - // never starts billed delivery), the legacy key grandfathers existing adopters onto Remote + // candidate key. Shared across tracers, so the precedence is deliberate: an explicit source + // wins over the legacy key, the legacy key grandfathers existing adopters onto Remote // Configuration, and everything else defaults to agentless. // net461 has no System.ValueTuple, so a tuple pattern over both values does not compile. DefaultResult defaultSource = enabled switch { - false => new(FeatureFlagsSource.Offline, OfflineSourceName), null when legacyEnabled is not null => legacyEnabled.Value ? new(FeatureFlagsSource.RemoteConfig, RemoteConfigSourceName) : new(FeatureFlagsSource.Offline, OfflineSourceName), @@ -80,10 +80,13 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr Source = config .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSource) - .GetAs( - defaultSource, - validator: enabled == false ? static _ => false : static _ => true, - converter: ConvertSource); + .GetAs(defaultSource, validator: null, converter: ConvertSource); + + // The stable kill switch turns the product off whatever the source says. Beyond that, only a + // delivery source has anything to run: offline delivers nothing, and an unrecognised value + // resolves to offline so that a typo fails closed instead of starting billed delivery. + Enabled = enabled != false + && Source is FeatureFlagsSource.Agentless or FeatureFlagsSource.RemoteConfig; var agentlessBaseUrl = config .WithKeys(ConfigurationKeys.FeatureFlags.FeatureFlagsConfigurationSourceAgentlessBaseUrl) @@ -122,9 +125,10 @@ public FeatureFlagsSettings(IConfigurationSource? source, IConfigurationTelemetr public FeatureFlagsSource Source { get; } /// - /// Gets a value indicating whether Feature Flags are enabled at all. + /// Gets a value indicating whether Feature Flags run at all. Independent of , + /// which says where configuration would come from. /// - public bool Enabled => Source != FeatureFlagsSource.Offline; + public bool Enabled { get; } /// /// Gets the configured override for the agentless endpoint, or null to derive it from the site. diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs index 96055611a967..79d67eedb464 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/FeatureFlagsSettingsTests.cs @@ -19,41 +19,42 @@ public class FeatureFlagsSettingsTests { // The source-selection contract is shared across tracers, so these cases mirror the // system-tests parametric suite (tests/parametric/test_ffe/test_configuration_sources.py). + // Where configuration would come from and whether Feature Flags run are asserted separately: + // the kill switch turns the product off without changing the source it would have used. [Theory] // Nothing configured: agentless is the default. - [InlineData(null, null, null, FeatureFlagsSource.Agentless)] + [InlineData(null, null, null, FeatureFlagsSource.Agentless, true)] // The stable kill switch wins over everything, including a legacy opt-in and an explicit source. - [InlineData("false", null, null, FeatureFlagsSource.Offline)] - [InlineData("false", null, "true", FeatureFlagsSource.Offline)] - [InlineData("false", "agentless", null, FeatureFlagsSource.Offline)] - [InlineData("false", "remote_config", null, FeatureFlagsSource.Offline)] + [InlineData("false", null, null, FeatureFlagsSource.Agentless, false)] + [InlineData("false", null, "true", FeatureFlagsSource.Agentless, false)] + [InlineData("false", "agentless", null, FeatureFlagsSource.Agentless, false)] + [InlineData("false", "remote_config", null, FeatureFlagsSource.RemoteConfig, false)] // Enabling explicitly does not imply the historical Remote Configuration source. - [InlineData("true", null, null, FeatureFlagsSource.Agentless)] + [InlineData("true", null, null, FeatureFlagsSource.Agentless, true)] // An explicit source wins over the legacy key, in both directions. - [InlineData(null, "agentless", "true", FeatureFlagsSource.Agentless)] - [InlineData(null, "remote_config", "false", FeatureFlagsSource.RemoteConfig)] + [InlineData(null, "agentless", "true", FeatureFlagsSource.Agentless, true)] + [InlineData(null, "remote_config", "false", FeatureFlagsSource.RemoteConfig, true)] // The legacy key grandfathers existing adopters, who opted in when RC was the only source. - [InlineData(null, null, "true", FeatureFlagsSource.RemoteConfig)] - [InlineData(null, null, "false", FeatureFlagsSource.Offline)] + [InlineData(null, null, "true", FeatureFlagsSource.RemoteConfig, true)] + [InlineData(null, null, "false", FeatureFlagsSource.Offline, false)] // An explicit new-key value takes precedence over the legacy key, so a stale legacy disable // does not silently keep Feature Flags off during migration. - [InlineData("true", null, "false", FeatureFlagsSource.Agentless)] - [InlineData("true", null, "true", FeatureFlagsSource.Agentless)] + [InlineData("true", null, "false", FeatureFlagsSource.Agentless, true)] + [InlineData("true", null, "true", FeatureFlagsSource.Agentless, true)] // An unrecognised source fails closed, and does so before the legacy key is considered: starting // billed delivery off a typo is worse than delivering nothing. Java and JS resolve it the same // way, and the system-tests parametric suite asserts no request is made. - [InlineData(null, "invalid", null, FeatureFlagsSource.Offline)] - [InlineData(null, "invalid", "true", FeatureFlagsSource.Offline)] - // "offline" is a reserved, recognised fail-closed sentinel (not an unrecognised value). - [InlineData(null, "offline", null, FeatureFlagsSource.Offline)] - [InlineData(null, "offline", "true", FeatureFlagsSource.Offline)] - public void ResolvesSource(string? enabled, string? source, string? legacyEnabled, object expected) + [InlineData(null, "invalid", null, FeatureFlagsSource.Offline, false)] + [InlineData(null, "invalid", "true", FeatureFlagsSource.Offline, false)] + // "offline" is a recognised source that delivers nothing, so there is nothing to run yet. + [InlineData(null, "offline", null, FeatureFlagsSource.Offline, false)] + [InlineData(null, "offline", "true", FeatureFlagsSource.Offline, false)] + public void ResolvesSource(string? enabled, string? source, string? legacyEnabled, object expected, bool expectedEnabled) { - var expectedSource = (FeatureFlagsSource)expected; var settings = CreateSettings(enabled, source, legacyEnabled); - settings.Source.Should().Be(expectedSource); - settings.Enabled.Should().Be(expectedSource != FeatureFlagsSource.Offline); + settings.Source.Should().Be((FeatureFlagsSource)expected); + settings.Enabled.Should().Be(expectedEnabled); } [Theory] From 42aaffc6297f1d2b2320805445acabfc85693d9d Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 14:53:55 +0300 Subject: [PATCH 45/62] feat(feature-flags): add UfcConfigurationParser for JSON:API envelope validation --- .../Agentless/UfcConfigurationParser.cs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs new file mode 100644 index 000000000000..58bbfe2691e9 --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -0,0 +1,90 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using Datadog.Trace.Vendors.Newtonsoft.Json; +using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; + +namespace Datadog.Trace.FeatureFlags.Agentless; + +/// +/// Reads the JSON:API envelope returned by the agentless endpoint. +/// +internal static class UfcConfigurationParser +{ + private const string ResourceType = "universal-flag-configuration"; + + /// + /// Validates a JSON:API Universal Flag Configuration response and returns data.attributes, + /// which is the document the evaluator consumes. A raw UFC document is rejected, including from + /// a custom endpoint, so that every source agrees on one wire format. + /// + /// The response body. + /// The parsed configuration. + /// Why the payload was rejected. + /// true when the payload matches the contract. + public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfiguration? configuration, out string? error) + { + configuration = null; + error = null; + + JToken payload; + try + { + using var stringReader = new StringReader(body ?? string.Empty); + + // Timestamps stay strings: the model carries createdAt verbatim, and letting Newtonsoft + // turn it into a date would also make the type check below fail. + using var jsonReader = new JsonTextReader(stringReader) { DateParseHandling = DateParseHandling.None }; + payload = JToken.ReadFrom(jsonReader); + } + catch (Exception) + { + error = "Malformed UFC payload"; + return false; + } + + if (payload is not JObject + || payload["data"] is not JObject data + || data["type"]?.Value() != ResourceType) + { + error = "Expected a JSON:API Universal Flag Configuration resource"; + return false; + } + + if (data["attributes"] is not JObject attributes + || attributes["format"]?.Type != JTokenType.String + || attributes["createdAt"]?.Type != JTokenType.String + || attributes["environment"] is not JObject environment + || environment["name"]?.Type != JTokenType.String + || attributes["flags"] is not JObject) + { + error = "Expected a Universal Flag Configuration v1 object"; + return false; + } + + try + { + configuration = attributes.ToObject(); + } + catch (Exception) + { + configuration = null; + } + + if (configuration is null) + { + error = "Expected a Universal Flag Configuration v1 object"; + return false; + } + + return true; + } +} From 5c818628c5affa4982717d16ac4386932836fcd1 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 15:01:50 +0300 Subject: [PATCH 46/62] test(feature-flags): add UfcConfigurationParserTests for JSON:API envelope validation --- .../Agentless/AgentlessConfigurationSource.cs | 399 ++++++++++++++++++ .../UfcConfigurationParserTests.cs | 108 +++++ 2 files changed, 507 insertions(+) create mode 100644 tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs create mode 100644 tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs new file mode 100644 index 000000000000..6d23a7045c31 --- /dev/null +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -0,0 +1,399 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Agent.Transports; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using Datadog.Trace.Headers; +using Datadog.Trace.Logging; +using Datadog.Trace.Telemetry; +using Datadog.Trace.Util; + +namespace Datadog.Trace.FeatureFlags.Agentless; + +/// +/// Polls the agentless endpoint for flag configuration. Polling is billable, so it is only +/// started once application code has activated the provider. +/// +internal sealed class AgentlessConfigurationSource : IDisposable +{ + private const int MaxAttempts = 3; + private const double RetryJitter = 0.2; + + private static readonly TimeSpan FirstRetryMin = TimeSpan.FromSeconds(2); + private static readonly TimeSpan FirstRetryMax = TimeSpan.FromSeconds(10); + private static readonly TimeSpan SecondRetryMin = TimeSpan.FromSeconds(5); + private static readonly TimeSpan SecondRetryMax = TimeSpan.FromSeconds(30); + + // A jittered retry delay never drops below this, so a short poll interval cannot turn + // retries into a burst against the endpoint. + private static readonly TimeSpan MinRetryDelay = TimeSpan.FromSeconds(1); + + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AgentlessConfigurationSource)); + + private readonly IApiRequestFactory _requestFactory; + private readonly Uri _endpoint; + private readonly TimeSpan _pollInterval; + private readonly Func _applyConfiguration; + private readonly Func _waitAsync; + private readonly CancellationTokenSource _shutdown = new(); + private readonly Random _random = new(); + + // Only ever touched from the poll loop. + private readonly HashSet _loggedFailureCategories = new(); + private bool _malformedPayloadLogged; + private bool _applyFailureLogged; + private string? _etag; + + private int _started; + + internal AgentlessConfigurationSource( + Uri endpoint, + IApiRequestFactory requestFactory, + TimeSpan pollInterval, + Func applyConfiguration, + Func? waitAsync = null) + { + _endpoint = endpoint; + _requestFactory = requestFactory; + _pollInterval = pollInterval; + _applyConfiguration = applyConfiguration; + _waitAsync = waitAsync ?? Task.Delay; + } + + /// + /// Creates the source, or returns null when it cannot be operated: a base URL that is + /// not a URL, or the managed endpoint without an API key. Polling anyway would only produce + /// failures every interval. + /// + public static AgentlessConfigurationSource? Create(FeatureFlagsSettings settings, Func applyConfiguration) + { + if (!AgentlessEndpoint.TryCreate(settings.Site, settings.Env, settings.AgentlessBaseUrl, out var endpoint, out var error)) + { + Log.Error("Feature Flags agentless source is unavailable: {Error}", error); + return null; + } + + if (endpoint.IsManaged && StringUtil.IsNullOrEmpty(settings.ApiKey)) + { + Log.Error("Feature Flags agentless source requires an API key. Set DD_API_KEY, or point DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL at an endpoint of your own."); + return null; + } + + return new AgentlessConfigurationSource( + endpoint.Uri, + CreateRequestFactory(endpoint, settings), + settings.PollInterval, + applyConfiguration); + } + + /// + /// Starts polling. Idempotent. + /// + public void Start() + { + if (Interlocked.CompareExchange(ref _started, 1, 0) != 0) + { + return; + } + + // Deliberately not wrapped in Task.Run: this is called from provider initialization, which + // is waiting for the first configuration, so the first request should go out on the calling + // thread rather than queue behind whatever else is on the thread pool. + _ = RunAsync().ContinueWith(t => Log.Error(t.Exception, "Feature Flags agentless poll loop failed"), TaskContinuationOptions.OnlyOnFaulted); + } + + /// + /// Runs a single poll, including its in-tick retries. + /// + internal async Task PollAsync() + { + var result = default(PollResult); + + for (var attempt = 1; attempt <= MaxAttempts; attempt++) + { + result = await RequestAsync().ConfigureAwait(false); + + if (_shutdown.IsCancellationRequested) + { + // A shutdown mid-poll leaves the response unusable for state transitions: keep + // last-known-good and the current ETag. + return; + } + + if (!IsRetryable(result)) + { + break; + } + + if (attempt == MaxAttempts) + { + // Every attempt failed in a retryable way. Last-known-good stays in place. + WarnFailure(result, MaxAttempts); + return; + } + + await WaitAsync(RetryDelay(attempt)).ConfigureAwait(false); + + if (_shutdown.IsCancellationRequested) + { + return; + } + } + + if (_shutdown.IsCancellationRequested) + { + // A shutdown during the final attempt leaves the response unusable for state + // transitions: keep last-known-good and the current ETag. + return; + } + + await ApplyAsync(result).ConfigureAwait(false); + } + + public void Dispose() + { + // The request in flight is bounded by the request timeout, and the loop is never joined, + // so a shutdown does not wait for it. A poll that completes after disposal is prevented + // from applying its result by the shutdown check in PollAsync. + try + { + _shutdown.Cancel(); + } + catch (Exception ex) + { + Log.Debug(ex, "Error cancelling the Feature Flags agentless poll loop"); + } + } + + // The concrete type is returned rather than the interface because CA1859 asks for it on a + // private member, which is also why the signature varies by target framework. +#if NETCOREAPP + private static HttpClientRequestFactory CreateRequestFactory(AgentlessEndpoint endpoint, FeatureFlagsSettings settings) +#else + private static ApiWebRequestFactory CreateRequestFactory(AgentlessEndpoint endpoint, FeatureFlagsSettings settings) +#endif + { + var headers = new List> + { + // The endpoint serves gzip, and neither transport decompresses for us. + new("Accept-Encoding", "gzip"), + new(TelemetryConstants.ClientLibraryLanguageHeader, TracerConstants.Language), + new(TelemetryConstants.ClientLibraryVersionHeader, TracerConstants.ThreePartVersion), + + // Without this the poll is itself instrumented, producing a span per poll and letting + // auto-instrumentation recurse through the poller's own client. + new(HttpHeaderNames.TracingEnabled, "false"), + }; + + if (endpoint.IsManaged) + { + // A custom endpoint is left to report its own authentication failure rather than + // having the Datadog credential sent to it. + headers.Add(new(TelemetryConstants.ApiKeyHeader, settings.ApiKey!)); + } + +#if NETCOREAPP + return new HttpClientRequestFactory(endpoint.Uri, headers.ToArray(), timeout: settings.RequestTimeout); +#else + return new ApiWebRequestFactory(endpoint.Uri, headers.ToArray(), timeout: settings.RequestTimeout); +#endif + } + + private static bool IsRetryable(in PollResult result) + => result.StatusCode is not { } status || status is 408 or 429 or (>= 500 and <= 599); + + private async Task RunAsync() + { + Log.Debug("AgentlessConfigurationSource::RunAsync -> Enter"); + + while (!_shutdown.IsCancellationRequested) + { + try + { + await PollAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + Log.Debug(ex, "Feature Flags agentless poll failed unexpectedly"); + } + + // Fixed delay after completion, so polls never overlap. + await WaitAsync(_pollInterval).ConfigureAwait(false); + } + + Log.Debug("AgentlessConfigurationSource::RunAsync -> Exit"); + } + + private async Task WaitAsync(TimeSpan delay) + { + try + { + await _waitAsync(delay, _shutdown.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Shutting down + } + } + + private TimeSpan RetryDelay(int attempt) + { + var seconds = attempt == 1 + ? Clamp(_pollInterval.TotalSeconds / 6, FirstRetryMin, FirstRetryMax) + : Clamp(_pollInterval.TotalSeconds / 3, SecondRetryMin, SecondRetryMax); + + double jitter; + lock (_random) + { + jitter = 1 - RetryJitter + (_random.NextDouble() * RetryJitter * 2); + } + + return TimeSpan.FromSeconds(Math.Max(MinRetryDelay.TotalSeconds, seconds * jitter)); + + static double Clamp(double value, TimeSpan minimum, TimeSpan maximum) + => Math.Max(minimum.TotalSeconds, Math.Min(maximum.TotalSeconds, value)); + } + + private async Task RequestAsync() + { + try + { + var request = _requestFactory.Create(_endpoint); + if (_etag is { } etag) + { + request.AddHeader("If-None-Match", etag); + } + + using var response = await request.GetAsync().ConfigureAwait(false); + + // Only a 200 carries configuration; other bodies are never decoded as one. + var body = response.StatusCode == 200 ? await ReadBodyAsync(response).ConfigureAwait(false) : null; + return new PollResult(response.StatusCode, response.GetHeader("ETag"), body, error: null); + } + catch (Exception ex) + { + return new PollResult(statusCode: null, etag: null, body: null, error: ex); + } + } + + private async Task ReadBodyAsync(IApiResponse response) + { + var stream = await response.GetStreamAsync().ConfigureAwait(false); + GZipStream? decompressed = null; + + try + { + if (response.GetContentEncodingType() == ContentEncodingType.GZip) + { + decompressed = new GZipStream(stream, CompressionMode.Decompress); + } + + using var reader = new StreamReader(decompressed ?? stream, response.GetCharsetEncoding()); + return await reader.ReadToEndAsync().ConfigureAwait(false); + } + finally + { + decompressed?.Dispose(); + } + } + + private Task ApplyAsync(PollResult result) + { + switch (result.StatusCode) + { + case 304: + // Nothing changed, and the ETag stays as it is. + return Task.CompletedTask; + case 401 or 403: + WarnFailure(result, attempts: 1); + return Task.CompletedTask; + case not 200: + WarnFailure(result, attempts: 1); + return Task.CompletedTask; + } + + if (!UfcConfigurationParser.TryParse(result.Body, out var configuration, out var error)) + { + if (!_malformedPayloadLogged) + { + _malformedPayloadLogged = true; + Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", error); + } + + return Task.CompletedTask; + } + + if (!_applyConfiguration(configuration)) + { + if (!_applyFailureLogged) + { + _applyFailureLogged = true; + Log.Warning("Feature Flags agentless configuration could not be applied"); + } + + return Task.CompletedTask; + } + + // The ETag advances only once parsing and applying have both succeeded. Advancing on + // receipt would acknowledge a payload that was never applied, and every later poll would + // answer 304, pinning the process to stale configuration with no way back. + var newEtag = result.ETag?.Trim(); + _etag = StringUtil.IsNullOrEmpty(newEtag) ? null : newEtag; + + return Task.CompletedTask; + } + + /// + /// Warns once per failure category. A dead endpoint would otherwise produce a warning every + /// poll interval, indefinitely. + /// + private void WarnFailure(in PollResult result, int attempts) + { + var category = result.StatusCode switch + { + 401 or 403 => "authentication", + not null => "http", + _ => "request", + }; + + if (!_loggedFailureCategories.Add(category)) + { + return; + } + + switch (result.StatusCode) + { + case 401 or 403: + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); + break; + case not null: + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); + break; + default: + Log.Warning(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); + break; + } + } + + internal readonly struct PollResult(int? statusCode, string? etag, string? body, Exception? error) + { + public int? StatusCode { get; } = statusCode; + + public string? ETag { get; } = etag; + + public string? Body { get; } = body; + + public Exception? Error { get; } = error; + } +} diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs new file mode 100644 index 000000000000..14fc51ab85a7 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs @@ -0,0 +1,108 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System.Collections.Generic; +using Datadog.Trace.FeatureFlags.Agentless; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.FeatureFlags; + +public class UfcConfigurationParserTests +{ + private const string ValidEnvelope = """ + { "data": { "type": "universal-flag-configuration", + "attributes": { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, "flags": {} } } } + """; + + private const string Attributes = """ + { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, "flags": {} } + """; + + [Fact] + public void ParsesValidEnvelope() + { + UfcConfigurationParser.TryParse(ValidEnvelope, out var configuration, out var error) + .Should().BeTrue(); + + error.Should().BeNull(); + configuration.Should().NotBeNull(); + configuration!.Environment!.Name.Should().Be("production"); + configuration.Flags.Should().BeEmpty(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("not json")] + [InlineData("{ \"data\": ")] + public void RejectsMalformedJson(string? body) + { + UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + + configuration.Should().BeNull(); + error.Should().Be("Malformed UFC payload"); + } + + [Theory] + // A raw UFC document is rejected too, so every source agrees on one wire format. + [InlineData(Attributes)] + // Wrong resource type + [InlineData("""{ "data": { "type": "wrong-type", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] + // Missing data + [InlineData("""{ "meta": {} }""")] + // data is not an object + [InlineData("""{ "data": "string" }""")] + public void RejectsInvalidEnvelope(string body) + { + UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + + configuration.Should().BeNull(); + error.Should().Be("Expected a JSON:API Universal Flag Configuration resource"); + } + + [Theory] + // Missing format + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] + // Missing createdAt + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "environment": { "name": "prod" }, "flags": {} } } }""")] + // Missing environment + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "flags": {} } } }""")] + // environment.name is not a string + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": 123 }, "flags": {} } } }""")] + // Missing flags + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" } } } }""")] + // flags is not an object + [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": [] } } }""")] + public void RejectsInvalidAttributes(string body) + { + UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + + configuration.Should().BeNull(); + error.Should().Be("Expected a Universal Flag Configuration v1 object"); + } + + [Fact] + public void ParsesFlagsFromEnvelope() + { + var body = """ + { "data": { "type": "universal-flag-configuration", + "attributes": { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, + "flags": { "test-flag": { "key": "test-flag", "enabled": true, "variationType": "BOOLEAN" } } } } } + """; + + UfcConfigurationParser.TryParse(body, out var configuration, out _).Should().BeTrue(); + + configuration!.Flags.Should().NotBeNull(); + configuration!.Flags!.Should().ContainKey("test-flag"); + configuration!.Flags!["test-flag"].Enabled.Should().BeTrue(); + } +} From c7a0d5fee7d3e12b7f07e21e32025d619178db46 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 13 Aug 2026 15:47:50 +0300 Subject: [PATCH 47/62] test(feature-flags): add AgentlessConfigurationSourceTests for poller retry, ETag, gzip, and shutdown --- .../AgentlessConfigurationSourceTests.cs | 284 ++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs new file mode 100644 index 000000000000..935c778b96f4 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -0,0 +1,284 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.FeatureFlags.Agentless; +using Datadog.Trace.FeatureFlags.Rcm.Model; +using Datadog.Trace.TestHelpers.TransportHelpers; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.FeatureFlags; + +public class AgentlessConfigurationSourceTests +{ + private const string Body = """ + { "data": { "type": "universal-flag-configuration", + "attributes": { "format": "SERVER", "createdAt": "2025-01-01T00:00:00Z", + "environment": { "name": "production" }, "flags": {} } } } + """; + + private static readonly Uri Endpoint = new("https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server"); + + [Fact] + public async Task AppliesConfigurationFromA200() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new TestApiRequest(uri, responseContent: Body)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().ContainSingle(); + applied[0].Environment!.Name.Should().Be("production"); + factory.RequestsSent.Should().ContainSingle(); + } + + [Fact] + public async Task SendsTheEtagOfTheLastAppliedConfiguration() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, responseContent: Body, responseHeaders: new() { { "ETag", "\"ufc-v1\"" } })); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + applied.Should().ContainSingle(); + + // Second poll should send If-None-Match + await source.PollAsync(); + factory.RequestsSent.Should().HaveCount(2); + factory.RequestsSent[1].ExtraHeaders.Should().ContainKey("If-None-Match"); + factory.RequestsSent[1].ExtraHeaders["If-None-Match"].Should().Be("\"ufc-v1\""); + } + + [Fact] + public async Task DoesNotApplyOn304() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 304, responseContent: "{}")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyOn401() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 401, responseContent: "Unauthorized")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyOnMalformedPayload() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 200, responseContent: "not json")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyAfterDisposal() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new TestApiRequest(uri, responseContent: Body)); + var source = CreateSource(factory, applied); + + // A shutdown mid-poll leaves the response unusable for a state transition. + source.Dispose(); + await source.PollAsync(); + + factory.RequestsSent.Should().ContainSingle(); + applied.Should().BeEmpty(); + } + + [Fact] + public async Task DoesNotApplyWhenDisposedAfterRequestSucceeds() + { + var applied = new List(); + AgentlessConfigurationSource? sourceRef = null; + var factory = new TestRequestFactory(uri => + { + var request = new DisposingApiRequest(uri, Body); + request.Source = sourceRef; + return request; + }); + using var source = CreateSource(factory, applied); + sourceRef = source; + + await source.PollAsync(); + + applied.Should().BeEmpty(); + } + + [Fact] + public async Task RetriesOn500ThenAppliesOnSuccess() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error"), + uri => new TestApiRequest(uri, responseContent: Body)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().ContainSingle(); + factory.RequestsSent.Should().HaveCount(2); + } + + [Fact] + public async Task RetriesUpToMaxAttemptsOn500() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error"), + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error"), + uri => new TestApiRequest(uri, statusCode: 500, responseContent: "error")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + factory.RequestsSent.Should().HaveCount(3); + } + + [Fact] + public async Task DoesNotRetryOn400() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, statusCode: 400, responseContent: "bad request")); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + factory.RequestsSent.Should().ContainSingle(); + } + + [Fact] + public async Task HandlesGzipResponse() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new GzipApiRequest(uri, Body)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().ContainSingle(); + applied[0].Environment!.Name.Should().Be("production"); + } + + [Fact] + public async Task HandlesNetworkError() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new ThrowingApiRequest(uri), + uri => new ThrowingApiRequest(uri), + uri => new ThrowingApiRequest(uri)); + using var source = CreateSource(factory, applied); + + await source.PollAsync(); + + applied.Should().BeEmpty(); + factory.RequestsSent.Should().HaveCount(3); + } + + private static AgentlessConfigurationSource CreateSource(TestRequestFactory factory, List applied) + => new( + Endpoint, + factory, + TimeSpan.FromSeconds(30), + configuration => + { + applied.Add(configuration); + return true; + }, + NoWait); + + private static Task NoWait(TimeSpan delay, CancellationToken cancellationToken) => Task.CompletedTask; + + private class ThrowingApiRequest(Uri endpoint) : TestApiRequest(endpoint) + { + public override Task GetAsync() => throw new IOException("The connection was refused"); + } + + private class GzipApiRequest(Uri endpoint, string body) : TestApiRequest(endpoint) + { + public override Task GetAsync() => Task.FromResult(new GzipApiResponse(body)); + } + + private class GzipApiResponse(string body) : IApiResponse + { + public int StatusCode => 200; + + public long ContentLength => -1; + + public string? ContentTypeHeader => "application/json"; + + public string? ContentEncodingHeader => "gzip"; + + public void Dispose() + { + } + + public string? GetHeader(string headerName) => null; + + public Encoding GetCharsetEncoding() => Encoding.UTF8; + + public ContentEncodingType GetContentEncodingType() => ContentEncodingType.GZip; + + public Task GetStreamAsync() + { + var compressed = new MemoryStream(); + using (var gzip = new GZipStream(compressed, CompressionMode.Compress, leaveOpen: true)) + { + var bytes = Encoding.UTF8.GetBytes(body); + gzip.Write(bytes, 0, bytes.Length); + } + + compressed.Position = 0; + return Task.FromResult(compressed); + } + } + + private class DisposingApiRequest(Uri endpoint, string body) : TestApiRequest(endpoint, responseContent: body) + { + public AgentlessConfigurationSource? Source { get; set; } + + public override Task GetAsync() + { + var response = base.GetAsync(); + // Simulate a shutdown arriving after the request completes but before ApplyAsync. + Source?.Dispose(); + return response; + } + } +} From e084a93f01b63abfb6d88bc45b6efcd3ab0cb42b Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Aug 2026 15:01:41 +0300 Subject: [PATCH 48/62] fix(feature-flags): reject non-string data.type, log failures as errors --- .../Agentless/AgentlessConfigurationSource.cs | 8 ++++---- .../FeatureFlags/Agentless/UfcConfigurationParser.cs | 1 + .../FeatureFlags/UfcConfigurationParserTests.cs | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 6d23a7045c31..7ec342a4fc6d 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -225,7 +225,7 @@ private async Task RunAsync() } catch (Exception ex) { - Log.Debug(ex, "Feature Flags agentless poll failed unexpectedly"); + Log.Error(ex, "Feature Flags agentless poll failed unexpectedly"); } // Fixed delay after completion, so polls never overlap. @@ -375,13 +375,13 @@ private void WarnFailure(in PollResult result, int attempts) switch (result.StatusCode) { case 401 or 403: - Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); + Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); break; case not null: - Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); + Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); break; default: - Log.Warning(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); + Log.Error(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); break; } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs index 58bbfe2691e9..a078f5f4374d 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -53,6 +53,7 @@ public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfigur if (payload is not JObject || payload["data"] is not JObject data + || data["type"]?.Type != JTokenType.String || data["type"]?.Value() != ResourceType) { error = "Expected a JSON:API Universal Flag Configuration resource"; diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs index 14fc51ab85a7..b57c9568ebcd 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs @@ -60,6 +60,10 @@ public void RejectsMalformedJson(string? body) [InlineData("""{ "meta": {} }""")] // data is not an object [InlineData("""{ "data": "string" }""")] + // data.type is not a string (object) + [InlineData("""{ "data": { "type": { "nested": true }, "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] + // data.type is not a string (array) + [InlineData("""{ "data": { "type": [1, 2], "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] public void RejectsInvalidEnvelope(string body) { UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); From 1f8f03df5e3f38797f3f7cd434c3002bb7c150db Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 14 Aug 2026 15:15:13 +0300 Subject: [PATCH 49/62] fix(feature-flags): add explicit request timeout race for net461 async WebRequest --- .../Agentless/AgentlessConfigurationSource.cs | 25 +++++++++++++++++++ .../AgentlessConfigurationSourceTests.cs | 1 + 2 files changed, 26 insertions(+) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 7ec342a4fc6d..4f2658fda111 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -44,6 +44,7 @@ internal sealed class AgentlessConfigurationSource : IDisposable private readonly IApiRequestFactory _requestFactory; private readonly Uri _endpoint; private readonly TimeSpan _pollInterval; + private readonly TimeSpan _requestTimeout; private readonly Func _applyConfiguration; private readonly Func _waitAsync; private readonly CancellationTokenSource _shutdown = new(); @@ -61,12 +62,14 @@ internal AgentlessConfigurationSource( Uri endpoint, IApiRequestFactory requestFactory, TimeSpan pollInterval, + TimeSpan requestTimeout, Func applyConfiguration, Func? waitAsync = null) { _endpoint = endpoint; _requestFactory = requestFactory; _pollInterval = pollInterval; + _requestTimeout = requestTimeout; _applyConfiguration = applyConfiguration; _waitAsync = waitAsync ?? Task.Delay; } @@ -94,6 +97,7 @@ internal AgentlessConfigurationSource( endpoint.Uri, CreateRequestFactory(endpoint, settings), settings.PollInterval, + settings.RequestTimeout, applyConfiguration); } @@ -275,7 +279,28 @@ private async Task RequestAsync() request.AddHeader("If-None-Match", etag); } +#if NETCOREAPP + // HttpClient.Timeout applies to async calls, so no explicit race is needed. using var response = await request.GetAsync().ConfigureAwait(false); +#else + // HttpWebRequest.Timeout does not apply to async calls (GetResponseAsync), so we race + // the request against an explicit delay to bound the wait on net461/netstandard2.0. + var getTask = request.GetAsync(); + var timeoutTask = Task.Delay(_requestTimeout); + + if (await Task.WhenAny(getTask, timeoutTask).ConfigureAwait(false) == timeoutTask) + { + // The request is still in flight. Dispose the response when it eventually completes + // (success or fault) so the underlying connection is released. + _ = getTask.ContinueWith( + t => { try { using var r = t.Result; } catch { } }, + TaskContinuationOptions.None); + + return new PollResult(statusCode: null, etag: null, body: null, error: new TimeoutException($"Feature Flags agentless request timed out after {_requestTimeout.TotalSeconds}s")); + } + + using var response = await getTask.ConfigureAwait(false); +#endif // Only a 200 carries configuration; other bodies are never decoded as one. var body = response.StatusCode == 200 ? await ReadBodyAsync(response).ConfigureAwait(false) : null; diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index 935c778b96f4..e2e39a15acde 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -216,6 +216,7 @@ private static AgentlessConfigurationSource CreateSource(TestRequestFactory fact Endpoint, factory, TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), configuration => { applied.Add(configuration); From d3b307e5ff771a66ab5c86ceffcd69dc66c4e42d Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 20 Aug 2026 18:18:37 +0300 Subject: [PATCH 50/62] [FeatureFlags] Poll for the current environment instead of the one captured at startup --- .../Agentless/AgentlessConfigurationSource.cs | 66 ++++++++++++++++--- .../AgentlessConfigurationSourceTests.cs | 47 ++++++++++++- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 4f2658fda111..a0bdd1279537 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -13,6 +13,7 @@ using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.Agent.Transports; +using Datadog.Trace.Configuration; using Datadog.Trace.FeatureFlags.Rcm.Model; using Datadog.Trace.Headers; using Datadog.Trace.Logging; @@ -42,7 +43,7 @@ internal sealed class AgentlessConfigurationSource : IDisposable private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AgentlessConfigurationSource)); private readonly IApiRequestFactory _requestFactory; - private readonly Uri _endpoint; + private readonly AgentlessEndpoint _endpoint; private readonly TimeSpan _pollInterval; private readonly TimeSpan _requestTimeout; private readonly Func _applyConfiguration; @@ -52,18 +53,26 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); + + // Written by the settings-change callback, read by the poll loop. + private string? _environment; + private IDisposable? _environmentSubscription; + + // Only ever touched from the poll loop. private bool _malformedPayloadLogged; private bool _applyFailureLogged; private string? _etag; + private Uri? _etagUri; private int _started; internal AgentlessConfigurationSource( - Uri endpoint, + AgentlessEndpoint endpoint, IApiRequestFactory requestFactory, TimeSpan pollInterval, TimeSpan requestTimeout, Func applyConfiguration, + string? environment = null, Func? waitAsync = null) { _endpoint = endpoint; @@ -71,6 +80,7 @@ internal AgentlessConfigurationSource( _pollInterval = pollInterval; _requestTimeout = requestTimeout; _applyConfiguration = applyConfiguration; + _environment = environment; _waitAsync = waitAsync ?? Task.Delay; } @@ -79,9 +89,12 @@ internal AgentlessConfigurationSource( /// not a URL, or the managed endpoint without an API key. Polling anyway would only produce /// failures every interval. /// - public static AgentlessConfigurationSource? Create(FeatureFlagsSettings settings, Func applyConfiguration) + public static AgentlessConfigurationSource? Create( + FeatureFlagsSettings settings, + TracerSettings.SettingsManager manager, + Func applyConfiguration) { - if (!AgentlessEndpoint.TryCreate(settings.Site, settings.Env, settings.AgentlessBaseUrl, out var endpoint, out var error)) + if (!AgentlessEndpoint.TryCreate(settings.Site, settings.AgentlessBaseUrl, out var endpoint, out var error)) { Log.Error("Feature Flags agentless source is unavailable: {Error}", error); return null; @@ -93,14 +106,34 @@ internal AgentlessConfigurationSource( return null; } - return new AgentlessConfigurationSource( - endpoint.Uri, + var source = new AgentlessConfigurationSource( + endpoint, CreateRequestFactory(endpoint, settings), settings.PollInterval, settings.RequestTimeout, - applyConfiguration); + applyConfiguration, + manager.InitialMutableSettings.Environment); + + // The environment is tracked rather than captured: customers can change it in code while + // the application runs, and flags are targeted per environment. Only the value is stored + // here, so that the poll loop stays the only thing that touches the request state. + source._environmentSubscription = manager.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + source.UpdateEnvironment(mutable.Environment); + } + }); + + return source; } + /// + /// Records the environment to request configuration for. Applied by the poll loop on its next + /// request, so a change never disturbs a request already in flight. + /// + internal void UpdateEnvironment(string? environment) => Volatile.Write(ref _environment, environment); + /// /// Starts polling. Idempotent. /// @@ -172,6 +205,7 @@ public void Dispose() // from applying its result by the shutdown check in PollAsync. try { + _environmentSubscription?.Dispose(); _shutdown.Cancel(); } catch (Exception ex) @@ -207,6 +241,8 @@ private static ApiWebRequestFactory CreateRequestFactory(AgentlessEndpoint endpo headers.Add(new(TelemetryConstants.ApiKeyHeader, settings.ApiKey!)); } + // The endpoint is only the factory's default: both transports honour the URI passed to + // Create, which is what carries the current environment. #if NETCOREAPP return new HttpClientRequestFactory(endpoint.Uri, headers.ToArray(), timeout: settings.RequestTimeout); #else @@ -273,7 +309,21 @@ private async Task RequestAsync() { try { - var request = _requestFactory.Create(_endpoint); + // The environment is applied per request rather than baked into the endpoint, because + // it can be changed in code after startup. + var uri = _endpoint.BuildRequestUri(Volatile.Read(ref _environment)); + + // An ETag only identifies the configuration served for the URI it came from. Sending it + // against a different environment would earn a 304 and pin the process to the previous + // environment's flags, with no way back. + if (_etagUri is not null && uri != _etagUri) + { + _etag = null; + } + + _etagUri = uri; + + var request = _requestFactory.Create(uri); if (_etag is { } etag) { request.AddHeader("If-None-Match", etag); diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index e2e39a15acde..a807554c0cc7 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -30,7 +30,7 @@ public class AgentlessConfigurationSourceTests "environment": { "name": "production" }, "flags": {} } } } """; - private static readonly Uri Endpoint = new("https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server"); + private const string EndpointUrl = "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server"; [Fact] public async Task AppliesConfigurationFromA200() @@ -64,6 +64,37 @@ public async Task SendsTheEtagOfTheLastAppliedConfiguration() factory.RequestsSent[1].ExtraHeaders["If-None-Match"].Should().Be("\"ufc-v1\""); } + [Fact] + public async Task RequestsTheConfiguredEnvironment() + { + var applied = new List(); + var factory = new TestRequestFactory(uri => new TestApiRequest(uri, responseContent: Body)); + using var source = CreateSource(factory, applied, environment: "production"); + + await source.PollAsync(); + + factory.RequestsSent[0].Endpoint.Should().Be(new Uri(EndpointUrl + "?dd_env=production")); + } + + [Fact] + public async Task DropsTheEtagWhenTheEnvironmentChanges() + { + var applied = new List(); + var factory = new TestRequestFactory( + uri => new TestApiRequest(uri, responseContent: Body, responseHeaders: new() { { "ETag", "\"ufc-v1\"" } })); + using var source = CreateSource(factory, applied, environment: "production"); + + await source.PollAsync(); + + // The ETag identifies production's configuration, so it must not be sent against staging: + // a 304 would pin the process to production's flags with no way back. + source.UpdateEnvironment("staging"); + await source.PollAsync(); + + factory.RequestsSent[1].Endpoint.Should().Be(new Uri(EndpointUrl + "?dd_env=staging")); + factory.RequestsSent[1].ExtraHeaders.Should().NotContainKey("If-None-Match"); + } + [Fact] public async Task DoesNotApplyOn304() { @@ -211,9 +242,12 @@ public async Task HandlesNetworkError() factory.RequestsSent.Should().HaveCount(3); } - private static AgentlessConfigurationSource CreateSource(TestRequestFactory factory, List applied) + private static AgentlessConfigurationSource CreateSource( + TestRequestFactory factory, + List applied, + string? environment = null) => new( - Endpoint, + CreateEndpoint(), factory, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(5), @@ -222,8 +256,15 @@ private static AgentlessConfigurationSource CreateSource(TestRequestFactory fact applied.Add(configuration); return true; }, + environment, NoWait); + private static AgentlessEndpoint CreateEndpoint() + { + AgentlessEndpoint.TryCreate("datadoghq.com", baseUrl: null, out var endpoint, out _).Should().BeTrue(); + return endpoint ?? throw new InvalidOperationException("TryCreate reported success without producing an endpoint."); + } + private static Task NoWait(TimeSpan delay, CancellationToken cancellationToken) => Task.CompletedTask; private class ThrowingApiRequest(Uri endpoint) : TestApiRequest(endpoint) From eae179d8abf478ba0821fd97cdd96ecb5702339e Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:30:42 +0300 Subject: [PATCH 51/62] [FeatureFlags] Signal agentless poller shutdown with a TaskCompletionSource --- .../Agentless/AgentlessConfigurationSource.cs | 35 +++++++++---------- .../AgentlessConfigurationSourceTests.cs | 3 +- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index a0bdd1279537..7058d0d01ed4 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -47,8 +47,11 @@ internal sealed class AgentlessConfigurationSource : IDisposable private readonly TimeSpan _pollInterval; private readonly TimeSpan _requestTimeout; private readonly Func _applyConfiguration; - private readonly Func _waitAsync; - private readonly CancellationTokenSource _shutdown = new(); + private readonly Func _waitAsync; + + // Not a CancellationTokenSource: cancellation throws, and an exception on the shutdown path can + // crash the runtime, so shutdown is signalled by completing a task instead. + private readonly TaskCompletionSource _shutdown = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly Random _random = new(); // Only ever touched from the poll loop. @@ -73,7 +76,7 @@ internal AgentlessConfigurationSource( TimeSpan requestTimeout, Func applyConfiguration, string? environment = null, - Func? waitAsync = null) + Func? waitAsync = null) { _endpoint = endpoint; _requestFactory = requestFactory; @@ -161,7 +164,7 @@ internal async Task PollAsync() { result = await RequestAsync().ConfigureAwait(false); - if (_shutdown.IsCancellationRequested) + if (_shutdown.Task.IsCompleted) { // A shutdown mid-poll leaves the response unusable for state transitions: keep // last-known-good and the current ETag. @@ -182,13 +185,13 @@ internal async Task PollAsync() await WaitAsync(RetryDelay(attempt)).ConfigureAwait(false); - if (_shutdown.IsCancellationRequested) + if (_shutdown.Task.IsCompleted) { return; } } - if (_shutdown.IsCancellationRequested) + if (_shutdown.Task.IsCompleted) { // A shutdown during the final attempt leaves the response unusable for state // transitions: keep last-known-good and the current ETag. @@ -203,14 +206,15 @@ public void Dispose() // The request in flight is bounded by the request timeout, and the loop is never joined, // so a shutdown does not wait for it. A poll that completes after disposal is prevented // from applying its result by the shutdown check in PollAsync. + _shutdown.TrySetResult(true); + try { _environmentSubscription?.Dispose(); - _shutdown.Cancel(); } catch (Exception ex) { - Log.Debug(ex, "Error cancelling the Feature Flags agentless poll loop"); + Log.Debug(ex, "Error unsubscribing the Feature Flags agentless poll loop from settings changes"); } } @@ -257,7 +261,7 @@ private async Task RunAsync() { Log.Debug("AgentlessConfigurationSource::RunAsync -> Enter"); - while (!_shutdown.IsCancellationRequested) + while (!_shutdown.Task.IsCompleted) { try { @@ -275,17 +279,10 @@ private async Task RunAsync() Log.Debug("AgentlessConfigurationSource::RunAsync -> Exit"); } + // A shutdown ends the wait early. The delay itself is left to expire on its own: it holds no + // thread, and the loop has already exited by the time it does. private async Task WaitAsync(TimeSpan delay) - { - try - { - await _waitAsync(delay, _shutdown.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - // Shutting down - } - } + => await Task.WhenAny(_waitAsync(delay), _shutdown.Task).ConfigureAwait(false); private TimeSpan RetryDelay(int attempt) { diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index a807554c0cc7..276f28b37251 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -11,7 +11,6 @@ using System.IO.Compression; using System.Linq; using System.Text; -using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.FeatureFlags.Agentless; @@ -265,7 +264,7 @@ private static AgentlessEndpoint CreateEndpoint() return endpoint ?? throw new InvalidOperationException("TryCreate reported success without producing an endpoint."); } - private static Task NoWait(TimeSpan delay, CancellationToken cancellationToken) => Task.CompletedTask; + private static Task NoWait(TimeSpan delay) => Task.CompletedTask; private class ThrowingApiRequest(Uri endpoint) : TestApiRequest(endpoint) { From f21deeb6b8c635b46ce606073f97e6136dd555cd Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:39:32 +0300 Subject: [PATCH 52/62] [FeatureFlags] Use ThreadSafeRandom.Shared for agentless retry jitter --- .../FeatureFlags/Agentless/AgentlessConfigurationSource.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 7058d0d01ed4..b2e5f47bd662 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -52,7 +52,6 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Not a CancellationTokenSource: cancellation throws, and an exception on the shutdown path can // crash the runtime, so shutdown is signalled by completing a task instead. private readonly TaskCompletionSource _shutdown = new(TaskCreationOptions.RunContinuationsAsynchronously); - private readonly Random _random = new(); // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); @@ -290,11 +289,7 @@ private TimeSpan RetryDelay(int attempt) ? Clamp(_pollInterval.TotalSeconds / 6, FirstRetryMin, FirstRetryMax) : Clamp(_pollInterval.TotalSeconds / 3, SecondRetryMin, SecondRetryMax); - double jitter; - lock (_random) - { - jitter = 1 - RetryJitter + (_random.NextDouble() * RetryJitter * 2); - } + var jitter = 1 - RetryJitter + (ThreadSafeRandom.Shared.NextDouble() * RetryJitter * 2); return TimeSpan.FromSeconds(Math.Max(MinRetryDelay.TotalSeconds, seconds * jitter)); From 8aeefb2633e9ac3eeec2846fec454da30b4e56c3 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:42:08 +0300 Subject: [PATCH 53/62] [FeatureFlags] Log agentless delivery failures as warnings --- .../FeatureFlags/Agentless/AgentlessConfigurationSource.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index b2e5f47bd662..bcd0d5dafddf 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -442,13 +442,13 @@ private void WarnFailure(in PollResult result, int attempts) switch (result.StatusCode) { case 401 or 403: - Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode}; verify endpoint authentication", result.StatusCode!.Value); break; case not null: - Log.Error("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); + Log.Warning("Feature Flags agentless endpoint returned HTTP {StatusCode} after {Attempts} attempts", result.StatusCode.Value, attempts); break; default: - Log.Error(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); + Log.Warning(result.Error, "Feature Flags agentless request failed after {Attempts} attempts", attempts); break; } } From 3640e9d284302a059e9b93f04123a74a3ca610ec Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:48:41 +0300 Subject: [PATCH 54/62] [FeatureFlags] Fix stream disposal when reading the agentless response body --- .../Agentless/AgentlessConfigurationSource.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index bcd0d5dafddf..68e358202fa5 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -356,23 +356,23 @@ private async Task RequestAsync() private async Task ReadBodyAsync(IApiResponse response) { - var stream = await response.GetStreamAsync().ConfigureAwait(false); - GZipStream? decompressed = null; - - try - { - if (response.GetContentEncodingType() == ContentEncodingType.GZip) - { - decompressed = new GZipStream(stream, CompressionMode.Decompress); - } - - using var reader = new StreamReader(decompressed ?? stream, response.GetCharsetEncoding()); - return await reader.ReadToEndAsync().ConfigureAwait(false); - } - finally - { - decompressed?.Dispose(); - } + using var stream = await response.GetStreamAsync().ConfigureAwait(false); + + using var decompressed = + response.GetContentEncodingType() == ContentEncodingType.GZip + ? new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true) + : null; + + // Every parameter has to be given to reach leaveOpen. A byte order mark is not expected, and + // letting one be detected would override the encoding the response declared. + using var reader = new StreamReader( + decompressed ?? stream, + response.GetCharsetEncoding(), + detectEncodingFromByteOrderMarks: false, + bufferSize: 1024, // the default + leaveOpen: true); + + return await reader.ReadToEndAsync().ConfigureAwait(false); } private Task ApplyAsync(PollResult result) From 7f9b72be924065a0212ba6c289eb2d4d7f7b135f Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:52:31 +0300 Subject: [PATCH 55/62] [FeatureFlags] Make the agentless apply step synchronous and pass PollResult by in --- .../Agentless/AgentlessConfigurationSource.cs | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 68e358202fa5..1cffba761f54 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -170,7 +170,7 @@ internal async Task PollAsync() return; } - if (!IsRetryable(result)) + if (!IsRetryable(in result)) { break; } @@ -178,7 +178,7 @@ internal async Task PollAsync() if (attempt == MaxAttempts) { // Every attempt failed in a retryable way. Last-known-good stays in place. - WarnFailure(result, MaxAttempts); + WarnFailure(in result, MaxAttempts); return; } @@ -197,7 +197,7 @@ internal async Task PollAsync() return; } - await ApplyAsync(result).ConfigureAwait(false); + Apply(in result); } public void Dispose() @@ -375,19 +375,19 @@ private async Task ReadBodyAsync(IApiResponse response) return await reader.ReadToEndAsync().ConfigureAwait(false); } - private Task ApplyAsync(PollResult result) + private void Apply(in PollResult result) { switch (result.StatusCode) { case 304: // Nothing changed, and the ETag stays as it is. - return Task.CompletedTask; + return; case 401 or 403: - WarnFailure(result, attempts: 1); - return Task.CompletedTask; + WarnFailure(in result, attempts: 1); + return; case not 200: - WarnFailure(result, attempts: 1); - return Task.CompletedTask; + WarnFailure(in result, attempts: 1); + return; } if (!UfcConfigurationParser.TryParse(result.Body, out var configuration, out var error)) @@ -398,7 +398,7 @@ private Task ApplyAsync(PollResult result) Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", error); } - return Task.CompletedTask; + return; } if (!_applyConfiguration(configuration)) @@ -409,7 +409,7 @@ private Task ApplyAsync(PollResult result) Log.Warning("Feature Flags agentless configuration could not be applied"); } - return Task.CompletedTask; + return; } // The ETag advances only once parsing and applying have both succeeded. Advancing on @@ -417,8 +417,6 @@ private Task ApplyAsync(PollResult result) // answer 304, pinning the process to stale configuration with no way back. var newEtag = result.ETag?.Trim(); _etag = StringUtil.IsNullOrEmpty(newEtag) ? null : newEtag; - - return Task.CompletedTask; } /// From c01d3f2fd5dff3affac0eafeae8ae1ce0cb8186f Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 16:53:54 +0300 Subject: [PATCH 56/62] [FeatureFlags] Start the agentless poll loop with Task.Run --- .../FeatureFlags/Agentless/AgentlessConfigurationSource.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 1cffba761f54..cf3b18ee6040 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -146,10 +146,9 @@ public void Start() return; } - // Deliberately not wrapped in Task.Run: this is called from provider initialization, which - // is waiting for the first configuration, so the first request should go out on the calling - // thread rather than queue behind whatever else is on the thread pool. - _ = RunAsync().ContinueWith(t => Log.Error(t.Exception, "Feature Flags agentless poll loop failed"), TaskContinuationOptions.OnlyOnFaulted); + // The loop runs on the thread pool, so nothing of it happens on the caller's thread. Nothing + // ever awaits it either, so a fault is observed here or not at all. + _ = Task.Run(RunAsync).ContinueWith(t => Log.Error(t.Exception, "Feature Flags agentless poll loop failed"), TaskContinuationOptions.OnlyOnFaulted); } /// From 0610bea7c8ffdba3ec3c1f6da57efeef94370431 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 17:01:55 +0300 Subject: [PATCH 57/62] [FeatureFlags] Rebuild the agentless request URI only when the environment changes --- .../Agentless/AgentlessConfigurationSource.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index cf3b18ee6040..f8c0ced5886b 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -56,8 +56,9 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); - // Written by the settings-change callback, read by the poll loop. - private string? _environment; + // Written by the settings-change callback, read by the poll loop. The URI is stored rather than + // the environment it carries, so it is only built when the environment changes. + private Uri _requestUri; private IDisposable? _environmentSubscription; // Only ever touched from the poll loop. @@ -82,7 +83,7 @@ internal AgentlessConfigurationSource( _pollInterval = pollInterval; _requestTimeout = requestTimeout; _applyConfiguration = applyConfiguration; - _environment = environment; + _requestUri = endpoint.BuildRequestUri(environment); _waitAsync = waitAsync ?? Task.Delay; } @@ -117,8 +118,8 @@ internal AgentlessConfigurationSource( manager.InitialMutableSettings.Environment); // The environment is tracked rather than captured: customers can change it in code while - // the application runs, and flags are targeted per environment. Only the value is stored - // here, so that the poll loop stays the only thing that touches the request state. + // the application runs, and flags are targeted per environment. Only the request URI is + // stored here, so that the poll loop stays the only thing that touches the request state. source._environmentSubscription = manager.SubscribeToChanges(changes => { if (changes.UpdatedMutable is { } mutable) @@ -131,10 +132,11 @@ internal AgentlessConfigurationSource( } /// - /// Records the environment to request configuration for. Applied by the poll loop on its next - /// request, so a change never disturbs a request already in flight. + /// Records the environment to request configuration for, as the URI that carries it. Picked up + /// by the poll loop on its next request, so a change never disturbs a request already in flight. /// - internal void UpdateEnvironment(string? environment) => Volatile.Write(ref _environment, environment); + internal void UpdateEnvironment(string? environment) + => Volatile.Write(ref _requestUri, _endpoint.BuildRequestUri(environment)); /// /// Starts polling. Idempotent. @@ -300,9 +302,9 @@ private async Task RequestAsync() { try { - // The environment is applied per request rather than baked into the endpoint, because - // it can be changed in code after startup. - var uri = _endpoint.BuildRequestUri(Volatile.Read(ref _environment)); + // Read per request rather than captured, because the environment it carries can be + // changed in code after startup. + var uri = Volatile.Read(ref _requestUri); // An ETag only identifies the configuration served for the URI it came from. Sending it // against a different environment would earn a 304 and pin the process to the previous From c3a452b82b971c13916e5e17c43f6bd3a64684ec Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 17:13:44 +0300 Subject: [PATCH 58/62] [FeatureFlags] Parse the agentless payload straight from the response stream --- .../Agentless/AgentlessConfigurationSource.cs | 69 +++++++++++-------- .../Agentless/UfcConfigurationParser.cs | 11 ++- .../UfcConfigurationParserTests.cs | 18 +++-- 3 files changed, 57 insertions(+), 41 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index f8c0ced5886b..058d06e9d98f 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -339,43 +339,50 @@ private async Task RequestAsync() t => { try { using var r = t.Result; } catch { } }, TaskContinuationOptions.None); - return new PollResult(statusCode: null, etag: null, body: null, error: new TimeoutException($"Feature Flags agentless request timed out after {_requestTimeout.TotalSeconds}s")); + return new PollResult(statusCode: null, etag: null, configuration: null, parseError: null, error: new TimeoutException($"Feature Flags agentless request timed out after {_requestTimeout.TotalSeconds}s")); } using var response = await getTask.ConfigureAwait(false); #endif - // Only a 200 carries configuration; other bodies are never decoded as one. - var body = response.StatusCode == 200 ? await ReadBodyAsync(response).ConfigureAwait(false) : null; - return new PollResult(response.StatusCode, response.GetHeader("ETag"), body, error: null); + // Only a 200 carries configuration; other bodies are never decoded as one. The payload + // is parsed here, while the response is still open, so it never has to be held as a + // string. A payload that does not parse is reported, not thrown: it is not retryable. + ServerConfiguration? configuration = null; + string? parseError = null; + + if (response.StatusCode == 200) + { + using var stream = await response.GetStreamAsync().ConfigureAwait(false); + + using var decompressed = + response.GetContentEncodingType() == ContentEncodingType.GZip + ? new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true) + : null; + + // Every parameter has to be given to reach leaveOpen. A byte order mark is not + // expected, and letting one be detected would override the declared encoding. + using var reader = new StreamReader( + decompressed ?? stream, + response.GetCharsetEncoding(), + detectEncodingFromByteOrderMarks: false, + bufferSize: 1024, // the default + leaveOpen: true); + + if (UfcConfigurationParser.TryParse(reader, out var parsed, out parseError)) + { + configuration = parsed; + } + } + + return new PollResult(response.StatusCode, response.GetHeader("ETag"), configuration, parseError, error: null); } catch (Exception ex) { - return new PollResult(statusCode: null, etag: null, body: null, error: ex); + return new PollResult(statusCode: null, etag: null, configuration: null, parseError: null, error: ex); } } - private async Task ReadBodyAsync(IApiResponse response) - { - using var stream = await response.GetStreamAsync().ConfigureAwait(false); - - using var decompressed = - response.GetContentEncodingType() == ContentEncodingType.GZip - ? new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true) - : null; - - // Every parameter has to be given to reach leaveOpen. A byte order mark is not expected, and - // letting one be detected would override the encoding the response declared. - using var reader = new StreamReader( - decompressed ?? stream, - response.GetCharsetEncoding(), - detectEncodingFromByteOrderMarks: false, - bufferSize: 1024, // the default - leaveOpen: true); - - return await reader.ReadToEndAsync().ConfigureAwait(false); - } - private void Apply(in PollResult result) { switch (result.StatusCode) @@ -391,12 +398,12 @@ private void Apply(in PollResult result) return; } - if (!UfcConfigurationParser.TryParse(result.Body, out var configuration, out var error)) + if (result.Configuration is not { } configuration) { if (!_malformedPayloadLogged) { _malformedPayloadLogged = true; - Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", error); + Log.Error("Feature Flags agentless endpoint returned an unusable payload: {Error}", result.ParseError); } return; @@ -452,13 +459,15 @@ private void WarnFailure(in PollResult result, int attempts) } } - internal readonly struct PollResult(int? statusCode, string? etag, string? body, Exception? error) + internal readonly struct PollResult(int? statusCode, string? etag, ServerConfiguration? configuration, string? parseError, Exception? error) { public int? StatusCode { get; } = statusCode; public string? ETag { get; } = etag; - public string? Body { get; } = body; + public ServerConfiguration? Configuration { get; } = configuration; + + public string? ParseError { get; } = parseError; public Exception? Error { get; } = error; } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs index a078f5f4374d..50e1fd6fa565 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -26,11 +26,11 @@ internal static class UfcConfigurationParser /// which is the document the evaluator consumes. A raw UFC document is rejected, including from /// a custom endpoint, so that every source agrees on one wire format. /// - /// The response body. + /// The response body. Read straight from the response, so it never has to be held as a string. /// The parsed configuration. /// Why the payload was rejected. /// true when the payload matches the contract. - public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfiguration? configuration, out string? error) + public static bool TryParse(TextReader body, [NotNullWhen(true)] out ServerConfiguration? configuration, out string? error) { configuration = null; error = null; @@ -38,11 +38,10 @@ public static bool TryParse(string? body, [NotNullWhen(true)] out ServerConfigur JToken payload; try { - using var stringReader = new StringReader(body ?? string.Empty); - // Timestamps stay strings: the model carries createdAt verbatim, and letting Newtonsoft - // turn it into a date would also make the type check below fail. - using var jsonReader = new JsonTextReader(stringReader) { DateParseHandling = DateParseHandling.None }; + // turn it into a date would also make the type check below fail. The reader belongs to + // the caller, which owns the response it came from. + using var jsonReader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false }; payload = JToken.ReadFrom(jsonReader); } catch (Exception) diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs index b57c9568ebcd..cfdf5f85adf9 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/UfcConfigurationParserTests.cs @@ -6,6 +6,7 @@ #nullable enable using System.Collections.Generic; +using System.IO; using Datadog.Trace.FeatureFlags.Agentless; using Datadog.Trace.FeatureFlags.Rcm.Model; using FluentAssertions; @@ -29,7 +30,7 @@ public class UfcConfigurationParserTests [Fact] public void ParsesValidEnvelope() { - UfcConfigurationParser.TryParse(ValidEnvelope, out var configuration, out var error) + Parse(ValidEnvelope, out var configuration, out var error) .Should().BeTrue(); error.Should().BeNull(); @@ -45,7 +46,7 @@ public void ParsesValidEnvelope() [InlineData("{ \"data\": ")] public void RejectsMalformedJson(string? body) { - UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + Parse(body, out var configuration, out var error).Should().BeFalse(); configuration.Should().BeNull(); error.Should().Be("Malformed UFC payload"); @@ -66,7 +67,7 @@ public void RejectsMalformedJson(string? body) [InlineData("""{ "data": { "type": [1, 2], "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": {} } } }""")] public void RejectsInvalidEnvelope(string body) { - UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + Parse(body, out var configuration, out var error).Should().BeFalse(); configuration.Should().BeNull(); error.Should().Be("Expected a JSON:API Universal Flag Configuration resource"); @@ -87,7 +88,7 @@ public void RejectsInvalidEnvelope(string body) [InlineData("""{ "data": { "type": "universal-flag-configuration", "attributes": { "format": "SERVER", "createdAt": "x", "environment": { "name": "prod" }, "flags": [] } } }""")] public void RejectsInvalidAttributes(string body) { - UfcConfigurationParser.TryParse(body, out var configuration, out var error).Should().BeFalse(); + Parse(body, out var configuration, out var error).Should().BeFalse(); configuration.Should().BeNull(); error.Should().Be("Expected a Universal Flag Configuration v1 object"); @@ -103,10 +104,17 @@ public void ParsesFlagsFromEnvelope() "flags": { "test-flag": { "key": "test-flag", "enabled": true, "variationType": "BOOLEAN" } } } } } """; - UfcConfigurationParser.TryParse(body, out var configuration, out _).Should().BeTrue(); + Parse(body, out var configuration, out _).Should().BeTrue(); configuration!.Flags.Should().NotBeNull(); configuration!.Flags!.Should().ContainKey("test-flag"); configuration!.Flags!["test-flag"].Enabled.Should().BeTrue(); } + + // The parser reads the response stream directly, so a body under test is handed to it as a reader. + private static bool Parse(string? body, out ServerConfiguration? configuration, out string? error) + { + using var reader = new StringReader(body ?? string.Empty); + return UfcConfigurationParser.TryParse(reader, out configuration, out error); + } } From 1f2d697d128bc313b67dd5aff7e707cf4bc3c370 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 26 Aug 2026 17:27:24 +0300 Subject: [PATCH 59/62] [FeatureFlags] Deserialize the UFC envelope without building a JSON tree --- .../Agentless/UfcConfigurationParser.cs | 149 ++++++++++++++---- .../Rcm/Model/FlagCollectionJsonConverter.cs | 9 ++ .../UfcConfigurationParserTests.cs | 16 +- 3 files changed, 140 insertions(+), 34 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs index 50e1fd6fa565..b03f17141196 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -10,7 +10,6 @@ using System.IO; using Datadog.Trace.FeatureFlags.Rcm.Model; using Datadog.Trace.Vendors.Newtonsoft.Json; -using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; namespace Datadog.Trace.FeatureFlags.Agentless; @@ -21,10 +20,19 @@ internal static class UfcConfigurationParser { private const string ResourceType = "universal-flag-configuration"; + private const string MalformedError = "Malformed UFC payload"; + private const string ResourceError = "Expected a JSON:API Universal Flag Configuration resource"; + private const string AttributesError = "Expected a Universal Flag Configuration v1 object"; + /// /// Validates a JSON:API Universal Flag Configuration response and returns data.attributes, /// which is the document the evaluator consumes. A raw UFC document is rejected, including from /// a custom endpoint, so that every source agrees on one wire format. + /// + /// The envelope is walked with the reader rather than loaded into a JSON tree, and + /// data.attributes is deserialized in place, so the payload is read exactly once and no + /// copy of it is ever held. + /// /// /// The response body. Read straight from the response, so it never has to be held as a string. /// The parsed configuration. @@ -35,56 +43,133 @@ public static bool TryParse(TextReader body, [NotNullWhen(true)] out ServerConfi configuration = null; error = null; - JToken payload; + var sawData = false; + string? resourceType = null; + ServerConfiguration? attributes = null; + try { // Timestamps stay strings: the model carries createdAt verbatim, and letting Newtonsoft // turn it into a date would also make the type check below fail. The reader belongs to // the caller, which owns the response it came from. - using var jsonReader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false }; - payload = JToken.ReadFrom(jsonReader); + using var reader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false }; + var serializer = new JsonSerializer { DateParseHandling = DateParseHandling.None }; + + if (!reader.Read()) + { + // Nothing at all, so there is no document to judge against the contract. + error = MalformedError; + return false; + } + + if (reader.TokenType != JsonToken.StartObject) + { + error = ResourceError; + return false; + } + + while (reader.Read() && reader.TokenType == JsonToken.PropertyName) + { + if ((string?)reader.Value != "data") + { + reader.Skip(); + continue; + } + + sawData = true; + + if (!reader.Read()) + { + // The document ended where the resource should have been. + error = MalformedError; + return false; + } + + if (reader.TokenType != JsonToken.StartObject) + { + error = ResourceError; + return false; + } + + while (reader.Read() && reader.TokenType == JsonToken.PropertyName) + { + switch ((string?)reader.Value) + { + case "type": + if (!reader.Read()) + { + error = MalformedError; + return false; + } + + // A type that is not a string cannot identify the resource. Checked on + // the token, because a number would otherwise be read as its digits. + if (reader.TokenType != JsonToken.String) + { + error = ResourceError; + return false; + } + + resourceType = (string?)reader.Value; + break; + + case "attributes": + if (!reader.Read()) + { + error = MalformedError; + return false; + } + + if (reader.TokenType != JsonToken.StartObject) + { + error = AttributesError; + return false; + } + + attributes = serializer.Deserialize(reader); + break; + + default: + reader.Skip(); + break; + } + } + } + + // A document that ends before the root object closes was truncated in transit, whatever + // was found in it up to that point. + if (reader.TokenType != JsonToken.EndObject) + { + error = MalformedError; + return false; + } } catch (Exception) { - error = "Malformed UFC payload"; + error = MalformedError; return false; } - if (payload is not JObject - || payload["data"] is not JObject data - || data["type"]?.Type != JTokenType.String - || data["type"]?.Value() != ResourceType) + if (!sawData || resourceType != ResourceType) { - error = "Expected a JSON:API Universal Flag Configuration resource"; + error = ResourceError; return false; } - if (data["attributes"] is not JObject attributes - || attributes["format"]?.Type != JTokenType.String - || attributes["createdAt"]?.Type != JTokenType.String - || attributes["environment"] is not JObject environment - || environment["name"]?.Type != JTokenType.String - || attributes["flags"] is not JObject) - { - error = "Expected a Universal Flag Configuration v1 object"; - return false; - } - - try - { - configuration = attributes.ToObject(); - } - catch (Exception) - { - configuration = null; - } - - if (configuration is null) + // Every member of the v1 contract has to be there. A member of the wrong shape arrives as + // null, because the flag collection rejects anything that is not an object and Newtonsoft + // leaves a member it cannot convert unset. + if (attributes is null + || attributes.Format is null + || attributes.CreatedAt is null + || attributes.Environment?.Name is null + || attributes.Flags is null) { - error = "Expected a Universal Flag Configuration v1 object"; + error = AttributesError; return false; } + configuration = attributes; return true; } } diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs b/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs index 87a4a8937986..101818093e56 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Rcm/Model/FlagCollectionJsonConverter.cs @@ -25,6 +25,15 @@ internal sealed class FlagCollectionJsonConverter : JsonConverter Date: Wed, 26 Aug 2026 17:31:53 +0300 Subject: [PATCH 60/62] [FeatureFlags] Subscribe to settings changes in the agentless constructor --- .../Agentless/AgentlessConfigurationSource.cs | 40 ++++++++++--------- .../AgentlessConfigurationSourceTests.cs | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 058d06e9d98f..35f966af1c4e 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -56,10 +56,11 @@ internal sealed class AgentlessConfigurationSource : IDisposable // Only ever touched from the poll loop. private readonly HashSet _loggedFailureCategories = new(); + private readonly IDisposable? _environmentSubscription; + // Written by the settings-change callback, read by the poll loop. The URI is stored rather than // the environment it carries, so it is only built when the environment changes. private Uri _requestUri; - private IDisposable? _environmentSubscription; // Only ever touched from the poll loop. private bool _malformedPayloadLogged; @@ -76,6 +77,7 @@ internal AgentlessConfigurationSource( TimeSpan requestTimeout, Func applyConfiguration, string? environment = null, + TracerSettings.SettingsManager? settingsManager = null, Func? waitAsync = null) { _endpoint = endpoint; @@ -85,6 +87,17 @@ internal AgentlessConfigurationSource( _applyConfiguration = applyConfiguration; _requestUri = endpoint.BuildRequestUri(environment); _waitAsync = waitAsync ?? Task.Delay; + + // Subscribed last, so every field the callback touches is already set. The environment is + // tracked rather than captured: customers can change it in code while the application runs, + // and flags are targeted per environment. + _environmentSubscription = settingsManager?.SubscribeToChanges(changes => + { + if (changes.UpdatedMutable is { } mutable) + { + UpdateEnvironment(mutable.Environment); + } + }); } /// @@ -109,26 +122,14 @@ internal AgentlessConfigurationSource( return null; } - var source = new AgentlessConfigurationSource( + return new AgentlessConfigurationSource( endpoint, CreateRequestFactory(endpoint, settings), settings.PollInterval, settings.RequestTimeout, applyConfiguration, - manager.InitialMutableSettings.Environment); - - // The environment is tracked rather than captured: customers can change it in code while - // the application runs, and flags are targeted per environment. Only the request URI is - // stored here, so that the poll loop stays the only thing that touches the request state. - source._environmentSubscription = manager.SubscribeToChanges(changes => - { - if (changes.UpdatedMutable is { } mutable) - { - source.UpdateEnvironment(mutable.Environment); - } - }); - - return source; + manager.InitialMutableSettings.Environment, + manager); } /// @@ -199,6 +200,10 @@ internal async Task PollAsync() } Apply(in result); + + // A failure with no status code never reached the endpoint, so it is worth another attempt. + static bool IsRetryable(in PollResult result) + => result.StatusCode is not { } status || status is 408 or 429 or (>= 500 and <= 599); } public void Dispose() @@ -254,9 +259,6 @@ private static ApiWebRequestFactory CreateRequestFactory(AgentlessEndpoint endpo #endif } - private static bool IsRetryable(in PollResult result) - => result.StatusCode is not { } status || status is 408 or 429 or (>= 500 and <= 599); - private async Task RunAsync() { Log.Debug("AgentlessConfigurationSource::RunAsync -> Enter"); diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index 276f28b37251..b87daf911085 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -256,7 +256,7 @@ private static AgentlessConfigurationSource CreateSource( return true; }, environment, - NoWait); + waitAsync: NoWait); private static AgentlessEndpoint CreateEndpoint() { From 2edc435bbd420ae0d51cc6f4ebb83d2dc8d208a3 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 27 Aug 2026 14:22:33 +0300 Subject: [PATCH 61/62] [FeatureFlags] Let the HTTP abstraction own the agentless request timeout --- .../Agentless/AgentlessConfigurationSource.cs | 25 ------------------- .../AgentlessConfigurationSourceTests.cs | 1 - 2 files changed, 26 deletions(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs index 35f966af1c4e..d42449987fdb 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/AgentlessConfigurationSource.cs @@ -45,7 +45,6 @@ internal sealed class AgentlessConfigurationSource : IDisposable private readonly IApiRequestFactory _requestFactory; private readonly AgentlessEndpoint _endpoint; private readonly TimeSpan _pollInterval; - private readonly TimeSpan _requestTimeout; private readonly Func _applyConfiguration; private readonly Func _waitAsync; @@ -74,7 +73,6 @@ internal AgentlessConfigurationSource( AgentlessEndpoint endpoint, IApiRequestFactory requestFactory, TimeSpan pollInterval, - TimeSpan requestTimeout, Func applyConfiguration, string? environment = null, TracerSettings.SettingsManager? settingsManager = null, @@ -83,7 +81,6 @@ internal AgentlessConfigurationSource( _endpoint = endpoint; _requestFactory = requestFactory; _pollInterval = pollInterval; - _requestTimeout = requestTimeout; _applyConfiguration = applyConfiguration; _requestUri = endpoint.BuildRequestUri(environment); _waitAsync = waitAsync ?? Task.Delay; @@ -126,7 +123,6 @@ internal AgentlessConfigurationSource( endpoint, CreateRequestFactory(endpoint, settings), settings.PollInterval, - settings.RequestTimeout, applyConfiguration, manager.InitialMutableSettings.Environment, manager); @@ -324,28 +320,7 @@ private async Task RequestAsync() request.AddHeader("If-None-Match", etag); } -#if NETCOREAPP - // HttpClient.Timeout applies to async calls, so no explicit race is needed. using var response = await request.GetAsync().ConfigureAwait(false); -#else - // HttpWebRequest.Timeout does not apply to async calls (GetResponseAsync), so we race - // the request against an explicit delay to bound the wait on net461/netstandard2.0. - var getTask = request.GetAsync(); - var timeoutTask = Task.Delay(_requestTimeout); - - if (await Task.WhenAny(getTask, timeoutTask).ConfigureAwait(false) == timeoutTask) - { - // The request is still in flight. Dispose the response when it eventually completes - // (success or fault) so the underlying connection is released. - _ = getTask.ContinueWith( - t => { try { using var r = t.Result; } catch { } }, - TaskContinuationOptions.None); - - return new PollResult(statusCode: null, etag: null, configuration: null, parseError: null, error: new TimeoutException($"Feature Flags agentless request timed out after {_requestTimeout.TotalSeconds}s")); - } - - using var response = await getTask.ConfigureAwait(false); -#endif // Only a 200 carries configuration; other bodies are never decoded as one. The payload // is parsed here, while the response is still open, so it never has to be held as a diff --git a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs index b87daf911085..c65b3f466c55 100644 --- a/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs +++ b/tracer/test/Datadog.Trace.Tests/FeatureFlags/AgentlessConfigurationSourceTests.cs @@ -249,7 +249,6 @@ private static AgentlessConfigurationSource CreateSource( CreateEndpoint(), factory, TimeSpan.FromSeconds(30), - TimeSpan.FromSeconds(5), configuration => { applied.Add(configuration); From da002db08715a7f978db774da277bed789b4622d Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 27 Aug 2026 15:43:04 +0300 Subject: [PATCH 62/62] [FeatureFlags] Rent the UFC parser's scratch buffers from the shared array pool --- .../FeatureFlags/Agentless/UfcConfigurationParser.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs index b03f17141196..1dddb52db0bc 100644 --- a/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs +++ b/tracer/src/Datadog.Trace/FeatureFlags/Agentless/UfcConfigurationParser.cs @@ -9,6 +9,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using Datadog.Trace.FeatureFlags.Rcm.Model; +using Datadog.Trace.Util.Json; using Datadog.Trace.Vendors.Newtonsoft.Json; namespace Datadog.Trace.FeatureFlags.Agentless; @@ -52,7 +53,7 @@ public static bool TryParse(TextReader body, [NotNullWhen(true)] out ServerConfi // Timestamps stay strings: the model carries createdAt verbatim, and letting Newtonsoft // turn it into a date would also make the type check below fail. The reader belongs to // the caller, which owns the response it came from. - using var reader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false }; + using var reader = new JsonTextReader(body) { DateParseHandling = DateParseHandling.None, CloseInput = false, ArrayPool = JsonArrayPool.Shared }; var serializer = new JsonSerializer { DateParseHandling = DateParseHandling.None }; if (!reader.Read())