[Feature Flags] Add privacy-protected flagevaluation EVP events through the Agent

Datadog.Trace.Build.g.sln

@@ -275,6 +275,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Datadog.AutoInstrumentation
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Datadog.AutoInstrumentation.Generator.Cli", "tracer\src\Datadog.AutoInstrumentation.Generator.Cli\Datadog.AutoInstrumentation.Generator.Cli.csproj", "{8D722514-8BAD-4069-84D0-15797026E0ED}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Datadog.FeatureFlags.OpenFeature.Tests", "tracer\test\Datadog.FeatureFlags.OpenFeature.Tests\Datadog.FeatureFlags.OpenFeature.Tests.csproj", "{372B229E-7999-4AB6-903E-BDAF958F1696}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenTelemetry.AutoInstrumentation.StartupHook", "tracer\src\OpenTelemetry.AutoInstrumentation.StartupHook\OpenTelemetry.AutoInstrumentation.StartupHook.csproj", "{14E32033-EB3F-B2BB-45AE-A8EBA0863324}"
 EndProject
 Global
@@ -573,6 +575,10 @@ Global
 		{8D722514-8BAD-4069-84D0-15797026E0ED}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8D722514-8BAD-4069-84D0-15797026E0ED}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8D722514-8BAD-4069-84D0-15797026E0ED}.Release|Any CPU.Build.0 = Release|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Release|Any CPU.Build.0 = Release|Any CPU
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -670,6 +676,7 @@ Global
 		{CFECF8D4-3A46-35A8-7CB1-BA359974A1A9} = {9E5F0022-0A50-40BF-AC6A-C3078585ECAB}
 		{B3A07CA3-3536-4D43-81A3-A5A73FB76F22} = {9E5F0022-0A50-40BF-AC6A-C3078585ECAB}
 		{8D722514-8BAD-4069-84D0-15797026E0ED} = {9E5F0022-0A50-40BF-AC6A-C3078585ECAB}
+		{372B229E-7999-4AB6-903E-BDAF958F1696} = {8CEC2042-F11C-49F5-A674-2355793B600A}
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324} = {9E5F0022-0A50-40BF-AC6A-C3078585ECAB}
 	EndGlobalSection
 EndGlobal

Datadog.Trace.sln

@@ -657,6 +657,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Datadog.AutoInstrumentation
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Datadog.AutoInstrumentation.Generator.Cli", "tracer\src\Datadog.AutoInstrumentation.Generator.Cli\Datadog.AutoInstrumentation.Generator.Cli.csproj", "{8D722514-8BAD-4069-84D0-15797026E0ED}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Datadog.FeatureFlags.OpenFeature.Tests", "tracer\test\Datadog.FeatureFlags.OpenFeature.Tests\Datadog.FeatureFlags.OpenFeature.Tests.csproj", "{372B229E-7999-4AB6-903E-BDAF958F1696}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenTelemetry.AutoInstrumentation.StartupHook", "tracer\src\OpenTelemetry.AutoInstrumentation.StartupHook\OpenTelemetry.AutoInstrumentation.StartupHook.csproj", "{14E32033-EB3F-B2BB-45AE-A8EBA0863324}"
 EndProject
 Global
@@ -1611,6 +1613,10 @@ Global
 		{8D722514-8BAD-4069-84D0-15797026E0ED}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8D722514-8BAD-4069-84D0-15797026E0ED}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8D722514-8BAD-4069-84D0-15797026E0ED}.Release|Any CPU.Build.0 = Release|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{372B229E-7999-4AB6-903E-BDAF958F1696}.Release|Any CPU.Build.0 = Release|Any CPU
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -1877,6 +1883,7 @@ Global
 		{08C6510D-42DC-C8B4-CC79-26349F2D6DBB} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A}
 		{B3A07CA3-3536-4D43-81A3-A5A73FB76F22} = {9E5F0022-0A50-40BF-AC6A-C3078585ECAB}
 		{8D722514-8BAD-4069-84D0-15797026E0ED} = {9E5F0022-0A50-40BF-AC6A-C3078585ECAB}
+		{372B229E-7999-4AB6-903E-BDAF958F1696} = {8CEC2042-F11C-49F5-A674-2355793B600A}
 		{14E32033-EB3F-B2BB-45AE-A8EBA0863324} = {9E5F0022-0A50-40BF-AC6A-C3078585ECAB}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution

tracer/build/supported_calltargets.g.json

@@ -14114,6 +14114,60 @@
     "IsAdoNetIntegration": false,
     "InstrumentationCategory": 1
   },
+  {
+    "IntegrationName": "OpenFeature",
+    "AssemblyName": "Datadog.FeatureFlags.OpenFeature",
+    "TargetTypeName": "Datadog.FeatureFlags.OpenFeature.FeatureFlagsSdk",
+    "TargetMethodName": "CanEnqueueEVP",
+    "TargetReturnType": "System.Boolean",
+    "TargetParameterTypes": [],
+    "MinimumVersion": {
+      "Item1": 2,
+      "Item2": 0,
+      "Item3": 0
+    },
+    "MaximumVersion": {
+      "Item1": 2,
+      "Item2": 65535,
+      "Item3": 65535
+    },
+    "InstrumentationTypeName": "Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.OpenFeature.OpenFeatureSdkCanEnqueueEVPIntegration",
+    "IntegrationKind": 0,
+    "IsAdoNetIntegration": false,
+    "InstrumentationCategory": 1
+  },
+  {
+    "IntegrationName": "OpenFeature",
+    "AssemblyName": "Datadog.FeatureFlags.OpenFeature",
+    "TargetTypeName": "Datadog.FeatureFlags.OpenFeature.FeatureFlagsSdk",
+    "TargetMethodName": "EnqueueEVP",
+    "TargetReturnType": "System.Void",
+    "TargetParameterTypes": [
+      "System.String",
+      "System.String",
+      "System.String",
+      "System.String",
+      "System.Int64",
+      "System.Boolean",
+      "System.String",
+      "System.Collections.Generic.IReadOnlyDictionary`2[System.String,System.Object]",
+      "System.Int32"
+    ],
+    "MinimumVersion": {
+      "Item1": 2,
+      "Item2": 0,
+      "Item3": 0
+    },
+    "MaximumVersion": {
+      "Item1": 2,
+      "Item2": 65535,
+      "Item3": 65535
+    },
+    "InstrumentationTypeName": "Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.OpenFeature.OpenFeatureSdkEnqueueEVPIntegration",
+    "IntegrationKind": 0,
+    "IsAdoNetIntegration": false,
+    "InstrumentationCategory": 1
+  },
   {
     "IntegrationName": "OpenFeature",
     "AssemblyName": "Datadog.FeatureFlags.OpenFeature",
@@ -14232,6 +14286,28 @@
     "IsAdoNetIntegration": false,
     "InstrumentationCategory": 1
   },
+  {
+    "IntegrationName": "OpenFeature",
+    "AssemblyName": "Datadog.FeatureFlags.OpenFeature",
+    "TargetTypeName": "Datadog.FeatureFlags.OpenFeature.FeatureFlagsSdk",
+    "TargetMethodName": "RecordEVPHookError",
+    "TargetReturnType": "System.Void",
+    "TargetParameterTypes": [],
+    "MinimumVersion": {
+      "Item1": 2,
+      "Item2": 0,
+      "Item3": 0
+    },
+    "MaximumVersion": {
+      "Item1": 2,
+      "Item2": 65535,
+      "Item3": 65535
+    },
+    "InstrumentationTypeName": "Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.OpenFeature.OpenFeatureSdkRecordEVPHookErrorIntegration",
+    "IntegrationKind": 0,
+    "IsAdoNetIntegration": false,
+    "InstrumentationCategory": 1
+  },
   {
     "IntegrationName": "OpenFeature",
     "AssemblyName": "Datadog.FeatureFlags.OpenFeature",

tracer/src/Datadog.FeatureFlags.OpenFeature/Datadog.FeatureFlags.OpenFeature.csproj

@@ -24,12 +24,14 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <InternalsVisibleTo Include="Datadog.FeatureFlags.OpenFeature.Tests" />
     <None Include="..\..\..\docs\Datadog.FeatureFlags.OpenFeature\README.md" Pack="true" PackagePath="\" />
   </ItemGroup>
 
   <ItemGroup>
     <Compile Include="..\Datadog.Trace\FeatureFlags\EvaluationReason.cs" Link="Sdk\EvaluationReason.cs" />
     <Compile Include="..\Datadog.Trace\FeatureFlags\FeatureFlagMetadataKeys.cs" Link="Sdk\FeatureFlagMetadataKeys.cs" />
+    <Compile Include="..\Datadog.Trace\FeatureFlags\FlagEvaluation\ContextOmissionReason.cs" Link="Sdk\ContextOmissionReason.cs" />
     <Compile Include="..\Datadog.Trace\FeatureFlags\IEvaluation.cs" Link="Sdk\IEvaluation.cs" />
     <Compile Include="..\Datadog.Trace\FeatureFlags\ValueType.cs" Link="Sdk\ValueType.cs" />
   </ItemGroup>

tracer/src/Datadog.FeatureFlags.OpenFeature/DatadogProvider.cs

@@ -40,6 +40,7 @@ public sealed class DatadogProvider : global::OpenFeature.FeatureProvider, IDisp
     // Span-enrichment hook is constructed ONLY when the gate is on; null otherwise so
     // nothing is allocated/registered when the feature is disabled.
     private readonly SpanEnrichmentHook? _spanEnrichmentHook;
+    private readonly FlagEvalEVPHook _evpHook = new();
 
     private int _status = StatusInitializing;
 
@@ -275,17 +276,17 @@ public override IImmutableList<Hook> GetProviderHooks()
 #if NET6_0_OR_GREATER
         if (_spanEnrichmentHook is not null)
         {
-            return ImmutableList.Create<Hook>(_metricsHook, _spanEnrichmentHook);
+            return ImmutableList.Create<Hook>(_metricsHook, _spanEnrichmentHook, _evpHook);
         }
 
-        return ImmutableList.Create<Hook>(_metricsHook);
+        return ImmutableList.Create<Hook>(_metricsHook, _evpHook);
 #else
         if (_spanEnrichmentHook is not null)
         {
-            return ImmutableList.Create<Hook>(_spanEnrichmentHook);
+            return ImmutableList.Create<Hook>(_spanEnrichmentHook, _evpHook);
         }
 
-        return ImmutableList<Hook>.Empty;
+        return ImmutableList.Create<Hook>(_evpHook);
 #endif
     }
 

tracer/src/Datadog.FeatureFlags.OpenFeature/FlagEvalEVPHook.cs

@@ -0,0 +1,124 @@
+// <copyright file="FlagEvalEVPHook.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Datadog.Trace.FeatureFlags;
+using Datadog.Trace.FeatureFlags.FlagEvaluation;
+using OpenFeature;
+using OpenFeature.Constant;
+using OpenFeature.Model;
+
+namespace Datadog.FeatureFlags.OpenFeature;
+
+internal sealed class FlagEvalEVPHook : Hook
+{
+    private readonly Func<bool> _canEnqueue;
+    private readonly Action<string, string?, string?, string?, long, bool, string?, IReadOnlyDictionary<string, object?>?, int> _enqueue;
+    private readonly Action _recordError;
+    private readonly CaptureSnapshot _capture;
+
+    internal FlagEvalEVPHook()
+        : this(FeatureFlagsSdk.CanEnqueueEVP, FeatureFlagsSdk.EnqueueEVP, FeatureFlagsSdk.RecordEVPHookError, FlagEvaluationContextSnapshot.Capture)
+    {
+    }
+
+    internal FlagEvalEVPHook(
+        Func<bool> canEnqueue,
+        Action<string, string?, string?, string?, long, bool, string?, IReadOnlyDictionary<string, object?>?, int> enqueue,
+        Action recordError,
+        CaptureSnapshot capture)
+    {
+        _canEnqueue = canEnqueue;
+        _enqueue = enqueue;
+        _recordError = recordError;
+        _capture = capture;
+    }
+
+    internal delegate IReadOnlyDictionary<string, object?> CaptureSnapshot(EvaluationContext? context, out int omissionReasons);
+
+    public override ValueTask FinallyAsync<T>(HookContext<T> context, FlagEvaluationDetails<T> details, IReadOnlyDictionary<string, object>? hints = null, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            if (!_canEnqueue())
+            {
+                return default;
+            }
+
+            var metadata = details.FlagMetadata;
+            var consent = metadata?.GetBool(FeatureFlagMetadataKeys.ObserveFullEvaluationData) == true;
+            var allocationKey = metadata?.GetString("__dd_allocation_key");
+            var timestamp = metadata?.GetDouble(FeatureFlagMetadataKeys.EvaluationTimestampMs);
+            // Missing metadata is possible when OpenFeature terminates before provider resolution.
+            var evalTimeMs = timestamp is >= 0 and <= 253402300799999d && timestamp == Math.Truncate(timestamp.Value)
+                                 ? (long)timestamp.Value
+                                 : DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+            var errorCode = ToErrorCode(details.ErrorType);
+            if (errorCode is null)
+            {
+                // Some evaluator failures carry descriptive messages that the provider does not
+                // map to an enum. Use only their stable metadata code, never the message.
+                errorCode = ToMetadataErrorCode(metadata?.GetString("errorCode"));
+            }
+
+            var targetingKey = context.EvaluationContext?.TargetingKey;
+            IReadOnlyDictionary<string, object?>? attributes = null;
+            var omissionReasons = 0;
+            if (consent)
+            {
+                try
+                {
+                    attributes = _capture(context.EvaluationContext, out omissionReasons);
+                }
+                catch (Exception)
+                {
+                    // Preserve the observation, but discard context and never expose exception text.
+                    omissionReasons = (int)ContextOmissionReason.SnapshotError;
+                }
+            }
+
+            _enqueue(context.FlagKey, details.Variant, allocationKey, targetingKey, evalTimeMs, consent, errorCode, attributes, omissionReasons);
+        }
+        catch (Exception)
+        {
+            try
+            {
+                _recordError();
+            }
+            catch (Exception)
+            {
+                // Telemetry failures must not change the customer's evaluation.
+            }
+        }
+
+        return default;
+    }
+
+    private static string? ToMetadataErrorCode(string? code) => code switch
+    {
+        null or "" => null,
+        "FLAG_NOT_FOUND" or "INVALID_CONTEXT" or "PARSE_ERROR" or "PROVIDER_FATAL" or
+        "PROVIDER_NOT_READY" or "TARGETING_KEY_MISSING" or "TYPE_MISMATCH" or "GENERAL" => code,
+        _ => "GENERAL",
+    };
+
+    private static string? ToErrorCode(ErrorType error) => error switch
+    {
+        ErrorType.None => null,
+        ErrorType.FlagNotFound => "FLAG_NOT_FOUND",
+        ErrorType.InvalidContext => "INVALID_CONTEXT",
+        ErrorType.ParseError => "PARSE_ERROR",
+        ErrorType.ProviderFatal => "PROVIDER_FATAL",
+        ErrorType.ProviderNotReady => "PROVIDER_NOT_READY",
+        ErrorType.TargetingKeyMissing => "TARGETING_KEY_MISSING",
+        ErrorType.TypeMismatch => "TYPE_MISMATCH",
+        _ => "GENERAL",
+    };
+}