From 54427be1d93b52df1c5f231f692c245543c092df Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 27 Aug 2026 17:08:08 -0400 Subject: [PATCH 1/2] fix(config): support stable OTel deployment environment --- ddtrace/opentelemetry/metric/resource.go | 12 ++-- ddtrace/opentelemetry/metric/resource_test.go | 69 +++++++++++++++++++ ddtrace/tracer/option_test.go | 10 +++ .../config/provider/otelenvconfigsource.go | 31 +++++++-- .../provider/otelenvconfigsource_test.go | 50 ++++++++++++++ 5 files changed, 163 insertions(+), 9 deletions(-) diff --git a/ddtrace/opentelemetry/metric/resource.go b/ddtrace/opentelemetry/metric/resource.go index 252e0f0c7ba..710df1ae73f 100644 --- a/ddtrace/opentelemetry/metric/resource.go +++ b/ddtrace/opentelemetry/metric/resource.go @@ -99,10 +99,11 @@ func buildDatadogResource(ctx context.Context, opts ...resource.Option) (*resour // 6. Add OTEL_RESOURCE_ATTRIBUTES (excluding reserved keys AND keys already set by DD_TAGS) // DD_TAGS has higher priority than OTEL_RESOURCE_ATTRIBUTES for custom tags excludeKeys := map[string]bool{ - "service.name": true, - "deployment.environment": true, - "service.version": true, - "host.name": true, + "service.name": true, + string(semconv.DeploymentEnvironmentNameKey): true, + "deployment.environment": true, + "service.version": true, + "host.name": true, } for key, val := range otelAttrs { if excludeKeys[key] { @@ -155,6 +156,9 @@ func environmentName(ddTags, otelAttrs map[string]string) string { if v, ok := ddTags["env"]; ok && v != "" { return v } + if v, ok := otelAttrs[string(semconv.DeploymentEnvironmentNameKey)]; ok && v != "" { + return v + } // OTEL_RESOURCE_ATTRIBUTES[deployment.environment] if v, ok := otelAttrs["deployment.environment"]; ok && v != "" { return v diff --git a/ddtrace/opentelemetry/metric/resource_test.go b/ddtrace/opentelemetry/metric/resource_test.go index 26ae32a7cf4..0711af7d7a2 100644 --- a/ddtrace/opentelemetry/metric/resource_test.go +++ b/ddtrace/opentelemetry/metric/resource_test.go @@ -105,6 +105,75 @@ func TestBuildDatadogResource_OtelFallback(t *testing.T) { assert.Equal(t, "otel-version", attrMap["service.version"]) } +func TestBuildDatadogResource_DeploymentEnvironment(t *testing.T) { + tests := []struct { + name string + ddEnv string + ddTags string + resourceAttributes string + expected string + }{ + { + name: "stable key only", + resourceAttributes: "deployment.environment.name=stable,custom=value", + expected: "stable", + }, + { + name: "legacy key only", + resourceAttributes: "deployment.environment=legacy,custom=value", + expected: "legacy", + }, + { + name: "stable key before legacy key", + resourceAttributes: "deployment.environment.name=stable,deployment.environment=legacy,custom=value", + expected: "stable", + }, + { + name: "legacy key before stable key", + resourceAttributes: "deployment.environment=legacy,deployment.environment.name=stable,custom=value", + expected: "stable", + }, + { + name: "DD_ENV takes precedence", + ddEnv: "datadog", + resourceAttributes: "deployment.environment=legacy,deployment.environment.name=stable,custom=value", + expected: "datadog", + }, + { + name: "DD_TAGS takes precedence", + ddTags: "env:datadog", + resourceAttributes: "deployment.environment=legacy,deployment.environment.name=stable,custom=value", + expected: "datadog", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(envDDEnv, tt.ddEnv) + t.Setenv(envDDTags, tt.ddTags) + t.Setenv(envOtelResourceAttributes, tt.resourceAttributes) + + res, err := buildDatadogResource(context.Background()) + require.NoError(t, err) + + attributes := make(map[string]string) + stableEnvironmentCount := 0 + for _, attr := range res.Attributes() { + key := string(attr.Key) + attributes[key] = attr.Value.AsString() + if attr.Key == semconv.DeploymentEnvironmentNameKey { + stableEnvironmentCount++ + } + } + + assert.Equal(t, tt.expected, attributes["deployment.environment.name"]) + assert.Equal(t, 1, stableEnvironmentCount) + assert.NotContains(t, attributes, "deployment.environment") + assert.Equal(t, "value", attributes["custom"]) + }) + } +} + // TestBuildDatadogResource_Hostname verifies hostname resolution priority: // 1. OTEL_RESOURCE_ATTRIBUTES[host.name] (always wins) // 2. DD_HOSTNAME (only if DD_TRACE_REPORT_HOSTNAME=true) diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 7fd469650a1..cbcb0aec631 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -1567,6 +1567,16 @@ func TestEnvConfig(t *testing.T) { assert.Equal("testing", c.internalConfig.Env()) }) + t.Run("DD_ENV takes precedence over stable OTEL_RESOURCE_ATTRIBUTES", func(t *testing.T) { + t.Setenv("DD_ENV", "datadog") + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=opentelemetry") + assert := assert.New(t) + c, err := newTestConfig() + assert.NoError(err) + + assert.Equal("datadog", c.internalConfig.Env()) + }) + t.Run("DD_TAGS", func(t *testing.T) { t.Setenv("DD_TAGS", "env:testing") assert := assert.New(t) diff --git a/internal/config/provider/otelenvconfigsource.go b/internal/config/provider/otelenvconfigsource.go index 6de3ca2426a..ac0ef9448f1 100644 --- a/internal/config/provider/otelenvconfigsource.go +++ b/internal/config/provider/otelenvconfigsource.go @@ -16,8 +16,10 @@ import ( ) const ( - ddPrefix = "config_datadog:" - otelPrefix = "config_opentelemetry:" + ddPrefix = "config_datadog:" + otelPrefix = "config_opentelemetry:" + deploymentEnvironment = "deployment.environment" + deploymentEnvironmentName = "deployment.environment.name" ) type otelEnvConfigSource struct{} @@ -92,9 +94,9 @@ var otelConfigs = map[string]*otelDDEnv{ } var ddTagsMapping = map[string]string{ - "service.name": "service", - "deployment.environment": "env", - "service.version": "version", + "service.name": "service", + deploymentEnvironment: "env", + "service.version": "version", } var unsupportedSamplerMapping = map[string]string{ @@ -200,8 +202,27 @@ func mapPropagationStyle(ot string) (string, error) { // mapDDTags maps OTEL_RESOURCE_ATTRIBUTES to DD_TAGS func mapDDTags(ot string) (string, error) { + stableEnvironment := "" + internal.ForEachStringTag(ot, internal.OtelTagsDelimeter, func(key, val string) { + if key == deploymentEnvironmentName && stableEnvironment == "" && val != "" { + stableEnvironment = val + } + }) + ddTags := make([]string, 0) + stableEnvironmentMapped := false internal.ForEachStringTag(ot, internal.OtelTagsDelimeter, func(key, val string) { + if key == deploymentEnvironmentName { + if stableEnvironment != "" && !stableEnvironmentMapped && val != "" { + ddTags = append([]string{"env" + internal.DDTagsDelimiter + stableEnvironment}, ddTags...) + stableEnvironmentMapped = true + } + return + } + if key == deploymentEnvironment && stableEnvironment != "" { + return + } + // replace otel delimiter with dd delimiter and normalize tag names if ddkey, ok := ddTagsMapping[key]; ok { ddTags = append([]string{ddkey + internal.DDTagsDelimiter + val}, ddTags...) diff --git a/internal/config/provider/otelenvconfigsource_test.go b/internal/config/provider/otelenvconfigsource_test.go index 958a4ffe1c9..0c5f0b8dd89 100644 --- a/internal/config/provider/otelenvconfigsource_test.go +++ b/internal/config/provider/otelenvconfigsource_test.go @@ -160,3 +160,53 @@ func TestOtelEnvConfigSource(t *testing.T) { assert.Equal(t, telemetry.OriginEnvVar, source.origin()) }) } + +func TestMapDDTagsDeploymentEnvironment(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + { + name: "stable key only", + in: "deployment.environment.name=stable", + want: "env:stable", + }, + { + name: "legacy key only", + in: "deployment.environment=legacy", + want: "env:legacy", + }, + { + name: "stable key before legacy key", + in: "deployment.environment.name=stable,deployment.environment=legacy", + want: "env:stable", + }, + { + name: "legacy key before stable key", + in: "deployment.environment=legacy,deployment.environment.name=stable", + want: "env:stable", + }, + { + name: "preserves mapped and unrelated attributes", + in: "custom.before=one,service.name=my-service,deployment.environment=legacy,custom.middle=two,deployment.environment.name=stable,service.version=1.2.3,custom.after=three", + want: "version:1.2.3,env:stable,service:my-service,custom.before:one,custom.middle:two,custom.after:three", + }, + { + name: "preserves ten attribute limit", + in: "deployment.environment.name=stable,tag1=value1,tag2=value2,tag3=value3,tag4=value4,tag5=value5,tag6=value6,tag7=value7,tag8=value8,tag9=value9,tag10=value10,tag11=value11", + want: "env:stable,tag1:value1,tag2:value2,tag3:value3,tag4:value4,tag5:value5,tag6:value6,tag7:value7,tag8:value8,tag9:value9", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mapDDTags(tt.in) + + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + assert.NotContains(t, got, "deployment.environment.name") + assert.NotContains(t, got, "deployment.environment") + }) + } +} From c6a0d7d393d75427c2bff15af989052d035b1f7e Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 2 Oct 2026 14:31:12 -0400 Subject: [PATCH 2/2] fix(config): align OTel environment precedence --- ddtrace/opentelemetry/metric/resource.go | 6 ++- ddtrace/opentelemetry/metric/resource_test.go | 37 +++++++++++++++- ddtrace/tracer/option.go | 7 +++- ddtrace/tracer/option_test.go | 28 +++++++++++++ internal/config/config.go | 10 ++++- .../config/provider/otelenvconfigsource.go | 42 ++++++++++++++----- .../provider/otelenvconfigsource_test.go | 40 ++++++++++++++++++ 7 files changed, 155 insertions(+), 15 deletions(-) diff --git a/ddtrace/opentelemetry/metric/resource.go b/ddtrace/opentelemetry/metric/resource.go index 710df1ae73f..650d6e776e7 100644 --- a/ddtrace/opentelemetry/metric/resource.go +++ b/ddtrace/opentelemetry/metric/resource.go @@ -39,7 +39,8 @@ const ( // // Priority order for each attribute: // - service.name: DD_SERVICE → DD_TAGS[service] → OTEL_SERVICE_NAME → OTEL_RESOURCE_ATTRIBUTES[service.name] -// - deployment.environment: DD_ENV → DD_TAGS[env] → OTEL_RESOURCE_ATTRIBUTES[deployment.environment] +// - deployment.environment: DD_ENV → DD_TAGS[env] → OTEL_RESOURCE_ATTRIBUTES[deployment.environment.name] +// → OTEL_RESOURCE_ATTRIBUTES[deployment.environment] // - service.version: DD_VERSION → DD_TAGS[version] → OTEL_RESOURCE_ATTRIBUTES[service.version] // - host.name: OTEL_RESOURCE_ATTRIBUTES[host.name] (highest priority, always used if present) // → If DD_TRACE_REPORT_HOSTNAME="true": DD_HOSTNAME → detected hostname (os.Hostname()) @@ -65,7 +66,8 @@ func buildDatadogResource(ctx context.Context, opts ...resource.Option) (*resour attrs = append(attrs, semconv.ServiceName(serviceName)) } - // 2. Environment priority: DD_ENV → DD_TAGS[env] → OTEL_RESOURCE_ATTRIBUTES[deployment.environment] + // 2. Environment priority: DD_ENV → DD_TAGS[env] → OTEL_RESOURCE_ATTRIBUTES[deployment.environment.name] + // → OTEL_RESOURCE_ATTRIBUTES[deployment.environment] envName := environmentName(ddTags, otelAttrs) if envName != "" { attrs = append(attrs, semconv.DeploymentEnvironmentNameKey.String(envName)) diff --git a/ddtrace/opentelemetry/metric/resource_test.go b/ddtrace/opentelemetry/metric/resource_test.go index 0711af7d7a2..09fea75b9b8 100644 --- a/ddtrace/opentelemetry/metric/resource_test.go +++ b/ddtrace/opentelemetry/metric/resource_test.go @@ -133,6 +133,37 @@ func TestBuildDatadogResource_DeploymentEnvironment(t *testing.T) { resourceAttributes: "deployment.environment=legacy,deployment.environment.name=stable,custom=value", expected: "stable", }, + { + name: "last duplicate stable key wins", + resourceAttributes: "deployment.environment.name=old,deployment.environment.name=new,custom=value", + expected: "new", + }, + { + name: "empty last stable key falls back to legacy key", + resourceAttributes: "deployment.environment.name=old,deployment.environment=legacy,deployment.environment.name=,custom=value", + expected: "legacy", + }, + { + name: "last duplicate legacy key wins", + resourceAttributes: "deployment.environment=old,deployment.environment=new,custom=value", + expected: "new", + }, + { + name: "empty last legacy key is unset", + resourceAttributes: "deployment.environment=old,deployment.environment=,custom=value", + expected: "", + }, + { + name: "unrelated DD_TAGS preserves stable environment", + ddTags: "team:platform", + resourceAttributes: "deployment.environment.name=stable,custom=value", + expected: "stable", + }, + { + name: "stable key overrides generic env", + resourceAttributes: "deployment.environment.name=stable,env=custom,custom=value", + expected: "stable", + }, { name: "DD_ENV takes precedence", ddEnv: "datadog", @@ -167,7 +198,11 @@ func TestBuildDatadogResource_DeploymentEnvironment(t *testing.T) { } assert.Equal(t, tt.expected, attributes["deployment.environment.name"]) - assert.Equal(t, 1, stableEnvironmentCount) + expectedEnvironmentCount := 1 + if tt.expected == "" { + expectedEnvironmentCount = 0 + } + assert.Equal(t, expectedEnvironmentCount, stableEnvironmentCount) assert.NotContains(t, attributes, "deployment.environment") assert.Equal(t, "value", attributes["custom"]) }) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 22af68d5dcb..fb8122227b7 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -269,10 +269,15 @@ func newConfig(opts ...StartOption) (*config, error) { _, globalTagsOrigin := c.internalConfig.GlobalTagsConfig().Baseline() if c.internalConfig.Env() == "" { if v, ok := globalTags["env"]; ok { - if e, ok := v.(string); ok { + if e, ok := v.(string); ok && e != "" { c.internalConfig.SetEnv(e, globalTagsOrigin, internalconfig.ProductTracer) } } + if c.internalConfig.Env() == "" { + if e := c.internalConfig.OTelResourceEnvironment(); e != "" { + c.internalConfig.SetEnv(e, telemetry.OriginEnvVar, internalconfig.ProductTracer) + } + } } if c.internalConfig.Version() == "" { if v, ok := globalTags["version"]; ok { diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index c0696cabb1f..c70d17a81af 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -1632,6 +1632,34 @@ func TestEnvConfig(t *testing.T) { assert.Equal("testing", c.internalConfig.Env()) }) + t.Run("unrelated DD_TAGS preserves stable OTEL_RESOURCE_ATTRIBUTES", func(t *testing.T) { + t.Setenv("DD_TAGS", "team:platform") + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=opentelemetry") + assert := assert.New(t) + c, err := newTestConfig() + + assert.NoError(err) + assert.Equal("opentelemetry", c.internalConfig.Env()) + }) + + t.Run("last duplicate legacy OTEL_RESOURCE_ATTRIBUTES key wins", func(t *testing.T) { + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=old,deployment.environment=new") + assert := assert.New(t) + c, err := newTestConfig() + + assert.NoError(err) + assert.Equal("new", c.internalConfig.Env()) + }) + + t.Run("stable OTEL_RESOURCE_ATTRIBUTES key overrides generic env", func(t *testing.T) { + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=stable,env=custom") + assert := assert.New(t) + c, err := newTestConfig() + + assert.NoError(err) + assert.Equal("stable", c.internalConfig.Env()) + }) + t.Run("override-chain", func(t *testing.T) { assert := assert.New(t) c, err := newTestConfig() diff --git a/internal/config/config.go b/internal/config/config.go index 7add855e276..d1404fdd8c2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -97,7 +97,8 @@ type Config struct { // configured service get the version tag. universalVersion bool // env contains the environment that this application will run under. - env string + env string + otelResourceEnvironment string // site specifies the Datadog site to send data to site string // serviceMappings holds a set of service mappings to dynamically rename services. @@ -376,6 +377,7 @@ func loadConfig() *Config { cfg.version = p.GetString("DD_VERSION", "") cfg.universalVersion = p.GetBool("DD_TRACE_UNIVERSAL_VERSION_ENABLED", false) cfg.env = p.GetString("DD_ENV", "") + cfg.otelResourceEnvironment = provider.OTelResourceEnvironment() cfg.site = p.GetString("DD_SITE", "datadoghq.com") cfg.serviceMappings = p.GetMap("DD_SERVICE_MAPPING", nil, internal.DDTagsDelimiter) cfg.runtimeMetrics = p.GetBool("DD_RUNTIME_METRICS_ENABLED", false) @@ -1357,6 +1359,12 @@ func (c *Config) Env() string { return c.env } +func (c *Config) OTelResourceEnvironment() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.otelResourceEnvironment +} + func (c *Config) SetEnv(env string, origin telemetry.Origin, product ...Product) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/config/provider/otelenvconfigsource.go b/internal/config/provider/otelenvconfigsource.go index ac0ef9448f1..abbc8632fd0 100644 --- a/internal/config/provider/otelenvconfigsource.go +++ b/internal/config/provider/otelenvconfigsource.go @@ -202,24 +202,26 @@ func mapPropagationStyle(ot string) (string, error) { // mapDDTags maps OTEL_RESOURCE_ATTRIBUTES to DD_TAGS func mapDDTags(ot string) (string, error) { - stableEnvironment := "" - internal.ForEachStringTag(ot, internal.OtelTagsDelimeter, func(key, val string) { - if key == deploymentEnvironmentName && stableEnvironment == "" && val != "" { - stableEnvironment = val - } - }) + stableEnvironment, legacyEnvironment := deploymentEnvironmentValues(ot) ddTags := make([]string, 0) - stableEnvironmentMapped := false + environmentMapped := false internal.ForEachStringTag(ot, internal.OtelTagsDelimeter, func(key, val string) { if key == deploymentEnvironmentName { - if stableEnvironment != "" && !stableEnvironmentMapped && val != "" { + if stableEnvironment != "" && !environmentMapped && val != "" { ddTags = append([]string{"env" + internal.DDTagsDelimiter + stableEnvironment}, ddTags...) - stableEnvironmentMapped = true + environmentMapped = true } return } - if key == deploymentEnvironment && stableEnvironment != "" { + if key == deploymentEnvironment { + if stableEnvironment == "" && legacyEnvironment != "" && !environmentMapped { + ddTags = append([]string{"env" + internal.DDTagsDelimiter + legacyEnvironment}, ddTags...) + environmentMapped = true + } + return + } + if key == "env" && (stableEnvironment != "" || legacyEnvironment != "") { return } @@ -238,3 +240,23 @@ func mapDDTags(ot string) (string, error) { return strings.Join(ddTags, ","), nil } + +func deploymentEnvironmentValues(ot string) (stable, legacy string) { + internal.ForEachStringTag(ot, internal.OtelTagsDelimeter, func(key, val string) { + switch key { + case deploymentEnvironmentName: + stable = val + case deploymentEnvironment: + legacy = val + } + }) + return stable, legacy +} + +func OTelResourceEnvironment() string { + stable, legacy := deploymentEnvironmentValues(env.Get("OTEL_RESOURCE_ATTRIBUTES")) + if stable != "" { + return stable + } + return legacy +} diff --git a/internal/config/provider/otelenvconfigsource_test.go b/internal/config/provider/otelenvconfigsource_test.go index 0c5f0b8dd89..7b491161cef 100644 --- a/internal/config/provider/otelenvconfigsource_test.go +++ b/internal/config/provider/otelenvconfigsource_test.go @@ -187,6 +187,46 @@ func TestMapDDTagsDeploymentEnvironment(t *testing.T) { in: "deployment.environment=legacy,deployment.environment.name=stable", want: "env:stable", }, + { + name: "last duplicate stable key wins", + in: "deployment.environment.name=old,deployment.environment.name=new", + want: "env:new", + }, + { + name: "empty last stable key falls back to legacy key", + in: "deployment.environment.name=old,deployment.environment=legacy,deployment.environment.name=", + want: "env:legacy", + }, + { + name: "last duplicate legacy key wins", + in: "deployment.environment=old,deployment.environment=new", + want: "env:new", + }, + { + name: "empty last legacy key is unset", + in: "deployment.environment=old,deployment.environment=", + want: "", + }, + { + name: "stable key overrides generic env before it", + in: "env=custom,deployment.environment.name=stable", + want: "env:stable", + }, + { + name: "stable key overrides generic env after it", + in: "deployment.environment.name=stable,env=custom", + want: "env:stable", + }, + { + name: "legacy key overrides generic env", + in: "deployment.environment=legacy,env=custom", + want: "env:legacy", + }, + { + name: "preserves generic env without deployment environment", + in: "env=custom", + want: "env:custom", + }, { name: "preserves mapped and unrelated attributes", in: "custom.before=one,service.name=my-service,deployment.environment=legacy,custom.middle=two,deployment.environment.name=stable,service.version=1.2.3,custom.after=three",