From f8c9679c647186c4d2ab4e0746e3c31b7cc246f0 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 27 Aug 2026 17:41:56 -0400 Subject: [PATCH 1/2] fix(config): map OTel deployment environment --- .../src/core/configuration/configuration.rs | 234 +++++++++++++++++- datadog-opentelemetry/src/lib.rs | 56 ++++- .../src/mappings/transform/mod.rs | 5 + .../src/mappings/transform/transform_tests.rs | 66 +++++ datadog-opentelemetry/src/otlp_utils.rs | 77 +++++- 5 files changed, 418 insertions(+), 20 deletions(-) diff --git a/datadog-opentelemetry/src/core/configuration/configuration.rs b/datadog-opentelemetry/src/core/configuration/configuration.rs index f621f123..5afaafa2 100644 --- a/datadog-opentelemetry/src/core/configuration/configuration.rs +++ b/datadog-opentelemetry/src/core/configuration/configuration.rs @@ -1079,6 +1079,7 @@ pub struct Config { // # Service tagging service: ConfigItemWithOverride, env: ConfigItem>, + otel_resource_environment: Option, version: ConfigItem>, // # Agent @@ -1296,6 +1297,31 @@ impl Config { } let cisu = ConfigItemSourceUpdater { sources }; + let env = cisu.update_string(default.env, Some); + let mut parsed_otel_resource_attributes = sources + .get_parse::(SupportedConfigurations::OTEL_RESOURCE_ATTRIBUTES); + let mut otel_resource_environment = None; + + if let Some(config_key) = parsed_otel_resource_attributes.value.as_mut() { + let OtelResourceAttributes(attributes) = &mut config_key.value; + otel_resource_environment = attributes + .iter() + .find(|(key, value)| key == "deployment.environment.name" && !value.is_empty()) + .or_else(|| { + attributes + .iter() + .find(|(key, value)| key == "deployment.environment" && !value.is_empty()) + }) + .map(|(_, value)| value.clone()); + attributes.retain(|(key, _)| { + key != "deployment.environment.name" && key != "deployment.environment" + }); + } + let otel_resource_attributes = cisu.apply_result( + default.otel_resource_attributes, + parsed_otel_resource_attributes, + |OtelResourceAttributes(attributes)| attributes, + ); Self { runtime_id: default.runtime_id, @@ -1307,15 +1333,13 @@ impl Config { SupportedConfigurations::OTEL_SERVICE_NAME, ServiceName::Configured, ), - env: cisu.update_string(default.env, Some), + env, + otel_resource_environment, version: cisu.update_string(default.version, Some), // TODO(paullgdc): tags should be merged, not replaced global_tags: cisu .update_parsed_with_transform(default.global_tags, |DdKeyValueTags(tags)| tags), - otel_resource_attributes: cisu.update_parsed_with_transform( - default.otel_resource_attributes, - |OtelResourceAttributes(attrs)| attrs, - ), + otel_resource_attributes, otel_metrics_exporter: cisu.update_string(default.otel_metrics_exporter, Cow::Owned), otel_metrics_temporality_preference: cisu.update_string( default.otel_metrics_temporality_preference, @@ -1522,9 +1546,17 @@ impl Config { /// Returns the configured environment name (e.g., "production", "staging"). pub fn env(&self) -> Option<&str> { + self.explicit_env().or(self.otel_resource_environment()) + } + + pub(crate) fn explicit_env(&self) -> Option<&str> { self.env.value().as_deref() } + pub(crate) fn otel_resource_environment(&self) -> Option<&str> { + self.otel_resource_environment.as_deref() + } + /// Returns the configured application version. pub fn version(&self) -> Option<&str> { self.version.value().as_deref() @@ -2012,6 +2044,7 @@ impl std::fmt::Debug for Config { .field("language_version", &self.language_version) .field("service", &self.service) .field("env", &self.env) + .field("otel_resource_environment", &self.otel_resource_environment) .field("version", &self.version) .field("global_tags", &self.global_tags) .field("trace_agent_url", &self.trace_agent_url) @@ -2058,6 +2091,7 @@ fn default_config() -> Config { Config { runtime_id: Config::process_runtime_id(), env: ConfigItem::new(SupportedConfigurations::DD_ENV, None), + otel_resource_environment: None, // TODO(paullgdc): Default service naming detection, probably from arg0 service: ConfigItemWithOverride::new_calculated( SupportedConfigurations::DD_SERVICE, @@ -2907,13 +2941,41 @@ impl ConfigBuilder { #[cfg(test)] mod tests { use libdd_telemetry::data::ConfigurationOrigin; + use opentelemetry::KeyValue; + use opentelemetry_sdk::Resource; use std::collections::HashMap; use super::Config; use super::*; use crate::core::configuration::sources::{CompositeSource, ConfigSourceOrigin, HashMapSource}; + use crate::mappings::transform_tests::{test_cases, test_span_to_sdk_span}; + use crate::mappings::{otel_span_to_dd_span, SpanStr}; use crate::propagation::config::{get_extractors, get_injectors}; + fn assert_resource_environment_on_span(config: Config, resource: Resource, expected_env: &str) { + let resource = crate::create_dd_resource(resource, &config); + let input_span = test_cases().remove(0).input_span; + let output = otel_span_to_dd_span(&test_span_to_sdk_span(&input_span), &resource); + + assert_eq!( + output + .meta + .get(&SpanStr::from_str("env")) + .map(SpanStr::as_str), + Some(expected_env) + ); + assert_eq!( + output + .meta + .get(&SpanStr::from_str("unrelated")) + .map(SpanStr::as_str), + Some("value") + ); + for source_key in ["deployment.environment.name", "deployment.environment"] { + assert!(!output.meta.contains_key(&SpanStr::from_str(source_key))); + } + } + #[test] fn test_config_from_source() { let mut sources = CompositeSource::new(); @@ -3017,6 +3079,168 @@ mod tests { assert!(config.service_is_default()); } + #[test] + fn test_otel_resource_deployment_environment_mapping() { + struct TestCase { + name: &'static str, + dd_env: Option<&'static str>, + resource_attributes: &'static str, + expected_env: &'static str, + } + + for case in [ + TestCase { + name: "stable only", + dd_env: None, + resource_attributes: + "first=one,deployment.environment.name=stable,second=two", + expected_env: "stable", + }, + TestCase { + name: "legacy only", + dd_env: None, + resource_attributes: "first=one,deployment.environment=legacy,second=two", + expected_env: "legacy", + }, + TestCase { + name: "stable before legacy", + dd_env: None, + resource_attributes: "first=one,deployment.environment.name=stable,deployment.environment=legacy,second=two", + expected_env: "stable", + }, + TestCase { + name: "legacy before stable", + dd_env: None, + resource_attributes: "first=one,deployment.environment=legacy,deployment.environment.name=stable,second=two", + expected_env: "stable", + }, + TestCase { + name: "DD_ENV takes precedence", + dd_env: Some("datadog"), + resource_attributes: "first=one,deployment.environment=legacy,deployment.environment.name=stable,second=two", + expected_env: "datadog", + }, + ] { + let mut values = vec![("OTEL_RESOURCE_ATTRIBUTES", case.resource_attributes)]; + if let Some(dd_env) = case.dd_env { + values.push(("DD_ENV", dd_env)); + } + + let mut sources = CompositeSource::new(); + sources.add_source(HashMapSource::from_iter( + values, + ConfigSourceOrigin::EnvVar, + )); + let config = Config::builder_with_sources(&sources).build(); + + assert_eq!(config.env(), Some(case.expected_env), "{} env", case.name); + assert_eq!( + config.otel_resource_attributes().collect::>(), + vec![("first", "one"), ("second", "two")], + "{} resource attributes", + case.name + ); + } + } + + #[test] + fn test_dd_env_overrides_otel_resource_environment_on_span() { + let mut sources = CompositeSource::new(); + sources.add_source(HashMapSource::from_iter( + [("DD_ENV", "datadog")], + ConfigSourceOrigin::EnvVar, + )); + let config = Config::builder_with_sources(&sources).build(); + let resource = Resource::builder_empty() + .with_attributes([ + KeyValue::new("deployment.environment", "legacy"), + KeyValue::new("deployment.environment.name", "stable"), + KeyValue::new("unrelated", "value"), + ]) + .build(); + assert_resource_environment_on_span(config, resource, "datadog"); + } + + #[test] + fn test_resource_environment_overrides_otel_resource_attributes_on_span() { + let mut sources = CompositeSource::new(); + sources.add_source(HashMapSource::from_iter( + [( + "OTEL_RESOURCE_ATTRIBUTES", + "deployment.environment.name=fallback", + )], + ConfigSourceOrigin::EnvVar, + )); + let config = Config::builder_with_sources(&sources).build(); + let resource = Resource::builder_empty() + .with_attributes([ + KeyValue::new("deployment.environment", "provided"), + KeyValue::new("unrelated", "value"), + ]) + .build(); + assert_resource_environment_on_span(config, resource, "provided"); + } + + #[test] + fn test_datadog_tags_environment_overrides_otel_resource_attributes() { + let mut sources = CompositeSource::new(); + sources.add_source(HashMapSource::from_iter( + [ + ("DD_TAGS", "env:datadog"), + ( + "OTEL_RESOURCE_ATTRIBUTES", + "deployment.environment.name=fallback", + ), + ], + ConfigSourceOrigin::EnvVar, + )); + let config = Config::builder_with_sources(&sources).build(); + let resource = crate::otlp_utils::build_otel_resource(&config, None); + + assert_eq!( + resource.get(&opentelemetry::Key::from_static_str( + "deployment.environment.name" + )), + Some(opentelemetry::Value::from("datadog")) + ); + assert_eq!( + resource.get(&opentelemetry::Key::from_static_str( + "deployment.environment" + )), + None + ); + } + + #[test] + fn test_resource_environment_overrides_otel_resource_attributes_for_otlp() { + let mut sources = CompositeSource::new(); + sources.add_source(HashMapSource::from_iter( + [( + "OTEL_RESOURCE_ATTRIBUTES", + "deployment.environment.name=fallback", + )], + ConfigSourceOrigin::EnvVar, + )); + let config = Config::builder_with_sources(&sources).build(); + let resource = Resource::builder_empty() + .with_attribute(KeyValue::new("deployment.environment", "provided")) + .build(); + let resource = crate::otlp_utils::build_otel_resource(&config, Some(resource)); + + assert_eq!( + resource.get(&opentelemetry::Key::from_static_str( + "deployment.environment.name" + )), + Some(opentelemetry::Value::from("provided")) + ); + assert_eq!( + resource.get(&opentelemetry::Key::from_static_str( + "deployment.environment" + )), + None + ); + } + #[test] fn test_sampling_rules() { let mut sources = CompositeSource::new(); diff --git a/datadog-opentelemetry/src/lib.rs b/datadog-opentelemetry/src/lib.rs index 0018b3ae..8768dfd0 100644 --- a/datadog-opentelemetry/src/lib.rs +++ b/datadog-opentelemetry/src/lib.rs @@ -324,11 +324,34 @@ mod telemetry_metrics_exporter; mod text_map_propagator; mod trace_id; -use std::sync::{Arc, RwLock}; +use std::{ + borrow::Cow, + sync::{Arc, RwLock}, +}; use opentelemetry::{Key, KeyValue, Value}; use opentelemetry_sdk::{trace::SdkTracerProvider, Resource}; -use opentelemetry_semantic_conventions::resource::{DEPLOYMENT_ENVIRONMENT_NAME, SERVICE_NAME}; +use opentelemetry_semantic_conventions::resource::SERVICE_NAME; + +pub(crate) const DEPLOYMENT_ENVIRONMENT_NAME: &str = + opentelemetry_semantic_conventions::resource::DEPLOYMENT_ENVIRONMENT_NAME; +pub(crate) const LEGACY_DEPLOYMENT_ENVIRONMENT: &str = "deployment.environment"; + +pub(crate) fn is_environment_attribute(key: &str) -> bool { + key == DEPLOYMENT_ENVIRONMENT_NAME || key == LEGACY_DEPLOYMENT_ENVIRONMENT +} + +pub(crate) fn resource_environment(resource: &Resource) -> Option> { + [DEPLOYMENT_ENVIRONMENT_NAME, LEGACY_DEPLOYMENT_ENVIRONMENT] + .into_iter() + .find_map(|key| { + resource + .iter() + .find(|(resource_key, _)| resource_key.as_str() == key) + .map(|(_, value)| value.as_str()) + .filter(|value| !value.is_empty()) + }) +} use crate::{ core::configuration::{Config, RemoteConfigUpdate}, @@ -629,7 +652,21 @@ fn merge_resource>( builder.build() } -fn create_dd_resource(resource: Resource, cfg: &Config) -> Resource { +fn without_environment_attributes(resource: &Resource) -> Resource { + let attributes = resource + .iter() + .filter(|(key, _)| !is_environment_attribute(key.as_str())) + .map(|(key, value)| KeyValue::new(key.clone(), value.clone())); + let mut builder = Resource::builder_empty(); + if let Some(schema_url) = resource.schema_url() { + builder = builder.with_schema_url(attributes, schema_url.to_string()); + } else { + builder = builder.with_attributes(attributes); + } + builder.build() +} + +fn create_dd_resource(mut resource: Resource, cfg: &Config) -> Resource { let otel_service_name: Option = resource.get(&Key::from_static_str(SERVICE_NAME)); // Collect attributes to add @@ -662,11 +699,14 @@ fn create_dd_resource(resource: Resource, cfg: &Config) -> Resource { )); } - // Handle environment - add it if configured and not already present - if let Some(env) = cfg.env() { - let otel_env: Option = - resource.get(&Key::from_static_str(DEPLOYMENT_ENVIRONMENT_NAME)); - if otel_env.is_none() { + if let Some(env) = cfg.explicit_env() { + resource = without_environment_attributes(&resource); + attributes.push(( + Key::from_static_str(DEPLOYMENT_ENVIRONMENT_NAME), + Value::from(env.to_string()), + )); + } else if resource_environment(&resource).is_none() { + if let Some(env) = cfg.otel_resource_environment() { attributes.push(( Key::from_static_str(DEPLOYMENT_ENVIRONMENT_NAME), Value::from(env.to_string()), diff --git a/datadog-opentelemetry/src/mappings/transform/mod.rs b/datadog-opentelemetry/src/mappings/transform/mod.rs index 1a21d6c9..efe043a5 100644 --- a/datadog-opentelemetry/src/mappings/transform/mod.rs +++ b/datadog-opentelemetry/src/mappings/transform/mod.rs @@ -572,6 +572,11 @@ pub fn otel_span_to_dd_span<'a>( } for (key, value) in otel_resource.iter() { + if key.as_str() == DEPLOYMENT_ENVIRONMENT_NAME.key() + || key.as_str() == DEPLOYMENT_ENVIRONMENT.key() + { + continue; + } set_meta_otlp_with_semconv_mappings(key.as_str(), value, &mut dd_span); } diff --git a/datadog-opentelemetry/src/mappings/transform/transform_tests.rs b/datadog-opentelemetry/src/mappings/transform/transform_tests.rs index 75ddc484..8ffc8111 100644 --- a/datadog-opentelemetry/src/mappings/transform/transform_tests.rs +++ b/datadog-opentelemetry/src/mappings/transform/transform_tests.rs @@ -769,6 +769,72 @@ mod tests { } } + #[test] + fn test_otel_span_to_dd_span_consumes_environment_resource_attributes() { + for (name, resource_attributes, expected_env) in [ + ( + "stable only", + vec![("deployment.environment.name", "stable")], + "stable", + ), + ( + "legacy only", + vec![("deployment.environment", "legacy")], + "legacy", + ), + ( + "stable before legacy", + vec![ + ("deployment.environment.name", "stable"), + ("deployment.environment", "legacy"), + ], + "stable", + ), + ( + "legacy before stable", + vec![ + ("deployment.environment", "legacy"), + ("deployment.environment.name", "stable"), + ], + "stable", + ), + ] { + let resource = Resource::builder_empty() + .with_attributes( + resource_attributes + .into_iter() + .chain([("unrelated", "value")]) + .map(|(key, value)| KeyValue::new(key, value)), + ) + .build(); + let input_span = test_cases().remove(0).input_span; + let output = otel_span_to_dd_span(&test_span_to_sdk_span(&input_span), &resource); + + assert_eq!( + output + .meta + .get(&CowStr::from_str("env")) + .map(CowStr::as_str), + Some(expected_env), + "{name} environment" + ); + assert_eq!( + output + .meta + .get(&CowStr::from_str("unrelated")) + .map(CowStr::as_str), + Some("value"), + "{name} unrelated resource attribute" + ); + for source_key in ["deployment.environment.name", "deployment.environment"] { + assert!( + !output.meta.contains_key(&CowStr::from_str(source_key)), + "{name} retained {source_key}" + ); + } + } + } + #[track_caller] fn hashmap_diff<'a, V: PartialEq + Debug>( output: &VecMap, V>, diff --git a/datadog-opentelemetry/src/otlp_utils.rs b/datadog-opentelemetry/src/otlp_utils.rs index cd8190b1..70e69c0f 100644 --- a/datadog-opentelemetry/src/otlp_utils.rs +++ b/datadog-opentelemetry/src/otlp_utils.rs @@ -8,6 +8,13 @@ feature = "logs-http" ))] use opentelemetry_sdk::Resource; +#[cfg(any( + feature = "metrics-grpc", + feature = "metrics-http", + feature = "logs-grpc", + feature = "logs-http" +))] +use std::borrow::Cow; #[cfg(any( feature = "metrics-grpc", @@ -15,7 +22,10 @@ use opentelemetry_sdk::Resource; feature = "logs-grpc", feature = "logs-http" ))] -use crate::{configuration::OtlpProtocol, core::configuration::Config}; +use crate::{ + configuration::OtlpProtocol, core::configuration::Config, is_environment_attribute, + resource_environment, DEPLOYMENT_ENVIRONMENT_NAME, +}; #[cfg(any( feature = "metrics-grpc", @@ -47,6 +57,16 @@ pub(crate) const DEFAULT_OTLP_HTTP_PORT: u16 = 4318; ))] pub(crate) fn build_otel_resource(config: &Config, resource: Option) -> Resource { let mut resource_attrs: Vec = Vec::new(); + let environment = config + .explicit_env() + .map(Cow::Borrowed) + .or_else(|| resource.as_ref().and_then(resource_environment)) + .or_else(|| { + config + .global_tags() + .find_map(|(key, value)| (key == "env").then(|| Cow::Owned(value.to_string()))) + }) + .or_else(|| config.otel_resource_environment().map(Cow::Borrowed)); for (key, value) in config.otel_resource_attributes() { resource_attrs.push(opentelemetry::KeyValue::new( @@ -58,7 +78,7 @@ pub(crate) fn build_otel_resource(config: &Config, resource: Option) - for (key, value) in config.global_tags() { let otel_key = match key { "service" => "service.name", - "env" => "deployment.environment", + "env" => DEPLOYMENT_ENVIRONMENT_NAME, "version" => "service.version", _ => key, }; @@ -70,7 +90,7 @@ pub(crate) fn build_otel_resource(config: &Config, resource: Option) - )); } - if let Some(resource) = resource { + if let Some(resource) = resource.as_ref() { for (k, v) in resource.iter() { resource_attrs.push(opentelemetry::KeyValue::new(k.clone(), v.clone())); } @@ -92,11 +112,11 @@ pub(crate) fn build_otel_resource(config: &Config, resource: Option) - )); } - if let Some(env) = config.env() { - resource_attrs.retain(|kv| kv.key.as_str() != "deployment.environment"); + resource_attrs.retain(|kv| !is_environment_attribute(kv.key.as_str())); + if let Some(env) = environment { resource_attrs.push(opentelemetry::KeyValue::new( - "deployment.environment", - env.to_string(), + DEPLOYMENT_ENVIRONMENT_NAME, + env.into_owned(), )); } @@ -219,3 +239,46 @@ pub(crate) fn get_otlp_logs_timeout(config: &Config) -> u32 { } config.otlp_timeout() } + +#[cfg(all( + test, + any( + feature = "metrics-grpc", + feature = "metrics-http", + feature = "logs-grpc", + feature = "logs-http" + ) +))] +mod tests { + use opentelemetry::{Key, KeyValue, Value}; + + use super::*; + use crate::LEGACY_DEPLOYMENT_ENVIRONMENT; + + #[test] + fn configured_environment_replaces_resource_aliases() { + let config = Config::builder().set_env("datadog".to_string()).build(); + let resource = Resource::builder_empty() + .with_attributes([ + KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, "stable"), + KeyValue::new(LEGACY_DEPLOYMENT_ENVIRONMENT, "legacy"), + KeyValue::new("unrelated", "value"), + ]) + .build(); + + let resource = build_otel_resource(&config, Some(resource)); + + assert_eq!( + resource.get(&Key::from_static_str(DEPLOYMENT_ENVIRONMENT_NAME)), + Some(Value::from("datadog")) + ); + assert_eq!( + resource.get(&Key::from_static_str(LEGACY_DEPLOYMENT_ENVIRONMENT)), + None + ); + assert_eq!( + resource.get(&Key::from_static_str("unrelated")), + Some(Value::from("value")) + ); + } +} From 0bddd9cb87ae32549fe101713102c63ede3d7eea Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 2 Oct 2026 14:31:14 -0400 Subject: [PATCH 2/2] fix(config): align OTel environment precedence --- .../src/core/configuration/configuration.rs | 241 +++++++++++++++--- .../src/core/configuration/mod.rs | 17 +- datadog-opentelemetry/src/lib.rs | 24 +- .../src/mappings/cached_config.rs | 1 + datadog-opentelemetry/src/otlp_utils.rs | 41 ++- datadog-opentelemetry/src/span_processor.rs | 39 +-- 6 files changed, 284 insertions(+), 79 deletions(-) diff --git a/datadog-opentelemetry/src/core/configuration/configuration.rs b/datadog-opentelemetry/src/core/configuration/configuration.rs index 2a3643f9..4822c408 100644 --- a/datadog-opentelemetry/src/core/configuration/configuration.rs +++ b/datadog-opentelemetry/src/core/configuration/configuration.rs @@ -24,10 +24,29 @@ use crate::core::configuration::sources::{ use crate::core::configuration::supported_configurations::SupportedConfigurations; use crate::core::log::LevelFilter; use crate::core::telemetry; -use crate::{dd_error, dd_warn}; +use crate::{ + dd_error, dd_warn, is_environment_attribute, DEPLOYMENT_ENVIRONMENT_NAME, + LEGACY_DEPLOYMENT_ENVIRONMENT, +}; pub(crate) use crate::core::configuration::sources::ConfigParser; +fn environment_from_attributes(attributes: &[(String, String)]) -> Option<&str> { + attributes + .iter() + .rev() + .find(|(key, _)| key == DEPLOYMENT_ENVIRONMENT_NAME) + .filter(|(_, value)| !value.is_empty()) + .or_else(|| { + attributes + .iter() + .rev() + .find(|(key, _)| key == LEGACY_DEPLOYMENT_ENVIRONMENT) + .filter(|(_, value)| !value.is_empty()) + }) + .map(|(_, value)| value.as_str()) +} + /// Different types of remote configuration updates that can trigger callbacks #[derive(Debug, Clone)] pub enum RemoteConfigUpdate { @@ -1125,7 +1144,7 @@ pub struct Config { // # Service tagging service: ConfigItemWithOverride, env: ConfigItem>, - otel_resource_environment: Option, + calculated_environment: OnceLock, version: ConfigItem>, // # Agent @@ -1370,31 +1389,6 @@ impl Config { } let cisu = ConfigItemSourceUpdater { sources }; - let env = cisu.update_string(default.env, Some); - let mut parsed_otel_resource_attributes = sources - .get_parse::(SupportedConfigurations::OTEL_RESOURCE_ATTRIBUTES); - let mut otel_resource_environment = None; - - if let Some(config_key) = parsed_otel_resource_attributes.value.as_mut() { - let attributes = &mut config_key.value; - otel_resource_environment = attributes - .iter() - .find(|(key, value)| key == "deployment.environment.name" && !value.is_empty()) - .or_else(|| { - attributes - .iter() - .find(|(key, value)| key == "deployment.environment" && !value.is_empty()) - }) - .map(|(_, value)| value.clone()); - attributes.retain(|(key, _)| { - key != "deployment.environment.name" && key != "deployment.environment" - }); - } - let otel_resource_attributes = cisu.apply_result( - default.otel_resource_attributes, - parsed_otel_resource_attributes, - |attributes| attributes, - ); Self { runtime_id: default.runtime_id, @@ -1406,12 +1400,13 @@ impl Config { SupportedConfigurations::OTEL_SERVICE_NAME, ServiceName::Configured, ), - env, - otel_resource_environment, + env: cisu.update_string(default.env, Some), + calculated_environment: default.calculated_environment, version: cisu.update_string(default.version, Some), // TODO(paullgdc): tags should be merged, not replaced global_tags: cisu.update_parse_custom::(default.global_tags), - otel_resource_attributes, + otel_resource_attributes: cisu + .update_parse_custom::(default.otel_resource_attributes), otel_metrics_exporter: cisu.update_string(default.otel_metrics_exporter, Cow::Owned), otel_metrics_temporality_preference: cisu.update_string( default.otel_metrics_temporality_preference, @@ -1640,7 +1635,10 @@ impl Config { /// Returns the configured environment name (e.g., "production", "staging"). pub fn env(&self) -> Option<&str> { - self.explicit_env().or(self.otel_resource_environment()) + self.explicit_env() + .or_else(|| self.calculated_environment.get().map(String::as_str)) + .or_else(|| self.global_tags_environment()) + .or_else(|| self.otel_resource_environment()) } pub(crate) fn explicit_env(&self) -> Option<&str> { @@ -1648,7 +1646,23 @@ impl Config { } pub(crate) fn otel_resource_environment(&self) -> Option<&str> { - self.otel_resource_environment.as_deref() + environment_from_attributes(self.otel_resource_attributes.value()) + } + + pub(crate) fn global_tags_environment(&self) -> Option<&str> { + self.global_tags + .value() + .iter() + .rev() + .find(|(key, _)| key == "env") + .filter(|(_, value)| !value.is_empty()) + .map(|(_, value)| value.as_str()) + } + + pub(crate) fn set_calculated_environment(&self, environment: Option) { + if let Some(environment) = environment { + let _ = self.calculated_environment.set(environment); + } } /// Returns the configured application version. @@ -1669,6 +1683,7 @@ impl Config { self.otel_resource_attributes .value() .iter() + .filter(|(key, _)| !is_environment_attribute(key)) .map(|attr| (attr.0.as_str(), attr.1.as_str())) } @@ -2162,7 +2177,7 @@ impl std::fmt::Debug for Config { .field("language_version", &self.language_version) .field("service", &self.service) .field("env", &self.env) - .field("otel_resource_environment", &self.otel_resource_environment) + .field("calculated_environment", &self.calculated_environment) .field("version", &self.version) .field("global_tags", &self.global_tags) .field("trace_agent_url", &self.trace_agent_url) @@ -2209,7 +2224,7 @@ fn default_config() -> Config { Config { runtime_id: Config::process_runtime_id(), env: ConfigItem::new(SupportedConfigurations::DD_ENV, None), - otel_resource_environment: None, + calculated_environment: OnceLock::new(), // TODO(paullgdc): Default service naming detection, probably from arg0 service: ConfigItemWithOverride::new_calculated( SupportedConfigurations::DD_SERVICE, @@ -3194,21 +3209,54 @@ impl ConfigBuilder { #[cfg(test)] mod tests { use libdd_telemetry::data::ConfigurationOrigin; + use opentelemetry::trace::{SpanContext, SpanId, TraceFlags, TraceId}; use opentelemetry::KeyValue; + use opentelemetry_sdk::trace::{SpanData, SpanEvents, SpanLinks}; use opentelemetry_sdk::Resource; + use std::borrow::Cow; use std::collections::HashMap; + use std::time::SystemTime; use super::Config; use super::*; use crate::core::configuration::sources::{CompositeSource, ConfigSourceOrigin, HashMapSource}; - use crate::mappings::transform_tests::{test_cases, test_span_to_sdk_span}; - use crate::mappings::{otel_span_to_dd_span, SpanStr}; + use crate::ddtrace_transform::otel_trace_chunk_to_dd_trace_chunk; + use crate::mappings::{CachedConfig, SpanStr}; use crate::propagation::config::{get_extractors, get_injectors}; + fn test_span_data() -> SpanData { + SpanData { + span_context: SpanContext::new( + TraceId::from_bytes([1; 16]), + SpanId::from_bytes([1; 8]), + TraceFlags::default(), + false, + Default::default(), + ), + parent_span_id: SpanId::INVALID, + span_kind: opentelemetry::trace::SpanKind::Internal, + name: Cow::Borrowed("test_span"), + start_time: SystemTime::now(), + end_time: SystemTime::now(), + attributes: Vec::new(), + dropped_attributes_count: 0, + events: SpanEvents::default(), + links: SpanLinks::default(), + status: opentelemetry::trace::Status::Unset, + instrumentation_scope: Default::default(), + parent_span_is_remote: false, + } + } + fn assert_resource_environment_on_span(config: Config, resource: Resource, expected_env: &str) { - let resource = crate::create_dd_resource(resource, &config); - let input_span = test_cases().remove(0).input_span; - let output = otel_span_to_dd_span(&test_span_to_sdk_span(&input_span), &resource); + let resource = crate::resolve_dd_resource(resource, &config); + assert_eq!(config.env(), Some(expected_env)); + + let span = test_span_data(); + let cached_config = CachedConfig::new(&config); + let output = + otel_trace_chunk_to_dd_trace_chunk(&cached_config, std::iter::once(&span), &resource) + .remove(0); assert_eq!( output @@ -3367,6 +3415,18 @@ mod tests { resource_attributes: "first=one,deployment.environment=legacy,deployment.environment.name=stable,second=two", expected_env: "stable", }, + TestCase { + name: "last duplicate stable key wins", + dd_env: None, + resource_attributes: "first=one,deployment.environment.name=old,deployment.environment.name=new,second=two", + expected_env: "new", + }, + TestCase { + name: "empty last stable key falls back to legacy key", + dd_env: None, + resource_attributes: "first=one,deployment.environment.name=old,deployment.environment=legacy,deployment.environment.name=,second=two", + expected_env: "legacy", + }, TestCase { name: "DD_ENV takes precedence", dd_env: Some("datadog"), @@ -3396,11 +3456,89 @@ mod tests { } } + #[test] + fn test_code_otel_resource_deployment_environment_mapping() { + struct TestCase { + name: &'static str, + source_attributes: Option<&'static str>, + code_attributes: &'static [(&'static str, &'static str)], + expected_env: Option<&'static str>, + } + + for case in [ + TestCase { + name: "stable only", + source_attributes: None, + code_attributes: &[("deployment.environment.name", "stable")], + expected_env: Some("stable"), + }, + TestCase { + name: "legacy only", + source_attributes: None, + code_attributes: &[("deployment.environment", "legacy")], + expected_env: Some("legacy"), + }, + TestCase { + name: "code replaces source environment", + source_attributes: Some("deployment.environment.name=source"), + code_attributes: &[("deployment.environment.name", "code")], + expected_env: Some("code"), + }, + TestCase { + name: "code clears source environment", + source_attributes: Some("deployment.environment.name=source"), + code_attributes: &[("unrelated", "value")], + expected_env: None, + }, + ] { + let mut sources = CompositeSource::new(); + if let Some(attributes) = case.source_attributes { + sources.add_source(HashMapSource::from_iter( + [("OTEL_RESOURCE_ATTRIBUTES", attributes)], + ConfigSourceOrigin::EnvVar, + )); + } + let mut builder = Config::builder_with_sources(&sources); + builder.set_otel_resource_attributes( + case.code_attributes + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + ); + let config = builder.build(); + + assert_eq!(config.env(), case.expected_env, "{} env", case.name); + assert_eq!( + config.otel_resource_attributes().collect::>(), + case.code_attributes + .iter() + .filter(|(key, _)| { + *key != "deployment.environment.name" && *key != "deployment.environment" + }) + .copied() + .collect::>(), + "{} resource attributes", + case.name + ); + let resource = crate::otlp_utils::build_otel_resource(&config, None); + assert_eq!( + resource + .get(&opentelemetry::Key::from_static_str( + "deployment.environment.name", + )) + .map(|value| value.as_str().into_owned()), + case.expected_env.map(str::to_string), + "{} OTLP resource", + case.name + ); + } + } + #[test] fn test_dd_env_overrides_otel_resource_environment_on_span() { let mut sources = CompositeSource::new(); sources.add_source(HashMapSource::from_iter( - [("DD_ENV", "datadog")], + [("DD_ENV", "datadog"), ("DD_TAGS", "env:global")], ConfigSourceOrigin::EnvVar, )); let config = Config::builder_with_sources(&sources).build(); @@ -3414,6 +3552,29 @@ mod tests { assert_resource_environment_on_span(config, resource, "datadog"); } + #[test] + fn test_resource_environment_overrides_datadog_tags_on_span() { + let mut sources = CompositeSource::new(); + sources.add_source(HashMapSource::from_iter( + [ + ("DD_TAGS", "env:datadog"), + ( + "OTEL_RESOURCE_ATTRIBUTES", + "deployment.environment.name=fallback", + ), + ], + ConfigSourceOrigin::EnvVar, + )); + let config = Config::builder_with_sources(&sources).build(); + let resource = Resource::builder_empty() + .with_attributes([ + KeyValue::new("deployment.environment.name", "provided"), + KeyValue::new("unrelated", "value"), + ]) + .build(); + assert_resource_environment_on_span(config, resource, "provided"); + } + #[test] fn test_resource_environment_overrides_otel_resource_attributes_on_span() { let mut sources = CompositeSource::new(); diff --git a/datadog-opentelemetry/src/core/configuration/mod.rs b/datadog-opentelemetry/src/core/configuration/mod.rs index 5f0bada4..6b196c5d 100644 --- a/datadog-opentelemetry/src/core/configuration/mod.rs +++ b/datadog-opentelemetry/src/core/configuration/mod.rs @@ -5,19 +5,10 @@ //! //! # Sources of configuration //! -//! ```text -//! ^ Highest precedence -//! | -//! * Opentelemetry Resource object -//! | -//! * ConfigBuilder setters -//! | -//! * "DD" prefixed env variables -//! | -//! * Default values -//! | -//! v Lowest level of precedence -//! ``` +//! General precedence is: OpenTelemetry Resource object, `ConfigBuilder` setters, "DD"-prefixed +//! environment variables, then defaults. Environment uses a field-specific order: +//! `DD_ENV`/`ConfigBuilder::set_env`, Resource `deployment.environment.name` or +//! `deployment.environment`, `DD_TAGS[env]`, then `OTEL_RESOURCE_ATTRIBUTES`. #[allow(clippy::module_inception)] mod configuration; diff --git a/datadog-opentelemetry/src/lib.rs b/datadog-opentelemetry/src/lib.rs index ba63561c..fedc78f1 100644 --- a/datadog-opentelemetry/src/lib.rs +++ b/datadog-opentelemetry/src/lib.rs @@ -392,7 +392,10 @@ impl DatadogTracingBuilder { /// context](https://github.com/open-telemetry/opentelemetry-specification/blob/main/oteps/profiles/4719-process-ctx.md). /// Publication errors are logged but otherwise ignored. pub fn init(self) -> SdkTracerProvider { - let config = self.config.unwrap_or_else(|| Config::builder().build()); + let config = Arc::new(self.config.unwrap_or_else(|| Config::builder().build())); + + let (tracer_provider, propagator) = + make_tracer(config.clone(), self.tracer_provider, self.resource); // For now, otel process context spec is linux-specific. #[cfg(target_os = "linux")] @@ -402,8 +405,6 @@ impl DatadogTracingBuilder { dd_warn!("Couldn't publish the tracer metadata during global initialization. External readers such as an eBPF profiler won't be able to access the corresponding resource attributes: {e}"); } - let (tracer_provider, propagator) = - make_tracer(Arc::new(config), self.tracer_provider, self.resource); opentelemetry::global::set_text_map_propagator(propagator); opentelemetry::global::set_tracer_provider(tracer_provider.clone()); tracer_provider @@ -565,6 +566,8 @@ fn make_tracer( resource: Option, ) -> (SdkTracerProvider, DatadogPropagator) { match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let dd_resource = + resolve_dd_resource(resource.unwrap_or(Resource::builder().build()), &config); let registry = TraceRegistry::new(config.clone()); let resource_slot = Arc::new(RwLock::new(Resource::builder_empty().build())); // Sampler only needs config for initialization (reads initial sampling rules) @@ -577,8 +580,6 @@ fn make_tracer( let agent_response_handler = sampler.on_agent_response(); - let dd_resource = - create_dd_resource(resource.unwrap_or(Resource::builder().build()), &config); tracer_provider_builder = tracer_provider_builder.with_resource(dd_resource); let propagator = DatadogPropagator::new(config.clone(), registry.clone()); @@ -706,7 +707,10 @@ fn create_dd_resource(mut resource: Resource, cfg: &Config) -> Resource { Value::from(env.to_string()), )); } else if resource_environment(&resource).is_none() { - if let Some(env) = cfg.otel_resource_environment() { + if let Some(env) = cfg + .global_tags_environment() + .or_else(|| cfg.otel_resource_environment()) + { attributes.push(( Key::from_static_str(DEPLOYMENT_ENVIRONMENT_NAME), Value::from(env.to_string()), @@ -722,6 +726,14 @@ fn create_dd_resource(mut resource: Resource, cfg: &Config) -> Resource { } } +fn resolve_dd_resource(resource: Resource, config: &Config) -> Resource { + let resource = create_dd_resource(resource, config); + config.set_calculated_environment( + resource_environment(&resource).map(|environment| environment.into_owned()), + ); + resource +} + #[cfg(feature = "test-utils")] pub fn make_test_tracer(shared_config: Arc) -> (SdkTracerProvider, DatadogPropagator) { #![allow(missing_docs)] diff --git a/datadog-opentelemetry/src/mappings/cached_config.rs b/datadog-opentelemetry/src/mappings/cached_config.rs index a10dfdb1..53eb6517 100644 --- a/datadog-opentelemetry/src/mappings/cached_config.rs +++ b/datadog-opentelemetry/src/mappings/cached_config.rs @@ -19,6 +19,7 @@ impl CachedConfig { let global_tags = cfg .global_tags() + .filter(|(key, _)| *key != "env") .map(|(key, value)| (key.to_string(), value.to_string())) .collect(); diff --git a/datadog-opentelemetry/src/otlp_utils.rs b/datadog-opentelemetry/src/otlp_utils.rs index 70e69c0f..c0282083 100644 --- a/datadog-opentelemetry/src/otlp_utils.rs +++ b/datadog-opentelemetry/src/otlp_utils.rs @@ -61,11 +61,7 @@ pub(crate) fn build_otel_resource(config: &Config, resource: Option) - .explicit_env() .map(Cow::Borrowed) .or_else(|| resource.as_ref().and_then(resource_environment)) - .or_else(|| { - config - .global_tags() - .find_map(|(key, value)| (key == "env").then(|| Cow::Owned(value.to_string()))) - }) + .or_else(|| config.global_tags_environment().map(Cow::Borrowed)) .or_else(|| config.otel_resource_environment().map(Cow::Borrowed)); for (key, value) in config.otel_resource_attributes() { @@ -281,4 +277,39 @@ mod tests { Some(Value::from("value")) ); } + + #[test] + fn final_global_tag_environment_wins() { + let mut builder = Config::builder(); + builder.set_global_tags(vec![ + ("env".to_string(), "first".to_string()), + ("env".to_string(), "last".to_string()), + ]); + let config = builder.build(); + + let resource = build_otel_resource(&config, None); + + assert_eq!(config.env(), Some("last")); + assert_eq!(resource_environment(&resource).as_deref(), Some("last")); + } + + #[test] + fn empty_final_global_tag_environment_falls_back_to_otel_resource() { + let mut builder = Config::builder(); + builder + .set_global_tags(vec![ + ("env".to_string(), "first".to_string()), + ("env".to_string(), String::new()), + ]) + .set_otel_resource_attributes(vec![( + DEPLOYMENT_ENVIRONMENT_NAME.to_string(), + "otel".to_string(), + )]); + let config = builder.build(); + + let resource = build_otel_resource(&config, None); + + assert_eq!(config.env(), Some("otel")); + assert_eq!(resource_environment(&resource).as_deref(), Some("otel")); + } } diff --git a/datadog-opentelemetry/src/span_processor.rs b/datadog-opentelemetry/src/span_processor.rs index 9461d06e..01e1073b 100644 --- a/datadog-opentelemetry/src/span_processor.rs +++ b/datadog-opentelemetry/src/span_processor.rs @@ -18,7 +18,7 @@ use crate::{ sampling::SamplingDecision, telemetry::{init_telemetry, wait_telemetry_stopped, TelemetryUser}, }, - create_dd_resource, dd_debug, dd_error, + dd_debug, dd_error, span_exporter::{DatadogExporter, DatadogExporterError, DatadogExporterInitError}, spans_metrics::TelemetryMetricsCollector, text_map_propagator::DatadogExtractData, @@ -715,13 +715,12 @@ impl opentelemetry_sdk::trace::SpanProcessor for DatadogSpanProcessor { } fn set_resource(&mut self, resource: &opentelemetry_sdk::Resource) { - let dd_resource = create_dd_resource(resource.clone(), &self.config); - self.span_exporter.set_resource(dd_resource.clone()); + self.span_exporter.set_resource(resource.clone()); // set the shared resource in the DatadogSpanProcessor - *self.resource.write().unwrap() = dd_resource.clone(); + *self.resource.write().unwrap() = resource.clone(); // update config's service name and init telemetry once service name has been resolved - let service_name = dd_resource + let service_name = resource .get(&Key::from_static_str(SERVICE_NAME)) .map(|service_name| service_name.as_str().to_string()); // Only set calculated service name if DD_SERVICE is default @@ -788,14 +787,16 @@ mod tests { let resource = Arc::new(RwLock::new(Resource::builder_empty().build())); let mut processor = - DatadogSpanProcessor::new(Arc::new(config), registry, resource.clone(), None).unwrap(); + DatadogSpanProcessor::new(Arc::new(config.clone()), registry, resource.clone(), None) + .unwrap(); let otel_resource = Resource::builder() // .with_service_name("otel-service") .with_attribute(KeyValue::new("key1", "value1")) .build(); - processor.set_resource(&otel_resource); + let dd_resource = crate::resolve_dd_resource(otel_resource, &config); + processor.set_resource(&dd_resource); let dd_resource = resource.read().unwrap(); assert_eq!( @@ -818,7 +819,8 @@ mod tests { let resource = Arc::new(RwLock::new(Resource::builder_empty().build())); let mut processor = - DatadogSpanProcessor::new(Arc::new(config), registry, resource.clone(), None).unwrap(); + DatadogSpanProcessor::new(Arc::new(config.clone()), registry, resource.clone(), None) + .unwrap(); let attributes = [KeyValue::new("key_schema", "value_schema")]; @@ -828,7 +830,8 @@ mod tests { .with_schema_url(attributes, "schema_url") .build(); - processor.set_resource(&otel_resource); + let dd_resource = crate::resolve_dd_resource(otel_resource, &config); + processor.set_resource(&dd_resource); let dd_resource = resource.read().unwrap(); assert_eq!( @@ -857,13 +860,15 @@ mod tests { let resource = Arc::new(RwLock::new(Resource::builder_empty().build())); let mut processor = - DatadogSpanProcessor::new(Arc::new(config), registry, resource.clone(), None).unwrap(); + DatadogSpanProcessor::new(Arc::new(config.clone()), registry, resource.clone(), None) + .unwrap(); let otel_resource = Resource::builder_empty() .with_attribute(KeyValue::new("key1", "value1")) .build(); - processor.set_resource(&otel_resource); + let dd_resource = crate::resolve_dd_resource(otel_resource, &config); + processor.set_resource(&dd_resource); let dd_resource = resource.read().unwrap(); assert_eq!( @@ -886,13 +891,15 @@ mod tests { let resource = Arc::new(RwLock::new(Resource::builder_empty().build())); let mut processor = - DatadogSpanProcessor::new(Arc::new(config), registry, resource.clone(), None).unwrap(); + DatadogSpanProcessor::new(Arc::new(config.clone()), registry, resource.clone(), None) + .unwrap(); let otel_resource = Resource::builder() .with_service_name("otel-service") .build(); - processor.set_resource(&otel_resource); + let dd_resource = crate::resolve_dd_resource(otel_resource, &config); + processor.set_resource(&dd_resource); let dd_resource = resource.read().unwrap(); assert_eq!( @@ -909,13 +916,15 @@ mod tests { let resource = Arc::new(RwLock::new(Resource::builder_empty().build())); let mut processor = - DatadogSpanProcessor::new(Arc::new(config), registry, resource.clone(), None).unwrap(); + DatadogSpanProcessor::new(Arc::new(config.clone()), registry, resource.clone(), None) + .unwrap(); let otel_resource = Resource::builder() .with_service_name("otel-service") .build(); - processor.set_resource(&otel_resource); + let dd_resource = crate::resolve_dd_resource(otel_resource, &config); + processor.set_resource(&dd_resource); let dd_resource = resource.read().unwrap(); assert_eq!(