feat(tracer): send traces as V1 payloads through the libdatadog builder FFI

.gitlab/generate-common.php

@@ -230,7 +230,7 @@ function windows_git_setup_with_packages() {
       KUBERNETES_SERVICE_MEMORY_LIMIT: 1Gi
 
   request-replayer:
-    name: registry.ddbuild.io/ci/dd-trace-php/request-replayer:4.0
+    name: registry.ddbuild.io/ci/dd-trace-php/request-replayer:5.0
     alias: request-replayer
     command: ["php", "-S", "<?= $service_bind_address ?>:80", "index.php"]
     variables:

.gitlab/generate-package.php

@@ -1421,6 +1421,10 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
     PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
     APT_CACHE: $CI_PROJECT_DIR/.cache/apt
     DOCKER_DEFAULT_PLATFORM: linux/amd64
+    # Override these to point at a fork/branch of system-tests without touching this file.
+    SYSTEM_TESTS_REPO: "https://github.com/DataDog/system-tests.git"
+    # TODO: point back at "main" once DataDog/system-tests#7843 (leiyks/php-v1-payload) is merged.
+    SYSTEM_TESTS_REF: "leiyks/php-v1-payload"
     # TODO DD_API_KEY; SYSTEM_TESTS_AWS_ACCESS_KEY_ID; SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY
   needs:
     - job: "package extension (bundles): [amd64, x86_64-unknown-linux-gnu]"
@@ -1450,7 +1454,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
       pip install -U pip virtualenv
 <?php dockerhub_login() ?>
     - /tmp/vault kv get --format=json "kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin)['data']['data']['key'])" > /tmp/.dd-api-key 2>/dev/null || true
-    - git clone https://github.com/DataDog/system-tests.git
+    - git clone --branch "$SYSTEM_TESTS_REF" --depth 1 "$SYSTEM_TESTS_REPO" system-tests
     - mv packages/{datadog-setup.php,dd-library-php-*x86_64-linux-gnu.tar.gz} system-tests/binaries
     - cd system-tests
     - ./build.sh $BUILD_SH_ARGS
@@ -1478,6 +1482,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
   parallel:
     matrix:
       - TESTSUITE:
+        - APM_TRACING_EFFICIENT_PAYLOAD
         - APPSEC_API_SECURITY
         - APPSEC_API_SECURITY_RC
         - APPSEC_API_SECURITY_NO_RESPONSE_BODY

.gitlab/generate-tracer.php

@@ -137,7 +137,10 @@ function windows_test_c_job($job_name, $thread_safety, $targets) {
 
     # Start the container network and services
     docker network create -d "nat" -o com.docker.network.windowsshim.dnsservers="1.1.1.1" net
+    # Force a pull so a runner with a stale cached digest doesn't skip the rebuilt image.
+    docker pull registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:httpbin-windows
     docker run --network net -d --name httpbin-integration registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:httpbin-windows
+    docker pull registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:php-request-replayer-3.0_windows
     docker run --network net -d --name request-replayer registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:php-request-replayer-3.0_windows
     docker run --env GITLAB_CI=$env:GITLAB_CI -v ${pwd}:C:\Users\ContainerAdministrator\app  --network net -d --name ${CONTAINER_NAME} ${IMAGE} ping -t localhost
 
@@ -681,7 +684,6 @@ function windows_test_c_job($job_name, $thread_safety, $targets) {
     ARCH: "amd64"
     DD_TRACE_STARTUP_LOGS: "0"
     DD_TRACE_WARN_CALL_STACK_DEPTH: "0"
-    DD_TRACE_WARN_LEGACY_DD_TRACE: "0"
     DD_TRACE_GIT_METADATA_ENABLED: "0"
     REPORT_EXIT_STATUS: "1"
     TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"

Cargo.toml

@@ -126,6 +126,7 @@ cc = "1.0"
 [dev-dependencies]
 criterion = "0.5.1"
 static_assertions = "1"
+rmpv = "1.3.0"
 
 [target.'cfg(target_arch = "x86_64")'.dev-dependencies]
 criterion-perf-events = "0.4.0"
@@ -302,6 +303,7 @@ semver = { version = "1.0", default-features = false }
 serde = { version = "1.0", default-features = false }
 serde-transcode = { version = "1.1", default-features = false }
 serde_bytes = { version = "0.11.9", default-features = false }
+serde_ignored = { version = "0.1.14", default-features = false }
 # allow(workspace-deps-features): serde_json requires at least one of `std` or
 # `alloc`. We set `alloc` here as the minimal working baseline, and to avoid
 # repeating alloc everywhere. Enabling `std` on top of it is fine.

Makefile

@@ -1289,7 +1289,7 @@ define run_composer_with_retry
 endef
 
 define run_tests_without_coverage
-	$(TEST_EXTRA_ENV) $(ENV_OVERRIDE) php $(TEST_EXTRA_INI) -d datadog.instrumentation_telemetry_enabled=$(shell (test $(TELEMETRY_ENABLED) && echo 1) || (test $(PHP_MAJOR_MINOR) -ge 83 && echo 1) || echo 0) -d datadog.trace.sidecar_trace_sender=$(shell test $(PHP_MAJOR_MINOR) -ge 83 && echo 1 || echo 0) $(TRACER_SOURCES_INI) $(PHPUNIT) $(1) --filter=$(FILTER)
+	$(TEST_EXTRA_ENV) $(ENV_OVERRIDE) php $(TEST_EXTRA_INI) -d datadog.instrumentation_telemetry_enabled=$(shell (test $(TELEMETRY_ENABLED) && echo 1) || (test $(PHP_MAJOR_MINOR) -ge 83 && echo 1) || echo 0) $(TRACER_SOURCES_INI) $(PHPUNIT) $(1) --filter=$(FILTER)
 endef
 
 define run_tests_with_coverage

appsec/src/extension/ddtrace.c

@@ -34,6 +34,7 @@ static int _mod_number;
 static const char *_mod_version;
 static bool _ddtrace_loaded;
 static zend_string *_ddtrace_root_span_fname;
+static zend_string *_attributes_propname;
 static zend_string *_meta_propname;
 static zend_string *_metrics_propname;
 static zend_string *_meta_struct_propname;
@@ -146,6 +147,7 @@ void dd_trace_startup(void)
 {
     _ddtrace_root_span_fname = zend_string_init_interned(
         LSTRARG("ddtrace\\root_span"), 1 /* permanent */);
+    _attributes_propname = zend_string_init_interned(LSTRARG("attributes"), 1);
     _meta_propname = zend_string_init_interned(LSTRARG("meta"), 1);
     _metrics_propname = zend_string_init_interned(LSTRARG("metrics"), 1);
     _meta_struct_propname =
@@ -340,14 +342,30 @@ static zval *_get_span_modifiable_array_property(
     return res;
 }
 
+// $meta and $metrics are views onto $attributes, the span's single tag store.
 zval *nullable dd_trace_span_get_meta(zend_object *nonnull zobj)
 {
-    return _get_span_modifiable_array_property(zobj, _meta_propname);
+    return _get_span_modifiable_array_property(zobj, _attributes_propname);
 }
 
 zval *nullable dd_trace_span_get_metrics(zend_object *nonnull zobj)
 {
-    return _get_span_modifiable_array_property(zobj, _metrics_propname);
+    return _get_span_modifiable_array_property(zobj, _attributes_propname);
+}
+
+// Reads $meta/$metrics through the span's handlers: an array of that bucket.
+static zval *_read_span_property(
+    zend_object *nonnull zobj, zend_string *nonnull propname, zval *nonnull rv)
+{
+#if PHP_VERSION_ID >= 80000
+    return zobj->handlers->read_property(zobj, propname, BP_VAR_R, NULL, rv);
+#else
+    zval obj;
+    ZVAL_OBJ(&obj, zobj);
+    zval prop;
+    ZVAL_STR(&prop, propname);
+    return zobj->handlers->read_property(&obj, &prop, BP_VAR_R, NULL, rv);
+#endif
 }
 
 zval *nullable dd_trace_span_get_meta_struct(zend_object *nonnull zobj)
@@ -621,9 +639,11 @@ static PHP_FUNCTION(datadog_appsec_testing_root_span_get_meta) // NOLINT
         RETURN_NULL();
     }
 
-    zval *meta_zv = dd_trace_span_get_meta(root_span);
-    if (meta_zv) {
-        RETURN_ZVAL(meta_zv, 1 /* copy */, 0 /* no destroy original */);
+    zval rv;
+    zval *meta_zv = _read_span_property(root_span, _meta_propname, &rv);
+    RETVAL_ZVAL(meta_zv, 1 /* copy */, 0 /* no destroy original */);
+    if (meta_zv == &rv) {
+        zval_ptr_dtor(&rv);
     }
 }
 
@@ -655,9 +675,11 @@ static PHP_FUNCTION(datadog_appsec_testing_root_span_get_metrics) // NOLINT
         RETURN_NULL();
     }
 
-    zval *metrics_zv = dd_trace_span_get_metrics(root_span);
-    if (metrics_zv) {
-        RETURN_ZVAL(metrics_zv, 1 /* copy */, 0 /* no destroy original */);
+    zval rv;
+    zval *metrics_zv = _read_span_property(root_span, _metrics_propname, &rv);
+    RETVAL_ZVAL(metrics_zv, 1 /* copy */, 0 /* no destroy original */);
+    if (metrics_zv == &rv) {
+        zval_ptr_dtor(&rv);
     }
 }
 

appsec/src/extension/ddtrace.h

@@ -55,8 +55,8 @@ void dd_trace_close_all_spans_and_flush(void);
 
 void dd_trace_emit_asm_event(void);
 
-// Provides the array zval representing $root_span->meta, if any.
-// It is ready for modification, with refcount == 1
+// Provides the array zval representing $root_span->attributes (which $meta
+// and $metrics are views onto), if any. Ready for modification, refcount == 1
 zval *nullable dd_trace_span_get_meta(zend_object *nonnull);
 zval *nullable dd_trace_span_get_metrics(zend_object *nonnull);
 zval *nullable dd_trace_span_get_meta_struct(zend_object *nonnull);

appsec/tests/extension/api_security_telemetry_missing_route.phpt

@@ -42,13 +42,13 @@ var_dump(rshutdown());
 $c = $helper->get_commands();
 echo "Sampler hash sent: ", $c[0][1][1], "\n";
 
-match_log('/Telemetry metric api_security\.missing_route added with tags framework:unknown and value 1/');
+match_log('/Telemetry metric api_security\.missing_route added with tags framework:cgi-fcgi and value 1/');
 no_match_log('/api_security\.request\./');
 
 ?>
 --EXPECT--
 bool(true)
 bool(true)
 Sampler hash sent: 0
-found message in log matching /Telemetry metric api_security\.missing_route added with tags framework:unknown and value 1/
+found message in log matching /Telemetry metric api_security\.missing_route added with tags framework:cgi-fcgi and value 1/
 no message in log matching /api_security\.request\./

appsec/tests/extension/client_init_record_span_tags.phpt

@@ -76,6 +76,7 @@ Array
     [http.url] => https://localhost:8888/foo
     [http.method] => GET
     [span.kind] => server
+    [component] => cgi-fcgi
     [http.useragent] => my user agent
 )
 rinit
@@ -94,6 +95,7 @@ Array
     [_dd.runtime_family] => php
     [_dd.sdk.otlp_export] => false
     [appsec.event] => true
+    [component] => cgi-fcgi
     [http.endpoint] => /foo
     [http.method] => GET
     [http.request.headers.user-agent] => my user agent

appsec/tests/extension/rinit_record_span_tags.phpt

@@ -71,6 +71,7 @@ Array
     [http.url] => https://localhost:8888/foo
     [http.method] => GET
     [span.kind] => server
+    [component] => cgi-fcgi
     [http.useragent] => my user agent
 )
 rinit
@@ -89,6 +90,7 @@ Array
     [_dd.runtime_family] => php
     [_dd.sdk.otlp_export] => false
     [appsec.event] => true
+    [component] => cgi-fcgi
     [http.endpoint] => /foo
     [http.method] => GET
     [http.request.headers.user-agent] => my user agent

appsec/tests/extension/rinit_root_span_add_tag.phpt

@@ -63,6 +63,7 @@ Array
     [_dd.p.dm] => -0
     [_dd.p.tid] => %s
     [_dd.sdk.otlp_export] => false
+    [component] => cgi-fcgi
     [ddappsec] => true
     [env] => staging
     [http.method] => GET

appsec/tests/extension/root_span_add_tag.phpt

@@ -29,9 +29,11 @@ bool(true)
 bool(false)
 array(1) {
   [0]=>
-  array(10) {
+  array(12) {
     ["trace_id"]=>
     string(%d) "%d"
+    ["trace_id_high"]=>
+    string(16) "%s"
     ["span_id"]=>
     string(%d) "%d"
     ["start"]=>
@@ -46,33 +48,14 @@ array(1) {
     string(12) "appsec_tests"
     ["type"]=>
     string(3) "cli"
-    ["meta"]=>
-    array(5) {
-      ["_dd.p.dm"]=>
-      string(2) "-0"
-      ["_dd.p.tid"]=>
-      string(16) "%s"
-      ["_dd.sdk.otlp_export"]=>
-      string(5) "false"
+    ["sampling_priority"]=>
+    int(1)
+    ["sampling_mechanism"]=>
+    int(0)
+    ["attributes"]=>
+    array(%d) {%A
       ["after"]=>
-      string(9) "root_span"
-      ["runtime-id"]=>
-      string(%d) %s
-    }
-    ["metrics"]=>
-    array(6) {
-      [%s"]=>
-      float(%d)
-      ["_sampling_priority_v1"]=>
-      float(1)
-      ["php.compilation.total_time_ms"]=>
-      float(%s)
-      ["php.memory.peak_real_usage_bytes"]=>
-      float(%f)
-      ["php.memory.peak_usage_bytes"]=>
-      float(%f)
-      ["process_id"]=>
-      float(%f)
+      string(9) "root_span"%A
     }
   }
 }

appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/EndpointFallbackSamplingTests.groovy

@@ -177,7 +177,9 @@ trait EndpointFallbackSamplingTests extends SamplingTestsInFpm {
                     assert metric.namespace == 'appsec'
                     assert metric.type == 'count'
                     assert metric.points[0][1] >= 1.0
-                    assert 'framework:unknown' in metric.tags
+                    // root span component now defaults to the SAPI name (fpm-fcgi here);
+                    // this trait is only mixed into Apache2FpmTests
+                    assert 'framework:fpm-fcgi' in metric.tags
             }
         } finally {
             resetFpm()