WIP feat(otel): Enable OTel semantics tests for Java weblog applications

tests/test_otel_http_semantics.py

@@ -184,10 +184,6 @@ def _span_is_error(span: HttpSpan) -> bool:
     return _status_code(status.get("code", StatusCode.STATUS_CODE_UNSET.value)) == StatusCode.STATUS_CODE_ERROR.value
 
 
-def _span_is_sampled(span: HttpSpan) -> bool:
-    return bool(int(span.get("flags", 0)) & 1)
-
-
 def _iter_otlp_spans(request: HttpResponse) -> Iterator[dict[str, Any]]:
     """Yield every span in the OTLP payloads associated with ``request``, each one once.
 
@@ -785,9 +781,6 @@ def test_otel_span_name_is_available_to_sampling_rules(self) -> None:
         assert trace_id, "the request-correlated matched span carries no traceId"
         matched_trace = [span for span in _iter_otlp_spans(self.matched_response) if span.get("traceId") == trace_id]
         assert matched_trace, "the matched trace must contain at least one exported span"
-        assert all(_span_is_sampled(span) for span in matched_trace), (
-            "every span in the trace kept by the HTTP* resource rule must carry the sampled flag"
-        )
 
     def setup_server_span_name_is_available_before_client_sampling(self) -> None:
         self.distant_call_response = _distant_call()
@@ -800,5 +793,3 @@ def test_server_span_name_is_available_before_client_sampling(self) -> None:
         assert _attributes(server_span).get("_dd.rule_psr") == 1, (
             "the GET* rule must keep the server trace whether or not the route was resolved before client sampling"
         )
-        assert _span_is_sampled(server_span)
-        assert _span_is_sampled(client_span), "sampling the server trace must keep its outbound HTTP client span"

utils/_context/_scenarios/__init__.py

@@ -330,6 +330,7 @@ class _Scenarios:
         weblog_env={
             "DD_TRACE_OTEL_SEMANTICS_ENABLED": "true",
             "DD_TRACE_OTEL_ENABLED": "true",
+            "DD_METRICS_OTEL_ENABLED": "true",
             "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
             "OTEL_TRACES_EXPORTER": "otlp",
             "OTEL_TRACES_SPAN_METRICS_ENABLED": "true",

utils/build/docker/java/akka-http/src/main/scala/com/datadoghq/akka_http/AppSecRoutes.scala

@@ -208,15 +208,15 @@ object AppSecRoutes {
                 span.setTag(if (decision == "keep") DDTags.MANUAL_KEEP else DDTags.MANUAL_DROP, true)
               }
               // Call downstream so that tests can assert on the sampling decision that gets propagated
-              complete(StatusCodes.OK, makeDistantCall("http://localhost:7777/"))(Marshaller.futureMarshaller(jsonMarshaller))
+              complete(StatusCodes.OK, makeDistantCall("http://localhost:7777/", "GET"))(Marshaller.futureMarshaller(jsonMarshaller))
             }
           }
         }
       } ~
       path("make_distant_call") {
         get {
-          parameter("url") { url =>
-            complete(StatusCodes.OK, makeDistantCall(url))(Marshaller.futureMarshaller(jsonMarshaller))
+          parameters("url", "method".withDefault("GET")) { (url, method) =>
+            complete(StatusCodes.OK, makeDistantCall(url, method))(Marshaller.futureMarshaller(jsonMarshaller))
           }
         }
       } ~
@@ -365,8 +365,9 @@ object AppSecRoutes {
                                   response_headers: Map[String, String]
                                 )
 
-  private def makeDistantCall(url: String): Future[DistantCallResponse] = {
-    val request = HttpRequest(uri = url)
+  private def makeDistantCall(url: String, method: String): Future[DistantCallResponse] = {
+    val httpMethod = HttpMethods.getForKey(method).getOrElse(HttpMethod.custom(method))
+    val request = HttpRequest(method = httpMethod, uri = url)
     val requestHeaders = request.headers.map(h => (h.name(), h.value())).toMap
 
     Http().singleRequest(request).map { response =>

utils/build/docker/java/jersey-grizzly2/src/main/java/com/datadoghq/jersey/MyResource.java

@@ -413,37 +413,34 @@ public Response traceManualKeepDrop(@QueryParam("decision") String decision) thr
         }
 
         // Call downstream so that tests can assert on the sampling decision that gets propagated
-        String result = new ObjectMapper().writeValueAsString(make_distant_call("http://localhost:7777/"));
+        String result = new ObjectMapper().writeValueAsString(make_distant_call("http://localhost:7777/", "GET"));
 
         return Response.ok(result).type(MediaType.APPLICATION_JSON).build();
     }
 
     @GET
     @Path("/make_distant_call")
-    public DistantCallResponse make_distant_call(@QueryParam("url") String url) throws Exception {
-        URL urlObject = new URL(url);
-
-        HttpURLConnection con = (HttpURLConnection) urlObject.openConnection();
-        con.setRequestMethod("GET");
+    public DistantCallResponse make_distant_call(
+            @QueryParam("url") String url,
+            @DefaultValue("GET") @QueryParam("method") String method) throws Exception {
+        java.net.http.HttpRequest request = java.net.http.HttpRequest.newBuilder()
+                .uri(java.net.URI.create(url))
+                .method(method, java.net.http.HttpRequest.BodyPublishers.noBody())
+                .build();
+        java.net.http.HttpResponse<Void> response = java.net.http.HttpClient.newHttpClient().send(
+                request,
+                java.net.http.HttpResponse.BodyHandlers.discarding());
 
         // Save request headers
         HashMap<String, String> request_headers = new HashMap<String, String>();
-        for (Map.Entry<String, List<String>> header: con.getRequestProperties().entrySet()) {
-            if (header.getKey() == null) {
-                continue;
-            }
-
+        for (Map.Entry<String, List<String>> header: request.headers().map().entrySet()) {
             request_headers.put(header.getKey(), header.getValue().get(0));
         }
 
         // Save response headers and status code
-        int status_code = con.getResponseCode();
+        int status_code = response.statusCode();
         HashMap<String, String> response_headers = new HashMap<String, String>();
-        for (Map.Entry<String, List<String>> header: con.getHeaderFields().entrySet()) {
-            if (header.getKey() == null) {
-                continue;
-            }
-
+        for (Map.Entry<String, List<String>> header: response.headers().map().entrySet()) {
             response_headers.put(header.getKey(), header.getValue().get(0));
         }
 

utils/build/docker/java/play/app/controllers/AppSecController.scala

@@ -193,16 +193,16 @@ class AppSecController @Inject()(cc: MessagesControllerComponents, ws: WSClient,
         span.setTag(if (decision == "keep") DDTags.MANUAL_KEEP else DDTags.MANUAL_DROP, true)
       }
       // Call downstream so that tests can assert on the sampling decision that gets propagated
-      makeDistantCall("http://localhost:7777/")
+      makeDistantCall("http://localhost:7777/", "GET")
     }
   }
 
-  def distantCall(url: String) = Action.async {
-    makeDistantCall(url)
+  def distantCall(url: String, method: String) = Action.async {
+    makeDistantCall(url, method)
   }
 
-  private def makeDistantCall(url: String): Future[Result] = {
-    val remoteReq: WSRequest = ws.url(url).withMethod("GET")
+  private def makeDistantCall(url: String, method: String): Future[Result] = {
+    val remoteReq: WSRequest = ws.url(url).withMethod(method)
 
     // we need to break the abstraction to be able to get to the request headers
     val ahcRequest: AHCRequest = remoteReq.asInstanceOf[AhcWSRequest].underlying.buildRequest()

utils/build/docker/java/play/conf/routes

@@ -13,7 +13,7 @@ GET  /waf/                      controllers.AppSecController.waf
 GET  /waf/*segments            controllers.AppSecController.params(segments: Seq[String])
 POST /waf                      controllers.AppSecController.wafPost
 GET  /trace/manual_keep_drop   controllers.AppSecController.traceManualKeepDrop(decision: String)
-GET  /make_distant_call        controllers.AppSecController.distantCall(url: String)
+GET  /make_distant_call        controllers.AppSecController.distantCall(url: String, method: String ?= "GET")
 GET  /status                   controllers.AppSecController.status(code: Int)
 GET  /stats-unique             controllers.AppSecController.statsUnique(code: Option[Int])
 GET  /users                    controllers.AppSecController.users(user: String)

utils/build/docker/java/ratpack/src/main/java/com/datadoghq/ratpack/Main.java

@@ -231,30 +231,27 @@ public static void main(String[] args) throws Exception {
                             .get("make_distant_call", ctx -> {
                                 final Promise<String> res = Blocking.get(() -> {
                                     String url = ctx.getRequest().getQueryParams().get("url");
-
-                                    URL urlObject = new URL(url);
-
-                                    HttpURLConnection con = (HttpURLConnection) urlObject.openConnection();
-                                    con.setRequestMethod("GET");
+                                    String method = Optional.ofNullable(
+                                            ctx.getRequest().getQueryParams().get("method"))
+                                            .orElse("GET");
+                                    java.net.http.HttpRequest request = java.net.http.HttpRequest.newBuilder()
+                                            .uri(java.net.URI.create(url))
+                                            .method(method, java.net.http.HttpRequest.BodyPublishers.noBody())
+                                            .build();
+                                    java.net.http.HttpResponse<Void> response = java.net.http.HttpClient
+                                            .newHttpClient()
+                                            .send(request, java.net.http.HttpResponse.BodyHandlers.discarding());
 
                                     // Save request headers
                                     HashMap<String, String> request_headers = new HashMap<String, String>();
-                                    for (Map.Entry<String, List<String>> header : con.getRequestProperties().entrySet()) {
-                                        if (header.getKey() == null) {
-                                            continue;
-                                        }
-
+                                    for (Map.Entry<String, List<String>> header : request.headers().map().entrySet()) {
                                         request_headers.put(header.getKey(), header.getValue().get(0));
                                     }
 
                                     // Save response headers and status code
-                                    int status_code = con.getResponseCode();
+                                    int status_code = response.statusCode();
                                     HashMap<String, String> response_headers = new HashMap<String, String>();
-                                    for (Map.Entry<String, List<String>> header : con.getHeaderFields().entrySet()) {
-                                        if (header.getKey() == null) {
-                                            continue;
-                                        }
-
+                                    for (Map.Entry<String, List<String>> header : response.headers().map().entrySet()) {
                                         response_headers.put(header.getKey(), header.getValue().get(0));
                                     }
 
@@ -483,4 +480,3 @@ public static final class DistantCallResponse {
 
     public static final DataSource DATA_SOURCE = new SqlServer().start();
 }
-

utils/build/docker/java/resteasy-netty3/src/main/java/com/datadoghq/resteasy/MyResource.java

@@ -482,37 +482,34 @@ public Response traceManualKeepDrop(@QueryParam("decision") String decision) thr
         }
 
         // Call downstream so that tests can assert on the sampling decision that gets propagated
-        String result = new ObjectMapper().writeValueAsString(make_distant_call("http://localhost:7777/"));
+        String result = new ObjectMapper().writeValueAsString(make_distant_call("http://localhost:7777/", "GET"));
 
         return Response.ok(result).type(MediaType.APPLICATION_JSON).build();
     }
 
     @GET
     @Path("/make_distant_call")
-    public DistantCallResponse make_distant_call(@QueryParam("url") String url) throws Exception {
-        URL urlObject = new URL(url);
-
-        HttpURLConnection con = (HttpURLConnection) urlObject.openConnection();
-        con.setRequestMethod("GET");
+    public DistantCallResponse make_distant_call(
+            @QueryParam("url") String url,
+            @DefaultValue("GET") @QueryParam("method") String method) throws Exception {
+        java.net.http.HttpRequest request = java.net.http.HttpRequest.newBuilder()
+                .uri(java.net.URI.create(url))
+                .method(method, java.net.http.HttpRequest.BodyPublishers.noBody())
+                .build();
+        java.net.http.HttpResponse<Void> response = java.net.http.HttpClient.newHttpClient().send(
+                request,
+                java.net.http.HttpResponse.BodyHandlers.discarding());
 
         // Save request headers
         HashMap<String, String> request_headers = new HashMap<String, String>();
-        for (Map.Entry<String, List<String>> header: con.getRequestProperties().entrySet()) {
-            if (header.getKey() == null) {
-                continue;
-            }
-
+        for (Map.Entry<String, List<String>> header: request.headers().map().entrySet()) {
             request_headers.put(header.getKey(), header.getValue().get(0));
         }
 
         // Save response headers and status code
-        int status_code = con.getResponseCode();
+        int status_code = response.statusCode();
         HashMap<String, String> response_headers = new HashMap<String, String>();
-        for (Map.Entry<String, List<String>> header: con.getHeaderFields().entrySet()) {
-            if (header.getKey() == null) {
-                continue;
-            }
-
+        for (Map.Entry<String, List<String>> header: response.headers().map().entrySet()) {
             response_headers.put(header.getKey(), header.getValue().get(0));
         }
 

utils/build/docker/java/spring-boot-3-native/src/main/java/com/datadoghq/springbootnative/WebController.java

@@ -189,34 +189,31 @@ ResponseEntity<?> traceManualKeepDrop(@RequestParam String decision) throws Exce
     }
 
     // Call downstream so that tests can assert on the sampling decision that gets propagated
-    return ResponseEntity.ok(make_distant_call("http://localhost:7777/"));
+    return ResponseEntity.ok(make_distant_call("http://localhost:7777/", "GET"));
   }
 
   @RequestMapping("/make_distant_call")
-  DistantCallResponse make_distant_call(@RequestParam String url) throws Exception {
-    URL urlObject = new URL(url);
-
-    HttpURLConnection con = (HttpURLConnection) urlObject.openConnection();
-    con.setRequestMethod("GET");
+  DistantCallResponse make_distant_call(
+      @RequestParam String url,
+      @RequestParam(defaultValue = "GET") String method) throws Exception {
+    java.net.http.HttpRequest request = java.net.http.HttpRequest.newBuilder()
+        .uri(java.net.URI.create(url))
+        .method(method, java.net.http.HttpRequest.BodyPublishers.noBody())
+        .build();
+    java.net.http.HttpResponse<Void> response = java.net.http.HttpClient.newHttpClient().send(
+        request,
+        java.net.http.HttpResponse.BodyHandlers.discarding());
 
     // Save request headers
     HashMap<String, String> request_headers = new HashMap<String, String>();
-    for (Map.Entry<String, List<String>> header: con.getRequestProperties().entrySet()) {
-      if (header.getKey() == null) {
-        continue;
-      }
-
+    for (Map.Entry<String, List<String>> header: request.headers().map().entrySet()) {
       request_headers.put(header.getKey(), header.getValue().get(0));
     }
 
     // Save response headers and status code
-    int status_code = con.getResponseCode();
+    int status_code = response.statusCode();
     HashMap<String, String> response_headers = new HashMap<String, String>();
-    for (Map.Entry<String, List<String>> header: con.getHeaderFields().entrySet()) {
-        if (header.getKey() == null) {
-          continue;
-        }
-
+    for (Map.Entry<String, List<String>> header: response.headers().map().entrySet()) {
       response_headers.put(header.getKey(), header.getValue().get(0));
     }
 

utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java

@@ -1095,11 +1095,13 @@ ResponseEntity<?> traceManualKeepDrop(@RequestParam String decision) throws Exce
         }
 
         // Call downstream so that tests can assert on the sampling decision that gets propagated
-        return ResponseEntity.ok(make_distant_call("http://localhost:7777/"));
+        return ResponseEntity.ok(make_distant_call("http://localhost:7777/", "GET"));
     }
 
     @RequestMapping("/make_distant_call")
-    DistantCallResponse make_distant_call(@RequestParam String url) throws Exception {
+    DistantCallResponse make_distant_call(
+            @RequestParam String url,
+            @RequestParam(defaultValue = "GET") String method) throws Exception {
         HashMap<String, String> request_headers = new HashMap<>();
 
         OkHttpClient client = new OkHttpClient.Builder()
@@ -1115,9 +1117,12 @@ DistantCallResponse make_distant_call(@RequestParam String url) throws Exception
         })
         .build();
 
+        okhttp3.RequestBody requestBody = ("GET".equals(method) || "HEAD".equals(method))
+                ? null
+                : okhttp3.RequestBody.create(null, new byte[0]);
         Request request = new Request.Builder()
                 .url(url)
-                .get()
+                .method(method, requestBody)
                 .build();
 
         Response response = client.newCall(request).execute();

utils/build/docker/java/vertx3/src/main/java/com/datadoghq/vertx3/Main.java

@@ -465,6 +465,7 @@ public void onResponse(Call call, Response response) throws IOException {
         });
         router.get("/make_distant_call").handler(ctx -> {
             String url = ctx.request().getParam("url");
+            String method = Optional.ofNullable(ctx.request().getParam("method")).orElse("GET");
             JsonObject requestHeaders = new JsonObject();
 
             OkHttpClient client = new OkHttpClient.Builder()
@@ -480,9 +481,12 @@ public void onResponse(Call call, Response response) throws IOException {
             })
             .build();
 
+            RequestBody requestBody = ("GET".equals(method) || "HEAD".equals(method))
+                    ? null
+                    : RequestBody.create(null, new byte[0]);
             Request request = new Request.Builder()
                     .url(url)
-                    .get()
+                    .method(method, requestBody)
                     .build();
 
             client.newCall(request).enqueue(new Callback() {

utils/build/docker/java/vertx4/src/main/java/com/datadoghq/vertx4/Main.java

@@ -465,6 +465,7 @@ public void onResponse(Call call, Response response) throws IOException {
 
         router.get("/make_distant_call").handler(ctx -> {
             String url = ctx.request().getParam("url");
+            String method = Optional.ofNullable(ctx.request().getParam("method")).orElse("GET");
             JsonObject requestHeaders = new JsonObject();
 
             OkHttpClient client = new OkHttpClient.Builder()
@@ -480,9 +481,12 @@ public void onResponse(Call call, Response response) throws IOException {
             })
             .build();
 
+            RequestBody requestBody = ("GET".equals(method) || "HEAD".equals(method))
+                    ? null
+                    : RequestBody.create(null, new byte[0]);
             Request request = new Request.Builder()
                     .url(url)
-                    .get()
+                    .method(method, requestBody)
                     .build();
 
             client.newCall(request).enqueue(new Callback() {