feat(aws): Apply data_collection filtering to URL query strings

sentry_sdk/integrations/aws_lambda.py

@@ -11,6 +11,7 @@
 import sentry_sdk
 from sentry_sdk.api import continue_trace
 from sentry_sdk.consts import OP
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
 from sentry_sdk.integrations import Integration
 from sentry_sdk.integrations._wsgi_common import _filter_headers
 from sentry_sdk.integrations.cloud_resource_context import (
@@ -27,6 +28,7 @@
     capture_internal_exceptions,
     ensure_integration_enabled,
     event_from_exception,
+    has_data_collection_enabled,
     logger,
     reraise,
 )
@@ -164,10 +166,20 @@ def sentry_handler(
                     "httpMethod"
                 ]
 
-            if should_send_default_pii() and "queryStringParameters" in request_data:
+            if "queryStringParameters" in request_data:
                 qs = request_data["queryStringParameters"]
                 if qs:
-                    additional_attributes["url.query"] = urlencode(qs)
+                    if has_data_collection_enabled(client.options):
+                        filtered_qs = _apply_key_value_collection_filtering(
+                            items=qs,
+                            behaviour=client.options["data_collection"][
+                                "url_query_params"
+                            ],
+                        )
+                        if filtered_qs:
+                            additional_attributes["url.query"] = urlencode(filtered_qs)
+                    elif should_send_default_pii():
+                        additional_attributes["url.query"] = urlencode(qs)
 
             sampling_context = {
                 "aws_event": aws_event,
@@ -409,7 +421,18 @@ def event_processor(
         request["url"] = _get_url(aws_event, aws_context)
 
         if "queryStringParameters" in aws_event:
-            request["query_string"] = aws_event["queryStringParameters"]
+            query_string = aws_event["queryStringParameters"]
+            client_options = sentry_sdk.get_client().options
+            if has_data_collection_enabled(client_options):
+                if query_string:
+                    filtered_qs = _apply_key_value_collection_filtering(
+                        items=query_string,
+                        behaviour=client_options["data_collection"]["url_query_params"],
+                    )
+                    if filtered_qs:
+                        request["query_string"] = filtered_qs
+            else:
+                request["query_string"] = query_string
 
         if "headers" in aws_event:
             request["headers"] = _filter_headers(aws_event["headers"])

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryAllowlist/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryAllowlist/index.py

@@ -0,0 +1,24 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    _experiments={
+        "data_collection": {
+            "url_query_params": {
+                "mode": "allowlist",
+                # "token" is allowlisted on purpose to show that an allowlist
+                # entry cannot override the built-in sensitive denylist.
+                "terms": ["page", "token"],
+            }
+        }
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryDenylist/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryDenylist/index.py

@@ -0,0 +1,24 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    _experiments={
+        "data_collection": {
+            "url_query_params": {
+                "mode": "denylist",
+                # Custom terms deny otherwise non-sensitive query params on top
+                # of the built-in sensitive denylist.
+                "terms": ["tracking"],
+            }
+        }
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryOff/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryOff/index.py

@@ -0,0 +1,19 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    _experiments={
+        "data_collection": {
+            "url_query_params": {"mode": "off"},
+        }
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkSpanStreamingDataCollection/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkSpanStreamingDataCollection/index.py

@@ -0,0 +1,23 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    _experiments={
+        "trace_lifecycle": "stream",
+        "data_collection": {
+            "url_query_params": {
+                "mode": "denylist",
+                "terms": ["tracking"],
+            }
+        },
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}

tests/integrations/aws_lambda/test_aws_lambda.py

@@ -626,6 +626,142 @@ def test_request_data_with_data_collection_off(lambda_client, test_environment):
     }
 
 
+def test_url_query_params_with_data_collection_denylist(
+    lambda_client, test_environment
+):
+    payload = b"""
+        {
+          "resource": "/asd",
+          "path": "/asd",
+          "httpMethod": "GET",
+          "headers": {
+            "Host": "iwsz2c7uwi.execute-api.us-east-1.amazonaws.com",
+            "X-Forwarded-Proto": "https"
+          },
+          "queryStringParameters": {
+            "page": "2",
+            "tracking": "campaign",
+            "token": "secret-token"
+          },
+          "pathParameters": null,
+          "stageVariables": null,
+          "requestContext": {
+            "identity": {
+              "sourceIp": "213.47.147.207",
+              "userArn": "42"
+            }
+          },
+          "body": null,
+          "isBase64Encoded": false
+        }
+    """
+
+    lambda_client.invoke(
+        FunctionName="BasicOkDataCollectionUrlQueryDenylist",
+        Payload=payload,
+    )
+    envelopes = test_environment["server"].envelopes
+
+    (transaction_event,) = envelopes
+
+    assert transaction_event["request"]["query_string"] == {
+        # Not denied by any term -> pass through.
+        "page": "2",
+        # Denied by custom terms.
+        "tracking": "[Filtered]",
+        # Denied by the built-in sensitive denylist.
+        "token": "[Filtered]",
+    }
+
+
+def test_url_query_params_with_data_collection_allowlist(
+    lambda_client, test_environment
+):
+    payload = b"""
+        {
+          "resource": "/asd",
+          "path": "/asd",
+          "httpMethod": "GET",
+          "headers": {
+            "Host": "iwsz2c7uwi.execute-api.us-east-1.amazonaws.com",
+            "X-Forwarded-Proto": "https"
+          },
+          "queryStringParameters": {
+            "page": "2",
+            "tracking": "campaign",
+            "token": "secret-token"
+          },
+          "pathParameters": null,
+          "stageVariables": null,
+          "requestContext": {
+            "identity": {
+              "sourceIp": "213.47.147.207",
+              "userArn": "42"
+            }
+          },
+          "body": null,
+          "isBase64Encoded": false
+        }
+    """
+
+    lambda_client.invoke(
+        FunctionName="BasicOkDataCollectionUrlQueryAllowlist",
+        Payload=payload,
+    )
+    envelopes = test_environment["server"].envelopes
+
+    (transaction_event,) = envelopes
+
+    assert transaction_event["request"]["query_string"] == {
+        # Allowlisted, non-sensitive -> pass through.
+        "page": "2",
+        # Not allowlisted -> substituted.
+        "tracking": "[Filtered]",
+        # Allowlisted but sensitive -> still filtered; an allowlist entry
+        # cannot override the built-in sensitive denylist.
+        "token": "[Filtered]",
+    }
+
+
+def test_url_query_params_with_data_collection_off(lambda_client, test_environment):
+    payload = b"""
+        {
+          "resource": "/asd",
+          "path": "/asd",
+          "httpMethod": "GET",
+          "headers": {
+            "Host": "iwsz2c7uwi.execute-api.us-east-1.amazonaws.com",
+            "X-Forwarded-Proto": "https"
+          },
+          "queryStringParameters": {
+            "page": "2",
+            "tracking": "campaign"
+          },
+          "pathParameters": null,
+          "stageVariables": null,
+          "requestContext": {
+            "identity": {
+              "sourceIp": "213.47.147.207",
+              "userArn": "42"
+            }
+          },
+          "body": null,
+          "isBase64Encoded": false
+        }
+    """
+
+    lambda_client.invoke(
+        FunctionName="BasicOkDataCollectionUrlQueryOff",
+        Payload=payload,
+    )
+    envelopes = test_environment["server"].envelopes
+
+    (transaction_event,) = envelopes
+
+    # With url_query_params collection turned off, no query string is collected.
+    assert "query_string" not in transaction_event["request"]
+
+
 def test_trace_continuation(lambda_client, test_environment):
     trace_id = "471a43a4192642f0b136d5159a501701"
     parent_span_id = "6e8f22c393e68f19"
@@ -933,6 +1069,38 @@ def test_span_streaming_request_attributes(lambda_client, test_environment):
     assert _get_span_attr(attrs, "aws.log.stream.names") == ["$LATEST"]
 
 
+def test_span_streaming_url_query_params_with_data_collection(
+    lambda_client, test_environment
+):
+    payload = {
+        "httpMethod": "GET",
+        "queryStringParameters": {
+            "page": "2",
+            "tracking": "campaign",
+            "token": "secret-token",
+        },
+        "path": "/test",
+    }
+
+    lambda_client.invoke(
+        FunctionName="BasicOkSpanStreamingDataCollection",
+        Payload=json.dumps(payload),
+    )
+    span_items = test_environment["server"].span_items
+
+    segment_spans = [s for s in span_items if s["is_segment"]]
+    assert len(segment_spans) == 1
+    segment_span = segment_spans[0]
+    attrs = segment_span["attributes"]
+
+    # "page" passes through; "tracking" is denied by a custom term and "token"
+    # by the built-in sensitive denylist.
+    assert (
+        _get_span_attr(attrs, "url.query")
+        == "page=2&tracking=%5BFiltered%5D&token=%5BFiltered%5D"
+    )
+
+
 @pytest.mark.parametrize(
     "lambda_function_name",
     ["RaiseErrorPerformanceEnabled", "RaiseErrorPerformanceDisabled"],