feat(ourlogs): add regex search support to the EAP query syntax

pyproject.toml

@@ -92,7 +92,7 @@ dependencies = [
   "sentry-ophio>=1.1.3",
   # sentry-options is only used in getsentry for now
   "sentry-options>=1.2.8",
-  "sentry-protos>=0.67.0",
+  "sentry-protos>=0.68.0",
   "sentry-redis-tools>=0.5.0",
   "sentry-relay>=0.9.28",
   "sentry-scm==1.5.0",

src/sentry/api/event_search.py

@@ -17,12 +17,14 @@
     DURATION_UNITS,
     NOT_HAS_FILTER_ERROR_MESSAGE,
     OPERATOR_NEGATION_MAP,
+    REGEX_OPERATOR,
     SEARCH_MAP,
     SEMVER_ALIAS,
     SEMVER_BUILD_ALIAS,
     SIZE_UNITS,
     TAG_KEY_RE,
     TEAM_KEY_TRANSACTION_ALIAS,
+    UNSUPPORTED_REGEX_SYNTAX,
     WILDCARD_OPERATOR_MAP,
 )
 from sentry.search.events.fields import FIELD_ALIASES, FUNCTIONS
@@ -189,7 +191,7 @@
 
 # NOTE: These wildcard operators are internal implementation details and
 # should not be included in product docs. Users should use `*` instead.
-wildcard_op            = wildcard_unicode (contains / starts_with / ends_with) wildcard_unicode
+wildcard_op            = wildcard_unicode (contains / starts_with / ends_with / matches) wildcard_unicode
 
 # See: https://stackoverflow.com/a/39617181/790169
 in_value_termination = in_value_char (!in_value_end in_value_char)* in_value_end
@@ -233,6 +235,7 @@
 contains             = "Contains"
 starts_with          = "StartsWith"
 ends_with            = "EndsWith"
+matches              = "Matches"
 comma                = ","
 spaces               = " "*
 
@@ -418,6 +421,44 @@ def get_wildcard_op(node: Node | Sequence[Node]) -> str:
     return ""
 
 
+def has_regex_op(node: Node | Sequence[Node]) -> bool:
+    return get_wildcard_op(node) == REGEX_OPERATOR
+
+
+def quote_regex_pattern(pattern: str) -> str:
+    """Regex patterns are always quoted when serialized back to a query string, because the
+    unquoted value grammar rejects the parentheses and spaces that patterns routinely contain."""
+    escaped = pattern.replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def validate_regex_pattern(key: str, pattern: str) -> None:
+    if not pattern:
+        raise InvalidSearchQuery(f"{key}: Empty regex pattern")
+
+    for match in UNSUPPORTED_REGEX_SYNTAX.finditer(pattern):
+        unsupported = match.group("unsupported")
+        if unsupported is not None:
+            raise InvalidSearchQuery(
+                f"{key}: Invalid regex: `{unsupported}` is not supported. "
+                "Patterns are matched with RE2, which has no backreferences, lookaround, "
+                "or other PCRE extensions."
+            )
+
+    try:
+        re.compile(pattern)
+    except re.error as exc:
+        raise InvalidSearchQuery(f"{key}: Invalid regex: {exc.msg}")
+
+
+def as_regex_value(key: str, value: SearchValue) -> SearchValue:
+    patterns = value.raw_value if isinstance(value.raw_value, (list, tuple)) else [value.raw_value]
+    for pattern in patterns:
+        if isinstance(pattern, str):
+            validate_regex_pattern(key, pattern)
+    return value._replace(is_regex=True)
+
+
 def add_leading_wildcard(value: str) -> str:
     if value.startswith('"') and value.endswith('"'):
         return f"*{value[1:-1]}"
@@ -526,10 +567,13 @@ class SearchValue(NamedTuple):
     raw_value: str | float | datetime | Sequence[float] | Sequence[str]
     # Used for top events where we don't want to modify the raw value at all
     use_raw_value: bool = False
+    is_regex: bool = False
 
     @property
     def value(self) -> Any:
-        if self.use_raw_value:
+        # Escape sequences are meaningful to the regex engine, so a pattern passes through
+        # untouched. `\*` is a literal asterisk there, not an escaped wildcard.
+        if self.use_raw_value or self.is_regex:
             return self.raw_value
         elif self.is_wildcard() and isinstance(self.raw_value, str):
             return translate_wildcard(self.raw_value)
@@ -549,17 +593,20 @@ def to_query_string(self) -> str:
         # we do that because a simple str() would not be usable for strings
         # str(["a","b"]) == "['a', 'b']" but we would like "[a,b]"
         if isinstance(self.raw_value, (list, tuple)):
-            ret_val = ", ".join(str(x) for x in self.raw_value)
+            ret_val = ", ".join(self._serialize(x) for x in self.raw_value)
             ret_val = f"[{ret_val}]"
             return ret_val
         elif isinstance(self.raw_value, datetime):
             return self.raw_value.isoformat()
         else:
-            return str(self.value)
+            return self._serialize(self.value)
+
+    def _serialize(self, value: Any) -> str:
+        return quote_regex_pattern(str(value)) if self.is_regex else str(value)
 
     def is_wildcard(self) -> bool:
-        # If we're using the raw value only it'll never be a wildcard
-        if self.use_raw_value:
+        # The raw value is never a wildcard, and a `*` in a regex is a quantifier
+        if self.use_raw_value or self.is_regex:
             return False
         if self.is_str_sequence():
             return isinstance(self.raw_value, list) and any(
@@ -665,12 +712,14 @@ def __str__(self) -> str:
         return f"{self.key.name}{self.operator}{self.value.raw_value}"
 
     def to_query_string(self) -> str:
+        # The marker sits between the `:` and the operator, matching the grammar's ordering
+        marker = REGEX_OPERATOR if self.value.is_regex else ""
         if self.operator == "IN":
-            return f"{self.key.name}:{self.value.to_query_string()}"
+            return f"{self.key.name}:{marker}{self.value.to_query_string()}"
         elif self.operator == "NOT IN":
-            return f"!{self.key.name}:{self.value.to_query_string()}"
+            return f"!{self.key.name}:{marker}{self.value.to_query_string()}"
         else:
-            return f"{self.key.name}:{self.operator}{self.value.to_query_string()}"
+            return f"{self.key.name}:{marker}{self.operator}{self.value.to_query_string()}"
 
     @property
     def is_negation(self) -> bool:
@@ -1443,7 +1492,9 @@ def visit_text_in_filter(
 
         operator = handle_negation(negation, operator)
 
-        if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, list):
+        if has_regex_op(wildcard_op):
+            search_value = as_regex_value(search_key.name, search_value)
+        elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, list):
             wildcarded_values = []
             found_wildcard_op = get_wildcard_op(wildcard_op)
             for value in search_value.raw_value:
@@ -1482,7 +1533,9 @@ def visit_text_filter(
 
         operator_s = handle_negation(negation, operator_s)
 
-        if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str):
+        if has_regex_op(wildcard_op):
+            search_value = as_regex_value(search_key.name, search_value)
+        elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str):
             wildcarded_value = gen_wildcard_value(
                 search_value.raw_value, get_wildcard_op(wildcard_op)
             )
@@ -1944,7 +1997,9 @@ def visit_array_includes_filter(
             raise InvalidSearchQuery("In Array Queries, only EQUAL/NOT_EQUAL operators are allowed")
         operator_s = handle_negation(negation, operator_s)
 
-        if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str):
+        if has_regex_op(wildcard_op):
+            search_value = as_regex_value(search_key.name, search_value)
+        elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str):
             wildcard_value = gen_wildcard_value(
                 search_value.raw_value, get_wildcard_op(wildcard_op)
             )

src/sentry/search/eap/constants.py

@@ -55,6 +55,16 @@
 }
 IN_OPERATORS = ["IN", "NOT IN"]
 
+# Snuba applies the same type rules to OP_REGEXP as to OP_LIKE: the pattern matches string
+# values, or the string elements of a string array.
+REGEXP_ATTRIBUTE_TYPES = frozenset(
+    {
+        AttributeKey.TYPE_STRING,
+        AttributeKey.TYPE_ARRAY,
+        AttributeKey.TYPE_ARRAY_STRING,
+    }
+)
+
 AGGREGATION_OPERATOR_MAP = {
     "=": AggregationComparisonFilter.OP_EQUALS,
     "!=": AggregationComparisonFilter.OP_NOT_EQUALS,

src/sentry/search/eap/resolver.py

@@ -541,6 +541,10 @@ def convert_term(self, term: event_search.SearchFilter) -> list[event_search.Sea
 
         converter = self.definitions.filter_aliases.get(name)
         if converter is not None:
+            if term.value.is_regex:
+                # The converters resolve values against Sentry models, so they would treat the
+                # pattern as a literal rather than matching against it
+                raise InvalidSearchQuery(f"Cannot use regular expressions with {name}")
             return converter(self.params, term, self)
 
         return [term]
@@ -566,6 +570,9 @@ def _resolve_term(
         resolved_column, context_definition = self.resolve_column(term.key.name)
         self._raise_if_hidden_api_attribute(term.key.name, resolved_column)
 
+        if context_definition is not None and term.value.is_regex:
+            raise InvalidSearchQuery(f"Cannot use regular expressions with {term.key.name}")
+
         value = term.value.value
         if self.params.is_timeseries_request and context_definition is not None:
             resolved_column, value = self.map_search_term_context_to_original_column(
@@ -582,6 +589,12 @@ def _resolve_term(
                 # Avoiding this for now, but we could theoretically do a wildcard search on the resolved contexts
                 raise InvalidSearchQuery(f"Cannot use wildcards with {term.key.name}")
 
+        if term.value.is_regex:
+            return (
+                self._resolve_regex_term(term, resolved_column),
+                context_definition,
+            )
+
         if term.value.is_wildcard():
             is_list = False
             if term.operator == "=":
@@ -886,6 +899,45 @@ def resolve_aggregate_term(
             context,
         )
 
+    def _resolve_regex_term(
+        self,
+        term: event_search.SearchFilter,
+        resolved_column: ResolvedAttribute,
+    ) -> TraceItemFilter:
+        if resolved_column.proto_definition.type not in constants.REGEXP_ATTRIBUTE_TYPES:
+            raise InvalidSearchQuery(
+                f"Cannot use regular expressions with {term.key.name}, it is not a string attribute"
+            )
+
+        patterns = to_list(term.value.raw_value)
+        # Snuba's `ignore_case` lowercases the pattern along with the value, rewriting `[A-Z]`
+        # and inverting escapes like `\D`. RE2's inline flag leaves the pattern intact.
+        prefix = "(?i)" if self.params.case_insensitive else ""
+        matches = [
+            TraceItemFilter(
+                comparison_filter=ComparisonFilter(
+                    key=resolved_column.proto_definition,
+                    op=ComparisonFilter.OP_REGEXP,
+                    value=AttributeValue(val_str=f"{prefix}{pattern}"),
+                )
+            )
+            for pattern in patterns
+        ]
+
+        matches_any = (
+            matches[0]
+            if len(matches) == 1
+            else TraceItemFilter(or_filter=OrFilter(filters=matches))
+        )
+
+        if term.operator in ("=", "IN"):
+            return matches_any
+        elif term.operator in ("!=", "NOT IN"):
+            # There is no OP_NOT_REGEXP, so negation is expressed by wrapping the match
+            return TraceItemFilter(not_filter=NotFilter(filters=[matches_any]))
+
+        raise InvalidSearchQuery(f"Cannot use operator: {term.operator} with regular expressions")
+
     def _resolve_search_value(
         self,
         column: ResolvedAttribute,

src/sentry/search/events/constants.py

@@ -327,6 +327,16 @@ class ThresholdDict(TypedDict):
     "ends_with": f"{WILDCARD_UNICODE}EndsWith{WILDCARD_UNICODE}",
 }
 
+# Deliberately kept out of WILDCARD_OPERATOR_MAP: it shares the marker encoding, but a regex
+# pattern must reach the backend verbatim rather than being rewritten into a wildcard pattern.
+REGEX_OPERATOR = f"{WILDCARD_UNICODE}Matches{WILDCARD_UNICODE}"
+
+# RE2, which backs the ClickHouse `match` this compiles to, rejects the PCRE extensions that
+# Python's `re` accepts, and ClickHouse only reports that as a query failure once the pattern
+# has already reached it. The first branch consumes escaped backslashes, so that a pattern
+# like `\\1` reads as a literal backslash followed by a digit.
+UNSUPPORTED_REGEX_SYNTAX = re.compile(r"\\\\|(?P<unsupported>\\[1-9]|\\Z|\(\?(?:[=!>#(]|<[=!]|P=))")
+
 MAX_SEARCH_RELEASES = 1000
 SEMVER_EMPTY_RELEASE = "____SENTRY_EMPTY_RELEASE____"
 SEMVER_WILDCARDS = frozenset(["X", "*"])