diff --git a/pyproject.toml b/pyproject.toml index 42cdb2eeadd7..de64598dcc7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ dependencies = [ "sentry-ophio>=1.1.3", # sentry-options is only used in getsentry for now "sentry-options>=1.2.8", - "sentry-protos>=0.67.0", + "sentry-protos>=0.68.0", "sentry-redis-tools>=0.5.0", "sentry-relay>=0.9.28", "sentry-scm==1.5.0", diff --git a/src/sentry/api/event_search.py b/src/sentry/api/event_search.py index 900a3c4b59f4..b7cd92fc4728 100644 --- a/src/sentry/api/event_search.py +++ b/src/sentry/api/event_search.py @@ -17,12 +17,14 @@ DURATION_UNITS, NOT_HAS_FILTER_ERROR_MESSAGE, OPERATOR_NEGATION_MAP, + REGEX_OPERATOR, SEARCH_MAP, SEMVER_ALIAS, SEMVER_BUILD_ALIAS, SIZE_UNITS, TAG_KEY_RE, TEAM_KEY_TRANSACTION_ALIAS, + UNSUPPORTED_REGEX_SYNTAX, WILDCARD_OPERATOR_MAP, ) from sentry.search.events.fields import FIELD_ALIASES, FUNCTIONS @@ -189,7 +191,7 @@ # NOTE: These wildcard operators are internal implementation details and # should not be included in product docs. Users should use `*` instead. -wildcard_op = wildcard_unicode (contains / starts_with / ends_with) wildcard_unicode +wildcard_op = wildcard_unicode (contains / starts_with / ends_with / matches) wildcard_unicode # See: https://stackoverflow.com/a/39617181/790169 in_value_termination = in_value_char (!in_value_end in_value_char)* in_value_end @@ -233,6 +235,7 @@ contains = "Contains" starts_with = "StartsWith" ends_with = "EndsWith" +matches = "Matches" comma = "," spaces = " "* @@ -418,6 +421,44 @@ def get_wildcard_op(node: Node | Sequence[Node]) -> str: return "" +def has_regex_op(node: Node | Sequence[Node]) -> bool: + return get_wildcard_op(node) == REGEX_OPERATOR + + +def quote_regex_pattern(pattern: str) -> str: + """Regex patterns are always quoted when serialized back to a query string, because the + unquoted value grammar rejects the parentheses and spaces that patterns routinely contain.""" + escaped = pattern.replace('"', '\\"') + return f'"{escaped}"' + + +def validate_regex_pattern(key: str, pattern: str) -> None: + if not pattern: + raise InvalidSearchQuery(f"{key}: Empty regex pattern") + + for match in UNSUPPORTED_REGEX_SYNTAX.finditer(pattern): + unsupported = match.group("unsupported") + if unsupported is not None: + raise InvalidSearchQuery( + f"{key}: Invalid regex: `{unsupported}` is not supported. " + "Patterns are matched with RE2, which has no backreferences, lookaround, " + "or other PCRE extensions." + ) + + try: + re.compile(pattern) + except re.error as exc: + raise InvalidSearchQuery(f"{key}: Invalid regex: {exc.msg}") + + +def as_regex_value(key: str, value: SearchValue) -> SearchValue: + patterns = value.raw_value if isinstance(value.raw_value, (list, tuple)) else [value.raw_value] + for pattern in patterns: + if isinstance(pattern, str): + validate_regex_pattern(key, pattern) + return value._replace(is_regex=True) + + def add_leading_wildcard(value: str) -> str: if value.startswith('"') and value.endswith('"'): return f"*{value[1:-1]}" @@ -526,10 +567,13 @@ class SearchValue(NamedTuple): raw_value: str | float | datetime | Sequence[float] | Sequence[str] # Used for top events where we don't want to modify the raw value at all use_raw_value: bool = False + is_regex: bool = False @property def value(self) -> Any: - if self.use_raw_value: + # Escape sequences are meaningful to the regex engine, so a pattern passes through + # untouched. `\*` is a literal asterisk there, not an escaped wildcard. + if self.use_raw_value or self.is_regex: return self.raw_value elif self.is_wildcard() and isinstance(self.raw_value, str): return translate_wildcard(self.raw_value) @@ -549,17 +593,20 @@ def to_query_string(self) -> str: # we do that because a simple str() would not be usable for strings # str(["a","b"]) == "['a', 'b']" but we would like "[a,b]" if isinstance(self.raw_value, (list, tuple)): - ret_val = ", ".join(str(x) for x in self.raw_value) + ret_val = ", ".join(self._serialize(x) for x in self.raw_value) ret_val = f"[{ret_val}]" return ret_val elif isinstance(self.raw_value, datetime): return self.raw_value.isoformat() else: - return str(self.value) + return self._serialize(self.value) + + def _serialize(self, value: Any) -> str: + return quote_regex_pattern(str(value)) if self.is_regex else str(value) def is_wildcard(self) -> bool: - # If we're using the raw value only it'll never be a wildcard - if self.use_raw_value: + # The raw value is never a wildcard, and a `*` in a regex is a quantifier + if self.use_raw_value or self.is_regex: return False if self.is_str_sequence(): return isinstance(self.raw_value, list) and any( @@ -665,12 +712,14 @@ def __str__(self) -> str: return f"{self.key.name}{self.operator}{self.value.raw_value}" def to_query_string(self) -> str: + # The marker sits between the `:` and the operator, matching the grammar's ordering + marker = REGEX_OPERATOR if self.value.is_regex else "" if self.operator == "IN": - return f"{self.key.name}:{self.value.to_query_string()}" + return f"{self.key.name}:{marker}{self.value.to_query_string()}" elif self.operator == "NOT IN": - return f"!{self.key.name}:{self.value.to_query_string()}" + return f"!{self.key.name}:{marker}{self.value.to_query_string()}" else: - return f"{self.key.name}:{self.operator}{self.value.to_query_string()}" + return f"{self.key.name}:{marker}{self.operator}{self.value.to_query_string()}" @property def is_negation(self) -> bool: @@ -1443,7 +1492,9 @@ def visit_text_in_filter( operator = handle_negation(negation, operator) - if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, list): + if has_regex_op(wildcard_op): + search_value = as_regex_value(search_key.name, search_value) + elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, list): wildcarded_values = [] found_wildcard_op = get_wildcard_op(wildcard_op) for value in search_value.raw_value: @@ -1482,7 +1533,9 @@ def visit_text_filter( operator_s = handle_negation(negation, operator_s) - if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): + if has_regex_op(wildcard_op): + search_value = as_regex_value(search_key.name, search_value) + elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): wildcarded_value = gen_wildcard_value( search_value.raw_value, get_wildcard_op(wildcard_op) ) @@ -1944,7 +1997,9 @@ def visit_array_includes_filter( raise InvalidSearchQuery("In Array Queries, only EQUAL/NOT_EQUAL operators are allowed") operator_s = handle_negation(negation, operator_s) - if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): + if has_regex_op(wildcard_op): + search_value = as_regex_value(search_key.name, search_value) + elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): wildcard_value = gen_wildcard_value( search_value.raw_value, get_wildcard_op(wildcard_op) ) diff --git a/src/sentry/search/eap/constants.py b/src/sentry/search/eap/constants.py index 36dddf44b2b2..c9ae55d977e2 100644 --- a/src/sentry/search/eap/constants.py +++ b/src/sentry/search/eap/constants.py @@ -55,6 +55,16 @@ } IN_OPERATORS = ["IN", "NOT IN"] +# Snuba applies the same type rules to OP_REGEXP as to OP_LIKE: the pattern matches string +# values, or the string elements of a string array. +REGEXP_ATTRIBUTE_TYPES = frozenset( + { + AttributeKey.TYPE_STRING, + AttributeKey.TYPE_ARRAY, + AttributeKey.TYPE_ARRAY_STRING, + } +) + AGGREGATION_OPERATOR_MAP = { "=": AggregationComparisonFilter.OP_EQUALS, "!=": AggregationComparisonFilter.OP_NOT_EQUALS, diff --git a/src/sentry/search/eap/resolver.py b/src/sentry/search/eap/resolver.py index 57972fc795a4..bfba1907e724 100644 --- a/src/sentry/search/eap/resolver.py +++ b/src/sentry/search/eap/resolver.py @@ -541,6 +541,10 @@ def convert_term(self, term: event_search.SearchFilter) -> list[event_search.Sea converter = self.definitions.filter_aliases.get(name) if converter is not None: + if term.value.is_regex: + # The converters resolve values against Sentry models, so they would treat the + # pattern as a literal rather than matching against it + raise InvalidSearchQuery(f"Cannot use regular expressions with {name}") return converter(self.params, term, self) return [term] @@ -566,6 +570,9 @@ def _resolve_term( resolved_column, context_definition = self.resolve_column(term.key.name) self._raise_if_hidden_api_attribute(term.key.name, resolved_column) + if context_definition is not None and term.value.is_regex: + raise InvalidSearchQuery(f"Cannot use regular expressions with {term.key.name}") + value = term.value.value if self.params.is_timeseries_request and context_definition is not None: resolved_column, value = self.map_search_term_context_to_original_column( @@ -582,6 +589,12 @@ def _resolve_term( # Avoiding this for now, but we could theoretically do a wildcard search on the resolved contexts raise InvalidSearchQuery(f"Cannot use wildcards with {term.key.name}") + if term.value.is_regex: + return ( + self._resolve_regex_term(term, resolved_column), + context_definition, + ) + if term.value.is_wildcard(): is_list = False if term.operator == "=": @@ -886,6 +899,45 @@ def resolve_aggregate_term( context, ) + def _resolve_regex_term( + self, + term: event_search.SearchFilter, + resolved_column: ResolvedAttribute, + ) -> TraceItemFilter: + if resolved_column.proto_definition.type not in constants.REGEXP_ATTRIBUTE_TYPES: + raise InvalidSearchQuery( + f"Cannot use regular expressions with {term.key.name}, it is not a string attribute" + ) + + patterns = to_list(term.value.raw_value) + # Snuba's `ignore_case` lowercases the pattern along with the value, rewriting `[A-Z]` + # and inverting escapes like `\D`. RE2's inline flag leaves the pattern intact. + prefix = "(?i)" if self.params.case_insensitive else "" + matches = [ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str=f"{prefix}{pattern}"), + ) + ) + for pattern in patterns + ] + + matches_any = ( + matches[0] + if len(matches) == 1 + else TraceItemFilter(or_filter=OrFilter(filters=matches)) + ) + + if term.operator in ("=", "IN"): + return matches_any + elif term.operator in ("!=", "NOT IN"): + # There is no OP_NOT_REGEXP, so negation is expressed by wrapping the match + return TraceItemFilter(not_filter=NotFilter(filters=[matches_any])) + + raise InvalidSearchQuery(f"Cannot use operator: {term.operator} with regular expressions") + def _resolve_search_value( self, column: ResolvedAttribute, diff --git a/src/sentry/search/events/constants.py b/src/sentry/search/events/constants.py index 4beb624f906b..73487abd32d5 100644 --- a/src/sentry/search/events/constants.py +++ b/src/sentry/search/events/constants.py @@ -327,6 +327,16 @@ class ThresholdDict(TypedDict): "ends_with": f"{WILDCARD_UNICODE}EndsWith{WILDCARD_UNICODE}", } +# Deliberately kept out of WILDCARD_OPERATOR_MAP: it shares the marker encoding, but a regex +# pattern must reach the backend verbatim rather than being rewritten into a wildcard pattern. +REGEX_OPERATOR = f"{WILDCARD_UNICODE}Matches{WILDCARD_UNICODE}" + +# RE2, which backs the ClickHouse `match` this compiles to, rejects the PCRE extensions that +# Python's `re` accepts, and ClickHouse only reports that as a query failure once the pattern +# has already reached it. The first branch consumes escaped backslashes, so that a pattern +# like `\\1` reads as a literal backslash followed by a digit. +UNSUPPORTED_REGEX_SYNTAX = re.compile(r"\\\\|(?P\\[1-9]|\\Z|\(\?(?:[=!>#(]|<[=!]|P=))") + MAX_SEARCH_RELEASES = 1000 SEMVER_EMPTY_RELEASE = "____SENTRY_EMPTY_RELEASE____" SEMVER_WILDCARDS = frozenset(["X", "*"]) diff --git a/tests/sentry/api/test_event_search.py b/tests/sentry/api/test_event_search.py index 50bd84443186..69a6c36cab08 100644 --- a/tests/sentry/api/test_event_search.py +++ b/tests/sentry/api/test_event_search.py @@ -27,6 +27,7 @@ from sentry.constants import MODULE_ROOT from sentry.exceptions import IncompatibleMetricsQuery, InvalidSearchQuery from sentry.search.events.constants import ( + REGEX_OPERATOR, TEAM_KEY_TRANSACTION_ALIAS, WILDCARD_OPERATOR_MAP, WILDCARD_UNICODE, @@ -1501,6 +1502,125 @@ def test_handles_starts_with_wildcard_op_translations(query, expected) -> None: assert actual == expected +@pytest.mark.parametrize( + ["query", "expected_operator", "expected_value"], + [ + pytest.param(f"span.op:{REGEX_OPERATOR}^test$", "=", "^test$", id="anchored"), + pytest.param(f"!span.op:{REGEX_OPERATOR}^test$", "!=", "^test$", id="negated"), + pytest.param(f"span.op:{REGEX_OPERATOR}a*b", "=", "a*b", id="quantifier"), + pytest.param(f"span.op:{REGEX_OPERATOR}a\\*b", "=", "a\\*b", id="escaped asterisk"), + pytest.param(f"span.op:{REGEX_OPERATOR}a\\d+", "=", "a\\d+", id="character class"), + pytest.param( + f"span.op:{REGEX_OPERATOR}a\\\\1", "=", "a\\\\1", id="escaped backslash before digit" + ), + pytest.param( + f"span.op:{REGEX_OPERATOR}a\\\\Z", "=", "a\\\\Z", id="escaped backslash before Z" + ), + pytest.param(f'span.op:{REGEX_OPERATOR}"a b|c"', "=", "a b|c", id="quoted"), + pytest.param(f"span.op:{REGEX_OPERATOR}[^foo, bar$]", "IN", ["^foo", "bar$"], id="in list"), + pytest.param( + f"!span.op:{REGEX_OPERATOR}[^foo, bar$]", "NOT IN", ["^foo", "bar$"], id="not in list" + ), + ], +) +def test_parses_regex_op_without_rewriting_the_pattern( + query, expected_operator, expected_value +) -> None: + filters = parse_search_query(query) + assert len(filters) == 1 + assert isinstance(filters[0], SearchFilter) + assert filters[0].operator == expected_operator + assert filters[0].value.is_regex is True + assert filters[0].value.is_wildcard() is False + assert filters[0].value.value == expected_value + + +@pytest.mark.parametrize( + "query", + [ + pytest.param(f"span.op:{REGEX_OPERATOR}^test$", id="scalar"), + pytest.param(f"!span.op:{REGEX_OPERATOR}^test$", id="negated"), + pytest.param(f"span.op:{REGEX_OPERATOR}[^foo, bar$]", id="in list"), + pytest.param(f"!span.op:{REGEX_OPERATOR}[^foo, bar$]", id="not in list"), + pytest.param(f'span.op:{REGEX_OPERATOR}"^(foo|bar) baz$"', id="parens and spaces"), + pytest.param(f'span.op:{REGEX_OPERATOR}"\\"quoted\\""', id="embedded quotes"), + pytest.param(f'span.op:{REGEX_OPERATOR}["^(a|b)", "(c|d)$"]', id="parens in list"), + ], +) +def test_round_trips_a_regex_op_through_to_query_string(query) -> None: + filters = parse_search_query(query) + assert len(filters) == 1 + assert isinstance(filters[0], SearchFilter) + assert parse_search_query(filters[0].to_query_string()) == filters + + +UNSUPPORTED_REGEX_MESSAGE = ( + "Patterns are matched with RE2, which has no backreferences, lookaround, " + "or other PCRE extensions." +) + + +@pytest.mark.parametrize( + ["query", "expected_message"], + [ + pytest.param( + f"span.op:{REGEX_OPERATOR}[a-", + "span.op: Invalid regex: unterminated character set", + id="unterminated character set", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(foo"', + "span.op: Invalid regex: missing ), unterminated subpattern", + id="unterminated group", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(foo)\\1"', + "span.op: Invalid regex: `\\1` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="backreference", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"foo(?=bar)"', + "span.op: Invalid regex: `(?=` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="lookahead", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"foo(?foo)"', + "span.op: Invalid regex: `(?>` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="atomic group", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(foo)(?(1)bar|baz)"', + "span.op: Invalid regex: `(?(` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="conditional", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(?Pfoo)(?P=name)"', + "span.op: Invalid regex: `(?P=` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="named backreference", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}""', + "span.op: Empty regex pattern", + id="empty quoted pattern", + ), + ], +) +def test_rejects_an_invalid_regex_pattern(query, expected_message) -> None: + with pytest.raises(InvalidSearchQuery) as err: + parse_search_query(query) + assert str(err.value) == expected_message + + @pytest.mark.parametrize( ["query", "expected"], [ diff --git a/tests/sentry/search/eap/test_ourlogs.py b/tests/sentry/search/eap/test_ourlogs.py index 3586a8438442..a0f17af44a25 100644 --- a/tests/sentry/search/eap/test_ourlogs.py +++ b/tests/sentry/search/eap/test_ourlogs.py @@ -13,13 +13,16 @@ from sentry_protos.snuba.v1.trace_item_filter_pb2 import ( AndFilter, ComparisonFilter, + NotFilter, OrFilter, TraceItemFilter, ) +from sentry.exceptions import InvalidSearchQuery from sentry.search.eap.ourlogs.definitions import OURLOG_DEFINITIONS from sentry.search.eap.resolver import SearchResolver from sentry.search.eap.types import SearchResolverConfig +from sentry.search.events.constants import REGEX_OPERATOR from sentry.search.events.types import SnubaParams @@ -305,6 +308,162 @@ def test_internal_name_resolves_with_normalizer(self) -> None: ) assert having is None + def test_regex_query(self) -> None: + where, having, _ = self.resolver.resolve_query(f"message:{REGEX_OPERATOR}^ERROR") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ) + assert having is None + + def test_regex_query_negated(self) -> None: + where, having, _ = self.resolver.resolve_query(f"!message:{REGEX_OPERATOR}^ERROR") + assert where == TraceItemFilter( + not_filter=NotFilter( + filters=[ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", type=AttributeKey.Type.TYPE_STRING + ), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ) + ] + ) + ) + assert having is None + + def test_regex_query_on_an_attribute(self) -> None: + where, having, _ = self.resolver.resolve_query(f"foo:{REGEX_OPERATOR}ba[rz]") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="foo", type=AttributeKey.Type.TYPE_STRING), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="ba[rz]"), + ) + ) + assert having is None + + def test_regex_query_keeps_the_pattern_verbatim(self) -> None: + """Regex metacharacters must not be rewritten the way wildcard patterns are.""" + where, _, _ = self.resolver.resolve_query(f"message:{REGEX_OPERATOR}a*b%c_d\\*e") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="a*b%c_d\\*e"), + ) + ) + + def test_regex_query_is_case_insensitive_when_requested(self) -> None: + resolver = SearchResolver( + params=SnubaParams(case_insensitive=True), + config=SearchResolverConfig(), + definitions=OURLOG_DEFINITIONS, + ) + where, _, _ = resolver.resolve_query(f"message:{REGEX_OPERATOR}^[A-Z]rror") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="(?i)^[A-Z]rror"), + ) + ) + + def test_regex_in_filter(self) -> None: + where, having, _ = self.resolver.resolve_query(f"message:{REGEX_OPERATOR}[^ERROR, ^WARN]") + assert where == TraceItemFilter( + or_filter=OrFilter( + filters=[ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", type=AttributeKey.Type.TYPE_STRING + ), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ), + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", type=AttributeKey.Type.TYPE_STRING + ), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="^WARN"), + ) + ), + ] + ) + ) + assert having is None + + def test_regex_not_in_filter(self) -> None: + where, having, _ = self.resolver.resolve_query(f"!message:{REGEX_OPERATOR}[^ERROR, ^WARN]") + assert where == TraceItemFilter( + not_filter=NotFilter( + filters=[ + TraceItemFilter( + or_filter=OrFilter( + filters=[ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", + type=AttributeKey.Type.TYPE_STRING, + ), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ), + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", + type=AttributeKey.Type.TYPE_STRING, + ), + op=ComparisonFilter.OP_REGEXP, + value=AttributeValue(val_str="^WARN"), + ) + ), + ] + ) + ) + ] + ) + ) + assert having is None + + def test_regex_query_raises_when_the_key_is_backed_by_a_filter_alias(self) -> None: + with pytest.raises(InvalidSearchQuery) as err: + self.resolver.resolve_query(f"release:{REGEX_OPERATOR}^1\\.2") + assert str(err.value) == "Cannot use regular expressions with release" + + def test_regex_query_raises_when_the_key_is_backed_by_a_virtual_column(self) -> None: + with pytest.raises(InvalidSearchQuery) as err: + self.resolver.resolve_query(f"project:{REGEX_OPERATOR}^sen") + assert str(err.value) == "Cannot use regular expressions with project" + + def test_regex_query_raises_on_a_virtual_column_in_a_timeseries_request(self) -> None: + resolver = SearchResolver( + params=SnubaParams(granularity_secs=60), + config=SearchResolverConfig(), + definitions=OURLOG_DEFINITIONS, + ) + with pytest.raises(InvalidSearchQuery) as err: + resolver.resolve_query(f"project:{REGEX_OPERATOR}^sen") + assert str(err.value) == "Cannot use regular expressions with project" + + def test_regex_query_raises_when_the_attribute_is_not_a_string(self) -> None: + with pytest.raises(InvalidSearchQuery) as err: + self.resolver.resolve_query(f"tags[foo,boolean]:{REGEX_OPERATOR}tru.") + assert "not a string attribute" in str(err.value) + def test_internal_trace_id_resolves_with_normalizer(self) -> None: """Using the internal name 'sentry.trace_id' resolves with normalizer.""" where, having, _ = self.resolver.resolve_query( diff --git a/tests/snuba/api/endpoints/test_organization_events_ourlogs.py b/tests/snuba/api/endpoints/test_organization_events_ourlogs.py index 3c43c20d0b6a..27eee02cb1e9 100644 --- a/tests/snuba/api/endpoints/test_organization_events_ourlogs.py +++ b/tests/snuba/api/endpoints/test_organization_events_ourlogs.py @@ -8,6 +8,7 @@ from sentry.conf.types.sentry_config import SentryMode from sentry.constants import DataCategory from sentry.search.eap import constants +from sentry.search.events.constants import REGEX_OPERATOR from sentry.testutils.cases import OutcomesSnubaTest from sentry.testutils.helpers import parse_link_header from sentry.testutils.helpers.datetime import before_now @@ -140,6 +141,150 @@ def test_free_text_wildcard_filter(self) -> None: assert meta["dataset"] == self.dataset + def test_regex_filter(self) -> None: + logs = [ + self.create_ourlog( + {"body": "ERROR [42] disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "WARN [7] disk filling up"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'message:{REGEX_OPERATOR}"^ERROR \\[\\d+\\]"', + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["ERROR [42] disk full"] + + def test_regex_filter_negated(self) -> None: + logs = [ + self.create_ourlog( + {"body": "ERROR [42] disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "WARN [7] disk filling up"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'!message:{REGEX_OPERATOR}"^ERROR \\[\\d+\\]"', + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["WARN [7] disk filling up"] + + def test_regex_filter_in_list(self) -> None: + logs = [ + self.create_ourlog( + {"body": "ERROR [42] disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "WARN [7] disk filling up"}, + timestamp=self.nine_mins_ago, + ), + self.create_ourlog( + {"body": "INFO [1] all good"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'message:{REGEX_OPERATOR}["^ERROR", "^WARN"]', + "orderby": "log.body", + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == [ + "ERROR [42] disk full", + "WARN [7] disk filling up", + ] + + def test_regex_filter_on_an_attribute(self) -> None: + logs = [ + self.create_ourlog( + {"body": "first"}, + attributes={"release": "1.2.3"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "second"}, + attributes={"release": "nightly"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'tags[release,string]:{REGEX_OPERATOR}"^\\d+\\.\\d+"', + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["first"] + + def test_regex_filter_case_insensitive(self) -> None: + logs = [ + self.create_ourlog( + {"body": "Error: disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "0 problems"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'message:{REGEX_OPERATOR}"^[A-Z]RROR\\D+"', + "caseInsensitive": "1", + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["Error: disk full"] + + def test_regex_filter_rejects_an_invalid_pattern(self) -> None: + response = self.do_request( + { + "field": ["log.body"], + "query": f"message:{REGEX_OPERATOR}[a-", + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 400, response.content + assert "Invalid regex" in response.data["detail"] + def test_pagination(self) -> None: logs = [ self.create_ourlog( diff --git a/uv.lock b/uv.lock index 210934d8b7ee..2dc52534c11c 100644 --- a/uv.lock +++ b/uv.lock @@ -2385,7 +2385,7 @@ requires-dist = [ { name = "sentry-kafka-schemas", specifier = ">=2.2.0" }, { name = "sentry-ophio", specifier = ">=1.1.3" }, { name = "sentry-options", specifier = ">=1.2.8" }, - { name = "sentry-protos", specifier = ">=0.67.0" }, + { name = "sentry-protos", specifier = ">=0.68.0" }, { name = "sentry-redis-tools", specifier = ">=0.5.0" }, { name = "sentry-relay", specifier = ">=0.9.28" }, { name = "sentry-scm", specifier = "==1.5.0" }, @@ -2565,7 +2565,7 @@ wheels = [ [[package]] name = "sentry-protos" -version = "0.67.0" +version = "0.69.0" source = { registry = "https://pypi.devinfra.sentry.io/simple" } dependencies = [ { name = "grpc-stubs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -2573,7 +2573,7 @@ dependencies = [ { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] wheels = [ - { url = "https://pypi.devinfra.sentry.io/wheels/sentry_protos-0.67.0-py3-none-any.whl", hash = "sha256:e09a083a630e7e58f5599b9c18ac777305077e066ffb4245721ea603eaed616f" }, + { url = "https://pypi.devinfra.sentry.io/wheels/sentry_protos-0.69.0-py3-none-any.whl", hash = "sha256:5ae4d39464c75ea3382c76c186fa4da952fad927a9f41230a579e91e809010c7" }, ] [[package]]