From 883c5a6c5122c0c615ccf324141be457e44ede7c Mon Sep 17 00:00:00 2001 From: Yi Ding Date: Thu, 11 Jun 2026 22:32:30 -0700 Subject: [PATCH] fix(processor): delimit sentinel escape fallback --- sentencesplit/processor.py | 80 +++++++++++-------- tests/regression/test_library_review_fixes.py | 16 ++++ 2 files changed, 64 insertions(+), 32 deletions(-) diff --git a/sentencesplit/processor.py b/sentencesplit/processor.py index 5b383f4..0cee9c9 100644 --- a/sentencesplit/processor.py +++ b/sentencesplit/processor.py @@ -142,9 +142,13 @@ def _resplit_multi_sentence_quote( # Private-use codepoints (BMP + both supplementary planes) used as escape # targets. Targets are chosen per call from this pool to be absent from the # input. If adversarial input occupies every single private-use character, the -# escape target grows into a private-use string token that is absent from the -# input, preserving a clean bijection without raising from segmentation. +# escape target grows into a delimited private-use string token. The delimiter +# is a noncharacter chosen absent from the input, which keeps restore matches +# aligned to whole escape tokens instead of arbitrary private-use substrings. _PRIVATE_USE_RANGES = ((0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)) +_NONCHARACTER_DELIMITER_RANGES = ((0xFDD0, 0xFDEF),) + tuple( + (plane + 0xFFFE, plane + 0xFFFF) for plane in range(0, 0x110000, 0x10000) +) def _iter_private_use_chars(): @@ -153,23 +157,26 @@ def _iter_private_use_chars(): yield chr(cp) -def _iter_private_use_tokens(token_len: int): - if token_len == 1: - yield from _iter_private_use_chars() - return +def _iter_delimited_private_use_tokens(body_len: int, delimiter: str): alphabet = tuple(_iter_private_use_chars()) - if len(alphabet) < 2: + if not alphabet: return - for chars in product(alphabet, repeat=token_len): - yield "".join(chars) + for chars in product(alphabet, repeat=body_len): + yield delimiter + "".join(chars) + delimiter + + +def _iter_noncharacter_delimiters(): + for lo, hi in _NONCHARACTER_DELIMITER_RANGES: + for cp in range(lo, hi + 1): + yield chr(cp) -def _private_use_substrings(text: str, token_len: int) -> set[str]: - if token_len == 1: - return set(text) - if len(text) < token_len: - return set() - return {text[i : i + token_len] for i in range(len(text) - token_len + 1)} +def _absent_noncharacter_delimiter(text: str) -> str: + occupied = set(text) + for delimiter in _iter_noncharacter_delimiters(): + if delimiter not in occupied: + return delimiter + raise ValueError("At least one absent noncharacter delimiter is required") def _build_sentinel_escape_tables( @@ -178,8 +185,10 @@ def _build_sentinel_escape_tables( """Return escape/restore tables for reserved sentinels in *text*. The escape values are private-use tokens that do not occur in the input. - Single private-use characters are used for normal inputs; longer tokens are - selected only if an adversarial input exhausts the single-character pool. + Single private-use characters are used for normal inputs; if an adversarial + input exhausts the single-character pool, longer private-use token bodies + are wrapped in an absent delimiter. The delimiter prevents restore matches + from starting inside neighboring original private-use text. Returns ``(escape, restore, restore_re)`` where ``escape`` maps codepoints to tokens for ``str.translate``, ``restore`` maps each token back to its @@ -189,23 +198,30 @@ def _build_sentinel_escape_tables( per-token ``str.replace`` could match a window straddling two adjacent escaped sentinels and corrupt the round-trip. """ - token_len = 1 - while True: - occupied = _private_use_substrings(text, token_len) - tokens: list[str] = [] - saw_candidate = False - for token in _iter_private_use_tokens(token_len): - saw_candidate = True - if token not in occupied: + tokens = [] + occupied = set(text) + for token in _iter_private_use_chars(): + if token not in occupied: + tokens.append(token) + if len(tokens) == len(_RESERVED_SENTINELS): + break + if len(tokens) < len(_RESERVED_SENTINELS): + delimiter = _absent_noncharacter_delimiter(text) + body_len = 1 + while len(tokens) < len(_RESERVED_SENTINELS): + saw_candidate = False + for token in _iter_delimited_private_use_tokens(body_len, delimiter): + saw_candidate = True tokens.append(token) if len(tokens) == len(_RESERVED_SENTINELS): - escape = {ord(ch): token for ch, token in zip(_RESERVED_SENTINELS, tokens, strict=True)} - restore = {token: ch for ch, token in zip(_RESERVED_SENTINELS, tokens, strict=True)} - restore_re = re.compile("|".join(re.escape(token) for token in tokens)) - return escape, restore, restore_re - if not saw_candidate: - raise ValueError("At least two private-use escape codepoints are required") - token_len += 1 + break + if not saw_candidate: + raise ValueError("At least one private-use escape codepoint is required") + body_len += 1 + escape = {ord(ch): token for ch, token in zip(_RESERVED_SENTINELS, tokens, strict=True)} + restore = {token: ch for ch, token in zip(_RESERVED_SENTINELS, tokens, strict=True)} + restore_re = re.compile("|".join(re.escape(token) for token in sorted(tokens, key=len, reverse=True))) + return escape, restore, restore_re def _split_on_uppercase_boundary(text: str, whitespace_re: re.Pattern[str]) -> list[str] | None: diff --git a/tests/regression/test_library_review_fixes.py b/tests/regression/test_library_review_fixes.py index d792a95..00c2868 100644 --- a/tests/regression/test_library_review_fixes.py +++ b/tests/regression/test_library_review_fixes.py @@ -379,6 +379,22 @@ def test_sentinel_restore_is_overlap_safe_for_adjacent_multichar_tokens(monkeypa assert clean.segment_clean(multi) == ["Pair ♭∯ here.", "And more."] +def test_sentinel_restore_does_not_match_across_original_private_use_boundary(monkeypatch): + """Delimited fallback tokens must not restore a substring that straddles an + original private-use character and an escaped reserved sentinel.""" + from sentencesplit import processor as _proc + from sentencesplit.languages import Language + + monkeypatch.setattr(_proc, "_PRIVATE_USE_RANGES", ((0xE000, 0xE001),)) + + en = Language.get_language_code("en") + clean = sentencesplit.Segmenter(language="en", clean=True) + text = "Has \ue000∯ here. And more." + + assert _proc.Processor(text, en).process() == ["Has \ue000∯ here.", "And more."] + assert clean.segment_clean(text) == ["Has \ue000∯ here.", "And more."] + + def test_escaped_html_rule_is_not_redos_vulnerable(): import time