diff --git a/benchmarks/abbr_scan_compare.py b/benchmarks/abbr_scan_compare.py new file mode 100644 index 0000000..4b575ac --- /dev/null +++ b/benchmarks/abbr_scan_compare.py @@ -0,0 +1,75 @@ +"""Head-to-head: our Aho-Corasick abbreviation scan vs a pySBD-style `in` loop. + +Settles which abbreviation-*discovery* mechanism is faster at each input size. +Both find the identical set of present abbreviations; this times only that +discovery step (not the per-occurrence regex replacement that follows it). + +Finding (see analysis/SHORT_STRING_LATENCY_PLAN.md §2a): our pure-Python +automaton is faster on short input; the `in` loop only overtakes around +~100-150 chars. So the scan is *not* the short-string bottleneck — it is ~3% of +the call — and must not be replaced. + +Run with: + uv run python benchmarks/abbr_scan_compare.py +""" + +from __future__ import annotations + +import time + +from sentencesplit.abbreviation_replacer import _AbbreviationData +from sentencesplit.languages import LANGUAGE_CODES + +_MEDIUM = ( + "dr. smith went to washington. he arrived on jan. 5th at 3 p.m. the model is gpt 3.1 and it is fast. " + "that is all for now. goodbye. she paid $4.50 for the u.s. edition (vol. 2, p. 17). mr. lee agreed." +) +CASES = { + "tiny (15c)": "dr. smith left.", + "short (87c)": "dr. smith went to washington. he arrived on jan. 5th at 3 p.m. and met with sen. jones.", + "medium (198c)": _MEDIUM, + "large (4k)": " ".join([_MEDIUM] * 20), + "huge (40k)": " ".join([_MEDIUM] * 200), +} + + +def _bench(fn, text: str) -> float: + iters = 20000 if len(text) < 300 else (3000 if len(text) < 6000 else 300) + for _ in range(50): + fn(text) + t0 = time.perf_counter() + for _ in range(iters): + fn(text) + return (time.perf_counter() - t0) / iters * 1e6 # us/call + + +def main() -> None: + data = _AbbreviationData(LANGUAGE_CODES["en"].Abbreviation) + automaton = data.automaton + # The automaton is keyed on "." (the trailing-period pre-filter), except + # 'i'-ending abbreviations which keep the bare key (U+0130 'İ' caveat). Mirror + # that here so the naive loop matches the automaton's match set. + abbr_keys = [a[1] if a[1].endswith("i") else a[1] + "." for a in data.abbreviations] + + def ac_scan(text: str) -> set[int]: + return automaton.search(text) + + def in_loop(text: str) -> set[int]: + return {i for i, a in enumerate(abbr_keys) if a in text} + + print(f"abbreviation discovery: Aho-Corasick vs `in` loop ({len(abbr_keys)} patterns)") + print("=" * 60) + print(f"{'input':<16}{'AC us':>10}{'in us':>10}{'winner':>10}{'ratio':>8}") + print("-" * 60) + for name, text in CASES.items(): + assert ac_scan(text) == in_loop(text), f"match-set mismatch for {name}" + ac = _bench(ac_scan, text) + inf = _bench(in_loop, text) + winner = "AC" if ac < inf else "in" + print(f"{name:<16}{ac:>10.2f}{inf:>10.2f}{winner:>10}{inf / ac:>8.2f}") + print("-" * 60) + print("ratio = in/AC (>1 means AC faster). Match-sets verified identical.") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/differential_profile.py b/benchmarks/differential_profile.py new file mode 100644 index 0000000..656ab65 --- /dev/null +++ b/benchmarks/differential_profile.py @@ -0,0 +1,132 @@ +"""Differential profiler: sentencesplit vs pySBD on identical input. + +Answers "what *exactly* makes us slower than pySBD per call" by profiling both +on the same text and surfacing the operation-level deltas that matter: + + * per-call wall time (ours vs pySBD), + * count + time of every regex op (sub / findall / search / finditer / split) + per call — the clearest "we run more passes" signal, + * total time spent inside the `re` module, + * the top tottime functions for each library, side by side. + +Run with: + uv run python benchmarks/differential_profile.py --size short + uv run python benchmarks/differential_profile.py --size medium --top 18 +""" + +from __future__ import annotations + +import argparse +import cProfile +import pstats +import time +from collections import Counter + +import pysbd + +import sentencesplit + +SHORT = "Dr. Smith went to Washington. He arrived on Jan. 5th at 3 p.m. and met with Sen. Jones." +MEDIUM = ( + "Dr. Smith went to Washington. He arrived on Jan. 5th at 3 p.m. " + "The model is GPT 3.1 and it is fast. That is all for now. Goodbye. " + "She paid $4.50 for the U.S. edition (vol. 2, p. 17). Mr. Lee agreed." +) +LARGE = " ".join([MEDIUM] * 20) +_SAMPLES = {"short": SHORT, "medium": MEDIUM, "large": LARGE} + +_REGEX_OPS = ("sub", "findall", "search", "match", "finditer", "split", "fullmatch") + + +def _walltime(fn, text: str, iters: int) -> float: + for _ in range(20): + fn(text) + t0 = time.perf_counter() + for _ in range(iters): + fn(text) + return (time.perf_counter() - t0) / iters * 1e6 # us/call + + +def _profile(fn, text: str, iters: int) -> pstats.Stats: + pr = cProfile.Profile() + for _ in range(5): + fn(text) + pr.enable() + for _ in range(iters): + fn(text) + pr.disable() + return pstats.Stats(pr) + + +def _regex_op_summary(stats: pstats.Stats, iters: int) -> tuple[Counter, float]: + """Per-call regex op counts and time spent in the regex engine. + + Counts ONLY the compiled-Pattern method calls (e.g. ````), not the module-level ``re.sub``/``re.findall`` + wrapper frames that delegate to them. Counting both double-counts an + uncompiled ``re.sub(str, ...)`` — which would bias a library that does not + pre-compile (it routes through the wrapper) against one that does. + """ + counts: Counter = Counter() + re_time = 0.0 + for (_filename, _lineno, funcname), (_cc, nc, tt, _ct, _cb) in stats.stats.items(): + if funcname.startswith(" list[tuple[str, float, int]]: + rows = [] + for (filename, lineno, funcname), (_cc, nc, tt, _ct, _cb) in stats.stats.items(): + short = filename.replace("\\", "/").split("/")[-1] + rows.append((f"{short}:{lineno}:{funcname}", tt / iters * 1e6, nc // iters)) + rows.sort(key=lambda r: r[1], reverse=True) + return rows[:n] + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--size", choices=["short", "medium", "large"], default="short") + ap.add_argument("--top", type=int, default=14) + args = ap.parse_args() + + text = _SAMPLES[args.size] + iters = 8000 if args.size != "large" else 1000 + + ours = sentencesplit.Segmenter(language="en", clean=False, char_span=False) + sbd = pysbd.Segmenter(language="en", clean=False) + engines = {"sentencesplit": ours.segment, "pysbd": sbd.segment} + + print(f"differential profile size={args.size} ({len(text)} chars) iters={iters}") + print("=" * 72) + walls = {name: _walltime(fn, text, iters) for name, fn in engines.items()} + print( + f"wall time: ours {walls['sentencesplit']:.2f} us/call " + f"pysbd {walls['pysbd']:.2f} us/call " + f"ours is {walls['sentencesplit'] / walls['pysbd']:.2f}x pysbd" + ) + print() + + for name, fn in engines.items(): + stats = _profile(fn, text, iters) + ops, re_us = _regex_op_summary(stats, iters) + total_ops = sum(ops.values()) + print(f"--- {name} ---") + print( + f" regex ops/call: {total_ops:.1f} " + f"({' '.join(f'{k}={v:.1f}' for k, v in sorted(ops.items(), key=lambda x: -x[1]))})" + ) + print(f" time in re/call: {re_us:.1f} us") + print(f" top {args.top} by tottime (us/call, calls/call):") + for label, us, calls in _top(stats, args.top, iters): + print(f" {us:7.2f} x{calls:<4} {label}") + print() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/phase_profile.py b/benchmarks/phase_profile.py new file mode 100644 index 0000000..42d80b6 --- /dev/null +++ b/benchmarks/phase_profile.py @@ -0,0 +1,127 @@ +"""Phase-level latency profiler for the segmentation pipeline. + +Where ``latency_baseline.py`` answers "how fast is a call?", this answers "which +*phase* of the call is the time in?" — it wraps each Processor / AbbreviationReplacer +/ Segmenter stage with a timer and reports per-phase wall time, call count, and +share of the total, for short / medium / large input. + +This is the exploration harness for the short/medium-vs-pySBD latency gap: the +fixed per-call cost (the phases that run on every call regardless of input) is +what dominates short strings, so this attributes that cost to concrete phases. + +Run with: + uv run python benchmarks/phase_profile.py + uv run python benchmarks/phase_profile.py --size short --iters 20000 +""" + +from __future__ import annotations + +import argparse +import time +from collections import defaultdict + +from sentencesplit import Segmenter +from sentencesplit.abbreviation_replacer import AbbreviationReplacer +from sentencesplit.processor import Processor +from sentencesplit.segmenter import Segmenter as _Seg + +SHORT = "Dr. Smith went to Washington. He arrived on Jan. 5th at 3 p.m. and met with Sen. Jones." +MEDIUM = ( + "Dr. Smith went to Washington. He arrived on Jan. 5th at 3 p.m. " + "The model is GPT 3.1 and it is fast. That is all for now. Goodbye. " + "She paid $4.50 for the U.S. edition (vol. 2, p. 17). Mr. Lee agreed." +) +LARGE = " ".join([MEDIUM] * 20) +_SAMPLES = {"short": SHORT, "medium": MEDIUM, "large": LARGE} + +# (class, method) pairs to attribute. Grouped by pipeline stage; the label is what +# the report prints. Only methods that exist are wrapped. +_TARGETS = [ + # --- text-processing phases (run once per call over the whole text) --- + (Processor, "_normalize_newlines", "text: normalize_newlines"), + (Processor, "_mark_list_item_boundaries", "text: list_item_boundaries"), + (Processor, "replace_abbreviations", "text: replace_abbreviations"), + (Processor, "replace_numbers", "text: replace_numbers"), + (Processor, "replace_continuous_punctuation", "text: continuous_punct"), + (Processor, "replace_periods_before_numeric_references", "text: numeric_refs"), + (Processor, "_protect_special_tokens", "text: special_tokens"), + # --- boundary-processing phases (per segment) --- + (Processor, "_ensure_terminal_marker", "bound: terminal_marker"), + (Processor, "_apply_exclamation_word_rules", "bound: exclamation_words"), + (Processor, "between_punctuation", "bound: between_punctuation"), + (Processor, "_apply_double_punctuation_rules", "bound: double_punct"), + (Processor, "_apply_quotation_punctuation_rules", "bound: quotation_punct"), + (Processor, "_replace_list_parens", "bound: list_parens"), + (Processor, "sentence_boundary_punctuation", "bound: sentence_boundary"), + # --- post-split passes --- + (Processor, "split_into_segments", "post: split_into_segments (incl. boundary)"), + (Processor, "_resplit_segments", "post: resplit_segments"), + (Processor, "_merge_orphan_fragments", "post: merge_orphans"), + # --- abbreviation internals (the suspected fixed cost) --- + (AbbreviationReplacer, "replace", "abbr: replace (whole)"), + (AbbreviationReplacer, "search_for_abbreviations_in_string", "abbr: search_in_string"), + (AbbreviationReplacer, "apply_ampm_boundary_rules", "abbr: ampm_rules"), + # --- span mapping (segmenter) --- + (_Seg, "_match_spans", "span: match_spans"), +] + +_stats: dict[str, list[float | int]] = defaultdict(lambda: [0.0, 0]) # label -> [total_s, calls] + + +def _wrap(cls, method_name: str, label: str) -> None: + original = getattr(cls, method_name, None) + if original is None: + return + + def timed(self, *args, _orig=original, _label=label, **kwargs): + t0 = time.perf_counter() + try: + return _orig(self, *args, **kwargs) + finally: + rec = _stats[_label] + rec[0] += time.perf_counter() - t0 + rec[1] += 1 + + setattr(cls, method_name, timed) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--size", choices=["short", "medium", "large"], default="short") + ap.add_argument("--iters", type=int, default=20000) + args = ap.parse_args() + + for cls, name, label in _TARGETS: + _wrap(cls, name, label) + + seg = Segmenter(language="en", clean=False, char_span=False) + text = _SAMPLES[args.size] + for _ in range(5): + seg.segment(text) + _stats.clear() + + t0 = time.perf_counter() + for _ in range(args.iters): + seg.segment(text) + total = time.perf_counter() - t0 + + print(f"phase profile size={args.size} iters={args.iters} ({len(text)} chars)") + print(f"total: {total * 1000 / args.iters:.4f} ms/call ({total:.2f}s)") + print("=" * 78) + print(f"{'phase':<46}{'ms/call':>10}{'calls/seg':>10}{'% total':>10}") + print("-" * 78) + rows = sorted(_stats.items(), key=lambda kv: kv[1][0], reverse=True) + for label, (total_s, calls) in rows: + ms_per_call = total_s * 1000 / args.iters + calls_per_seg = calls / args.iters + pct = 100 * total_s / total + # split_into_segments wraps the boundary phases, so its % overlaps them; + # mark it so the breakdown is not misread as additive. + note = " *wrapper" if "split_into_segments" in label or label == "abbr: replace (whole)" else "" + print(f"{label:<46}{ms_per_call:>10.4f}{calls_per_seg:>10.1f}{pct:>9.1f}%{note}") + print("-" * 78) + print("* wrapper rows contain the rows below them; do not sum across them.") + + +if __name__ == "__main__": + main() diff --git a/sentencesplit/abbreviation_replacer.py b/sentencesplit/abbreviation_replacer.py index c23e6a3..2f850f7 100644 --- a/sentencesplit/abbreviation_replacer.py +++ b/sentencesplit/abbreviation_replacer.py @@ -15,15 +15,29 @@ class AhoCorasickAutomaton: - """Pure-Python Aho-Corasick automaton for multi-pattern substring search.""" + """Pure-Python Aho-Corasick automaton for multi-pattern substring search. - __slots__ = ("goto", "fail", "output", "_built") + Thread-safety: an instance is mutated only by ``add_pattern``/``build`` and is + read-only thereafter. It carries no lock of its own — safe concurrent use + relies on the owner publishing it only after ``build()`` completes. In this + package the only instances live inside ``_AbbreviationData``, which is built + and then stored into ``AbbreviationReplacer._data_cache`` under + ``_cache_lock``, so every reader's ``search()`` happens-after ``build()``. + """ + + __slots__ = ("goto", "fail", "output", "delta", "_built") def __init__(self): # State 0 is the root. Each state maps char -> next_state. self.goto: list[dict[str, int]] = [{}] self.output: list[list[int]] = [[]] # pattern IDs at each state self.fail: list[int] = [0] + # Fail-link-collapsed transition table, built once in build(). Each + # delta[state] maps an observed-alphabet char -> next state with the fail + # walk already resolved, so search() is one dict.get per char (no inner + # loop). Chars outside the alphabet are absent and .get(ch, 0) sends them + # to the root, exactly as the fail walk would. + self.delta: list[dict[str, int]] = [] self._built = False def add_pattern(self, pattern: str, pattern_id: int) -> None: @@ -58,21 +72,38 @@ def build(self) -> None: self.fail[s] = 0 if self.output[self.fail[s]]: self.output[s] = self.output[s] + self.output[self.fail[s]] + + # Collapse the fail links into a DFA transition table. For each state and + # each observed-alphabet char: take the goto if present, else inherit the + # fail state's already-resolved transition. fail[r] is strictly shallower + # than r, so a goto-tree BFS visits it first and delta[fail[r]] is ready. + alphabet: set[str] = set() + for trans in self.goto: + alphabet.update(trans) + delta: list[dict[str, int]] = [{} for _ in self.goto] + root_goto = self.goto[0] + delta[0] = {ch: root_goto.get(ch, 0) for ch in alphabet} + queue = deque(self.goto[0].values()) + while queue: + r = queue.popleft() + gr = self.goto[r] + dfail = delta[self.fail[r]] + delta[r] = {ch: (gr[ch] if ch in gr else dfail[ch]) for ch in alphabet} + queue.extend(gr.values()) + self.delta = delta self._built = True def search(self, text: str) -> set[int]: """Scan text in one pass, return set of matched pattern IDs.""" state = 0 found: set[int] = set() - goto = self.goto - fail = self.fail + delta = self.delta output = self.output for ch in text: - while state != 0 and ch not in goto[state]: - state = fail[state] - state = goto[state].get(ch, 0) - if output[state]: - found.update(output[state]) + state = delta[state].get(ch, 0) + out = output[state] + if out: + found.update(out) return found @@ -141,7 +172,23 @@ def __init__(self, lang_abbreviation_class): next_word_re, ) ) - self.automaton.add_pattern(stripped_lower, idx) + # Add the trailing period to the automaton key. search_for_abbreviations + # only ever acts on an abbreviation when it occurs at a word boundary + # *followed by a period*; any such occurrence contains the substring + # ".", so keying on "." is a byte-identical pre-filter that + # skips the per-abbreviation full-text finditer for abbreviations whose + # bare form merely appears inside other words (e.g. "al" in "called", + # "no" in "no one") with no following period — the dominant cost on + # real prose, where common short abbreviations match everywhere. + # + # Exception: the automaton is searched on ``text.lower()`` and U+0130 + # 'İ' is the only Unicode char whose .lower() changes length ('İ' -> + # 'i' + U+0307 combining dot). An occurrence ending in 'İ' followed by + # a period lowers to '...i̇.', so the "." key (e.g. "vi.") would + # not match. Abbreviations ending in 'i' therefore keep the bare key + # (the original, always-correct behavior). + key = stripped_lower if stripped_lower.endswith("i") else stripped_lower + "." + self.automaton.add_pattern(key, idx) self.automaton.build() self.abbr_set = frozenset(a.strip().lower() for a in raw) self.prepositive_set = frozenset(a.lower() for a in lang_abbreviation_class.PREPOSITIVE_ABBREVIATIONS) diff --git a/sentencesplit/lang/common/standard.py b/sentencesplit/lang/common/standard.py index 1e5b73a..06ffd7c 100644 --- a/sentencesplit/lang/common/standard.py +++ b/sentencesplit/lang/common/standard.py @@ -4,11 +4,21 @@ from sentencesplit.abbreviation_replacer import AbbreviationReplacer from sentencesplit.utils import Rule +# A modification requires a run of 4+ dots immediately followed by an ASCII +# lowercase letter, which always contains the substring "4 dots then lowercase". +# So this is a necessary condition; when it fails the per-char loop below makes +# no change and is skipped. Uses a FIXED ``{4}`` count (not ``{4,}``) so the C +# search stays linear on adversarial long period runs — ``{4,}`` would backtrack +# O(n^2) (see test_glued_ellipsis_..._handles_long_period_runs_linearly). +_GLUED_RUNON_GUARD_RE = re.compile(r"\.{4}[a-z]") + class _GluedLowercaseRunOnRegex: """Linear-time replacer for glued 4+ dot lowercase run-ons.""" def sub(self, replacement: str, text: str) -> str: + if not _GLUED_RUNON_GUARD_RE.search(text): + return text chars: list[str] | None = None index = 0 length = len(text) diff --git a/sentencesplit/languages.py b/sentencesplit/languages.py index 8dd8cfe..fe59ce5 100644 --- a/sentencesplit/languages.py +++ b/sentencesplit/languages.py @@ -206,8 +206,20 @@ def __ror__(self, other): def _evict_profile(code: str) -> None: - """Drop any cached LanguageProfile for the class currently bound to ``code`` - so a re-registration (or override) is rebuilt fresh.""" + """Drop the cached state for the class currently bound to ``code`` so a + re-registration (or override) is rebuilt fresh. + + This drops both the cached :class:`LanguageProfile` (keyed on the language + class) and the per-``Abbreviation``-class Aho-Corasick data (keyed on + ``language_cls.Abbreviation``); otherwise a re-registered class whose + abbreviation list changed would keep a stale automaton. + + Lock ordering (load-bearing): the two cache locks are acquired *sequentially* + (never co-held) while the caller holds ``_LANGUAGE_LOCK``. No reader path ever + acquires ``_LANGUAGE_LOCK`` while holding ``_PROFILE_CACHE_LOCK`` or + ``_cache_lock``, so there is no lock-ordering cycle. Preserve that invariant. + """ + from sentencesplit.abbreviation_replacer import AbbreviationReplacer from sentencesplit.language_profile import _PROFILE_CACHE, _PROFILE_CACHE_LOCK with _LANGUAGE_LOCK: @@ -215,6 +227,10 @@ def _evict_profile(code: str) -> None: if existing is not None: with _PROFILE_CACHE_LOCK: _PROFILE_CACHE.pop(existing, None) + abbr_class = getattr(existing, "Abbreviation", None) + if abbr_class is not None: + with AbbreviationReplacer._cache_lock: + AbbreviationReplacer._data_cache.pop(abbr_class, None) def register_language(code: str, language_cls: type) -> None: diff --git a/sentencesplit/lists_item_replacer.py b/sentencesplit/lists_item_replacer.py index 2547a05..e9650f5 100644 --- a/sentencesplit/lists_item_replacer.py +++ b/sentencesplit/lists_item_replacer.py @@ -11,6 +11,9 @@ # Newline-separated list-marker guards: a list spanning lines is not collapsed. _MULTILINE_BULLET_GUARD_RE = re.compile(r"♨.+(\n|\r).+♨") _MULTILINE_PAREN_MARKER_GUARD_RE = re.compile(r"☝.+\n.+☝|☝.+\r.+☝") +# Every numbered-list pattern requires a digit; used to skip the scan on +# digit-free text (matches re's \d Unicode-digit semantics exactly). +_DIGIT_RE = re.compile(r"\d") class ListItemReplacer: @@ -107,6 +110,11 @@ def add_line_breaks_for_alphabetical_list_with_parens(self, roman_numeral=False) self.iterate_alphabet_array(self.ALPHABETICAL_LIST_WITH_PARENS, parens=True, roman_numeral=roman_numeral) def scan_lists(self, regex1, regex2, replacement, strip=False): + # All numbered-list patterns require a digit (and the body does int()), + # so digit-free text can't match — skip the two finditer scans. The loop + # below never runs on empty matches, so this is byte-identical. + if not _DIGIT_RE.search(self.text): + return matches = list(re.finditer(regex1, self.text)) list_array = [(int(m.group().strip()), m.start()) for m in matches] for ind, (item, pos) in enumerate(list_array): @@ -268,6 +276,11 @@ def other_items_replacement(self, a, i, alphabet, alphabet_index, list_array, pa def iterate_alphabet_array(self, regex, parens=False, roman_numeral=False): list_array = re.findall(regex, self.text, re.IGNORECASE) + # Common case on list-free text: no markers found. Skip the lowercasing, + # the per-call alphabet-index dict build, and the filter — with an empty + # list the replacement loop below never runs, so this is byte-identical. + if not list_array: + return list_array = [i.lower() for i in list_array] alphabet = self.ROMAN_NUMERALS if roman_numeral else self.LATIN_NUMERALS alphabet_index = {value: index for index, value in enumerate(alphabet)} diff --git a/sentencesplit/processor.py b/sentencesplit/processor.py index 525856e..eed5c68 100644 --- a/sentencesplit/processor.py +++ b/sentencesplit/processor.py @@ -52,6 +52,10 @@ # lookahead excludes a doubled comma (",,"), which in Dutch typography is an # *opening* quotation mark beginning a new sentence (e.g. "...einde. ,,Nieuwe..."). _PERIOD_BEFORE_COMMA_RE = re.compile(r"\.(?=\s*,(?!,))") +# The five ReinsertEllipsisRules each require one of these placeholder sentinels; +# a segment with none of them passes through them unchanged, so the per-segment +# pass can be skipped on the common case (one C scan vs five no-op subs). +_REINSERT_ELLIPSIS_RE = re.compile(r"[ƪ♟♝☏∮]") # The between-punctuation pass protects everything from an opening quote to its # closing quote as one unsplittable region, so a quotation that wraps several @@ -642,7 +646,8 @@ def post_process_segments(self, txt: str) -> list[str]: if len(txt) > 2 and _ALPHA_ONLY_RE.search(txt): return [txt] - txt = apply_rules(txt, *self.lang.ReinsertEllipsisRules.All) + if _REINSERT_ELLIPSIS_RE.search(txt): + txt = apply_rules(txt, *self.lang.ReinsertEllipsisRules.All) if self.profile.latin_uppercase_resplit: quoted_parts = _split_on_uppercase_boundary(txt, self.profile.split_quotation_re) if quoted_parts is not None: diff --git a/sentencesplit/segmenter.py b/sentencesplit/segmenter.py index 0e145f8..bb9138c 100644 --- a/sentencesplit/segmenter.py +++ b/sentencesplit/segmenter.py @@ -399,17 +399,20 @@ def _find_sentence_start(self, sent: str, original_text: str, prior_end: int): return self._find_sentence_start_tolerant(sent, original_text, prior_end) # Some post-processing rules may normalize spaces around punctuation, - # so allow flexible whitespace when mapping back to original text. - whitespace_flexible = re.escape(sent).replace(r"\ ", r"\s*") - match = re.search(whitespace_flexible, original_text[prior_end:]) + # so allow flexible whitespace when mapping back to original text. Search + # from ``prior_end`` via the pattern's ``pos`` arg rather than slicing + # ``original_text[prior_end:]`` (which copies the whole remaining text on + # every sentence — O(n^2) over a document). The flexible patterns are + # anchor-free (escaped literal + ``\s*``/zero-width joiners), so matching + # at ``pos`` is identical to matching the slice. + whitespace_flexible = re.compile(re.escape(sent).replace(r"\ ", r"\s*")) + match = whitespace_flexible.search(original_text, prior_end) if match is None: - match = re.search(self._zero_width_flexible_pattern(sent), original_text[prior_end:]) + match = re.compile(self._zero_width_flexible_pattern(sent)).search(original_text, prior_end) if match is None: return None - start_idx = prior_end + match.start() - end_idx = prior_end + match.end() - return start_idx, end_idx + return match.start(), match.end() @staticmethod def _match_tolerant_at(sent: str, original_text: str, start: int): diff --git a/tests/regression/test_abbreviation_dotted_i.py b/tests/regression/test_abbreviation_dotted_i.py new file mode 100644 index 0000000..343f733 --- /dev/null +++ b/tests/regression/test_abbreviation_dotted_i.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +"""Regression: the abbreviation period-prefilter must handle U+0130 'İ'. + +The automaton is keyed on '.' as a pre-filter, but it is searched on +``text.lower()``. U+0130 'İ' (LATIN CAPITAL LETTER I WITH DOT ABOVE) is the only +Unicode character whose ``.lower()`` changes length — it expands to 'i' + a +combining dot above (U+0307). So an abbreviation occurrence ending in 'İ' and +followed by a period becomes '...i̇.' when lowered, and the '.' key (e.g. +'vi.') no longer matches: the abbreviation is missed and the period over-splits. +Abbreviations ending in 'i' therefore keep the bare key (the original behavior). +""" + +import unicodedata + +from sentencesplit import Segmenter + + +def test_dotted_capital_i_does_not_break_abbreviation_prefilter(): + # 'vi' is a German NUMBER_ABBREVIATION (Roman numeral). The 'İ' spelling must + # segment identically to the plain-ASCII 'vi' control: one joined sentence. + ascii_control = Segmenter(language="de").segment("Band vi. Der Rest folgt.") + dotted_i = Segmenter(language="de").segment("Band vİ. Der Rest folgt.") + assert len(dotted_i) == len(ascii_control) == 1, dotted_i + assert dotted_i == ["Band vİ. Der Rest folgt."], dotted_i + + +def test_dotted_capital_i_is_still_the_only_length_changing_lowercase(): + # The fix assumes 'İ' is the sole char whose .lower() is multi-char (so only + # 'i'-ending abbreviations are affected). If a future Unicode update adds + # another, this fails loudly so the prefilter logic can be revisited. + expanding = [chr(c) for c in range(0x110000) if len(chr(c).lower()) != 1] + assert expanding == ["İ"], expanding + assert "İ".lower() == "i" + unicodedata.lookup("COMBINING DOT ABOVE") diff --git a/tests/regression/test_language_reregistration.py b/tests/regression/test_language_reregistration.py new file mode 100644 index 0000000..f0561e1 --- /dev/null +++ b/tests/regression/test_language_reregistration.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +"""Regression: re-registering a language must rebuild its abbreviation data. + +``_evict_profile`` is documented to make a re-registration "rebuilt fresh", but +it only dropped the cached ``LanguageProfile`` — the per-``Abbreviation``-class +Aho-Corasick data in ``AbbreviationReplacer._data_cache`` survived, so a class +re-registered after its abbreviation list changed kept the stale automaton. +""" + +from sentencesplit import Segmenter +from sentencesplit.abbreviation_replacer import AbbreviationReplacer +from sentencesplit.lang.english import English +from sentencesplit.languages import register_language, unregister_language + + +def test_reregistration_rebuilds_abbreviation_data(): + class CustomAbbr(English.Abbreviation): + ABBREVIATIONS = list(English.Abbreviation.ABBREVIATIONS) + + class CustomEn(English): + Abbreviation = CustomAbbr + + code = "zz" + try: + register_language(code, CustomEn) + # Build and cache the abbreviation automaton for CustomAbbr. + Segmenter(language=code).segment("Dr. Smith arrived.") + assert "zorp" not in AbbreviationReplacer._data_cache[CustomAbbr].abbr_set + + # Change the abbreviation set and re-register the (same) class. + CustomAbbr.ABBREVIATIONS.append("zorp") + register_language(code, CustomEn) + + # The next use must rebuild the automaton fresh from the new list. + Segmenter(language=code).segment("Zorp. Smith arrived.") + assert "zorp" in AbbreviationReplacer._data_cache[CustomAbbr].abbr_set + finally: + unregister_language(code)