From 553a36ee4441be2af427a11a67ec08db91177095 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 01:20:35 +0000 Subject: [PATCH] ci: add competitive CodSpeed benchmarks vs pySBD and punkt Add benchmarks/test_competitive_codspeed.py comparing sentencesplit's per-call latency and a throughput-proxy against pySBD (the rule-based library it derives from) and nltk's punkt. All three are pure Python, so CodSpeed's instruction-count measurement compares them fairly; C/Cython tokenizers are intentionally excluded. Each engine is constructed once in a fixture so only steady-state segmentation is timed. Adds pysbd + nltk to the isolated bench group and a punkt-model download step (run outside CodSpeed instrumentation), and runs the new file alongside the existing latency benchmarks. --- .github/workflows/codspeed.yml | 15 ++++- benchmarks/test_competitive_codspeed.py | 84 +++++++++++++++++++++++++ pyproject.toml | 4 ++ uv.lock | 12 ++++ 4 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 benchmarks/test_competitive_codspeed.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index c274db6..84aa156 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -37,10 +37,15 @@ jobs: run: uv python install 3.13 - name: Install benchmark dependencies - # Only the project + the isolated `bench` group (pytest + pytest-codspeed), - # not the full dev matrix. + # Only the project + the isolated `bench` group (pytest, pytest-codspeed, + # and the competitive peers pysbd + nltk), not the full dev matrix. run: uv sync --python 3.13 --no-default-groups --group bench + - name: Download punkt model + # nltk's sent_tokenize needs the punkt data. Fetch it here (outside the + # CodSpeed run) so no network/model-load work happens under instrumentation. + run: uv run --no-sync --python 3.13 python -c "import nltk; nltk.download('punkt_tab', quiet=True); nltk.download('punkt', quiet=True)" + - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@v3 with: @@ -48,4 +53,8 @@ jobs: # without an implicit re-sync, which would otherwise pull the default # `dev` group back in (defeating isolation) and run install work under # CodSpeed instrumentation. - run: uv run --no-sync --python 3.13 pytest benchmarks/test_latency_codspeed.py --codspeed + run: >- + uv run --no-sync --python 3.13 pytest + benchmarks/test_latency_codspeed.py + benchmarks/test_competitive_codspeed.py + --codspeed diff --git a/benchmarks/test_competitive_codspeed.py b/benchmarks/test_competitive_codspeed.py new file mode 100644 index 0000000..1e1e311 --- /dev/null +++ b/benchmarks/test_competitive_codspeed.py @@ -0,0 +1,84 @@ +"""Competitive CodSpeed benchmarks: sentencesplit vs pySBD vs punkt. + +Tracks sentencesplit's latency/throughput against the two pure-Python sentence +boundary detectors it is most often compared to: + + * **pySBD** — the rule-based library sentencesplit was derived from. + * **punkt** — nltk's classic unsupervised statistical tokenizer. + +Why only these two: CodSpeed's instrumentation mode measures CPU *instructions*, +which is a fair proxy for wall-clock time only among libraries of the same +execution character. All three here are pure Python, so the comparison is +apples-to-apples. C/Cython tokenizers (spaCy, blingfire, stanza) would have a +very different instruction-to-wallclock ratio and are intentionally excluded — +compare against those with a wall-clock harness (see the ``compare-segmenters`` +skill / ``benchmarks/benchmark_sbd_tools.py``) instead. + +Reading the results: CodSpeed compares each benchmark against itself across +commits, not library-to-library. To compare the three engines, read the +absolute per-benchmark numbers side by side in the CodSpeed dashboard — the +``[size-ours]`` / ``[size-pysbd]`` / ``[size-punkt]`` rows for one size are +directly comparable (identical input, each engine constructed once outside the +timed call). The per-PR delta still guards each engine against regression. + +Run locally (smoke test, no measurement):: + + uv run --no-sync pytest benchmarks/test_competitive_codspeed.py +""" + +from __future__ import annotations + +import pytest + +SHORT = "Dr. Smith went to Washington. He arrived on Jan. 5th at 3 p.m. and met with Sen. Jones." +MEDIUM = ( + "Dr. Smith went to Washington. He arrived on Jan. 5th at 3 p.m. " + "The model is GPT 3.1 and it is fast. That is all for now. Goodbye. " + "She paid $4.50 for the U.S. edition (vol. 2, p. 17). Mr. Lee agreed." +) +LARGE = " ".join([MEDIUM] * 20) +# A larger document (~10 KB) used as a throughput proxy: per-run cost is inversely +# proportional to sentences/sec, so the relative costs rank the engines' throughput. +THROUGHPUT_DOC = " ".join([MEDIUM] * 50) + +_SAMPLES = {"short": SHORT, "medium": MEDIUM, "large": LARGE} +_LIBRARIES = ["ours", "pysbd", "punkt"] + + +@pytest.fixture(scope="module") +def segmenters() -> dict[str, object]: + """Each engine constructed once; only the per-call segment is benchmarked. + + Construction cost (sentencesplit's automaton, pySBD's compiled rules, punkt's + pickled model) is amortized exactly as it is in real reuse, so the timed call + is steady-state segmentation only. + """ + import nltk + import pysbd + + import sentencesplit + + ours = sentencesplit.Segmenter(language="en", clean=False, char_span=False) + sbd = pysbd.Segmenter(language="en", clean=False) + # Warm punkt so its one-time model load is not measured (nltk caches the + # loaded tokenizer, so subsequent calls reuse it). + nltk.sent_tokenize("Warm up the punkt model. It is ready now.") + + return { + "ours": ours.segment, + "pysbd": sbd.segment, + "punkt": nltk.sent_tokenize, + } + + +@pytest.mark.parametrize("library", _LIBRARIES) +@pytest.mark.parametrize("size", ["short", "medium", "large"]) +def test_segment(benchmark, segmenters: dict[str, object], size: str, library: str) -> None: + segment = segmenters[library] + benchmark(segment, _SAMPLES[size]) + + +@pytest.mark.parametrize("library", _LIBRARIES) +def test_throughput(benchmark, segmenters: dict[str, object], library: str) -> None: + segment = segmenters[library] + benchmark(segment, THROUGHPUT_DOC) diff --git a/pyproject.toml b/pyproject.toml index a64ece8..1e9e6bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,10 @@ dev = [ bench = [ "pytest>=8.0", "pytest-codspeed>=2.2", + # Competitive comparison peers (both pure Python, so instruction-count + # comparisons against them are fair). See benchmarks/test_competitive_codspeed.py. + "pysbd>=0.3.4", + "nltk>=3.9.2", ] [tool.uv.build-backend] diff --git a/uv.lock b/uv.lock index 9bdca99..ff77c7b 100644 --- a/uv.lock +++ b/uv.lock @@ -1427,6 +1427,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pysbd" +version = "0.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/0a/c99fb7d7e176f8b176ef19704a32e6a9c6aafdf19ef75a187f701fc15801/pysbd-0.3.4-py3-none-any.whl", hash = "sha256:cd838939b7b0b185fcf86b0baf6636667dfb6e474743beeff878e9f42e022953", size = 71082, upload-time = "2021-02-11T16:36:33.351Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -1787,6 +1795,8 @@ spacy = [ [package.dev-dependencies] bench = [ + { name = "nltk" }, + { name = "pysbd" }, { name = "pytest" }, { name = "pytest-codspeed" }, ] @@ -1814,6 +1824,8 @@ provides-extras = ["spacy", "benchmark", "release"] [package.metadata.requires-dev] bench = [ + { name = "nltk", specifier = ">=3.9.2" }, + { name = "pysbd", specifier = ">=0.3.4" }, { name = "pytest", specifier = ">=8.0" }, { name = "pytest-codspeed", specifier = ">=2.2" }, ]