From d4356a377fe6fa30bf26d9c86ce612895a076b40 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Fri, 28 Aug 2026 21:15:37 -0400 Subject: [PATCH] fix(baseline): widen pattern file lists to accept README.rst / .txt / no-ext Follow-up to #402 (which was TOML-only inside `llm_eval` blocks). Widens the sibling `pattern` handler's `files` list on the same 5 controls that gate an `llm_eval` pass, so the deterministic tier resolves PASS/FAIL conclusively on more repos rather than falling through to the LLM tier. The bug pattern before this change: a Sphinx-based Python project that ships README.rst but no README.md would have `pattern.files` scan zero on-disk candidates, return INCONCLUSIVE via `_regex_no_files_result`, and the pipeline would fall through to `llm_eval`. This was the exact trigger for the empty-file_contents consultation surveyed across 29 repos (4 on tqdm, 5 on flask when its README.md was renamed to README.rst). Controls widened: - OSPS-DO-03.02 (VerifyReleaseAuthor): + README.rst / README.txt / README - OSPS-DO-04.01 (HasSupportDocs): + README.rst / README.txt / README + docs/support.md - OSPS-DO-05.01 (SupportEndOfLife): + README.rst / README.txt / README - OSPS-SA-01.01 (ArchitectureDocs): + rst variants for ARCHITECTURE/DESIGN/README/docs - OSPS-SA-02.01 (ApiUsageDocs): + rst variants for API/README/USAGE + docs/api.rst `_regex_match_files` reads each listed file in full; the additions stay well under the handler's `_FILE_SCAN_LIMIT = 100` cap and the per-file 2000-byte truncation. Regression guard: `tests/darnit_baseline/controls/test_pattern_file_variants.py` - Parametrized across the 5 gated controls; asserts README.rst is present in each pattern list. - Also asserts README.txt / README (no ext) on the DO-* controls where they were added. Not in scope: framework code changes to the pattern handler's file resolution or generalizing to auto-derive .rst/.txt variants from .md entries. Both are separate. --- .../src/darnit_baseline/openssf-baseline.toml | 19 +++- .../controls/test_pattern_file_variants.py | 95 +++++++++++++++++++ 2 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 tests/darnit_baseline/controls/test_pattern_file_variants.py diff --git a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml index 3ba794f..63e4f28 100644 --- a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml +++ b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml @@ -3849,7 +3849,10 @@ help_md = """Document how to verify release author. [[controls."OSPS-DO-03.02".passes]] handler = "pattern" -files = ["SECURITY.md", "README.md", "docs/releases.md", "CONTRIBUTING.md", "docs/RELEASE-VERIFICATION.md"] +# rst/txt/no-ext README variants added so this pass can PASS/FAIL +# deterministically on projects that don't ship .md-based READMEs +# (Python/Sphinx projects like tqdm surfaced the gap; see issue #402). +files = ["SECURITY.md", "README.md", "README.rst", "README.txt", "README", "docs/releases.md", "CONTRIBUTING.md", "docs/RELEASE-VERIFICATION.md"] [controls."OSPS-DO-03.02".passes.pattern.patterns] verify_author = "(verify|verif|signature|sign|GPG|Sigstore).*(author|release|identity)" @@ -3905,7 +3908,8 @@ help_md = """Document support policy. [[controls."OSPS-DO-04.01".passes]] handler = "pattern" -files = ["SUPPORT.md", "SECURITY.md", "README.md"] +# rst/txt/no-ext README variants added (see #402 sibling change). +files = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "README.txt", "README", "docs/support.md"] [controls."OSPS-DO-04.01".passes.pattern.patterns] support_scope = "(support.*(scope|duration|period|timeline)|version.*(support|maintenance))" @@ -3968,7 +3972,8 @@ help_md = """Document end-of-life policy. [[controls."OSPS-DO-05.01".passes]] handler = "pattern" -files = ["SUPPORT.md", "SECURITY.md", "README.md"] +# rst/txt/no-ext README variants added (see #402 sibling change). +files = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "README.txt", "README", "CONTRIBUTING.md"] [controls."OSPS-DO-05.01".passes.pattern.patterns] eol_policy = "(end.of.(life|support)|EOL|deprecat|sunset)" @@ -4106,7 +4111,9 @@ look_for_urls = true [[controls."OSPS-SA-01.01".passes]] handler = "pattern" -files = ["ARCHITECTURE.md", "docs/architecture.md", "docs/design.md", "DESIGN.md", "README.md"] +# rst variants added; ARCHITECTURE/DESIGN commonly live as .rst +# in Sphinx-based Python projects. +files = ["ARCHITECTURE.md", "ARCHITECTURE.rst", "docs/architecture.md", "docs/architecture.rst", "docs/design.md", "docs/design.rst", "DESIGN.md", "DESIGN.rst", "README.md", "README.rst"] pass_if_any = true [controls."OSPS-SA-01.01".passes.pattern.patterns] @@ -4179,7 +4186,9 @@ look_for_urls = true # Check for API documentation in dedicated files OR README sections [[controls."OSPS-SA-02.01".passes]] handler = "pattern" -files = ["API.md", "docs/api.md", "README.md", "docs/README.md", "USAGE.md", "docs/getting-started.md", "openapi.yaml", "openapi.json", "swagger.yaml", "swagger.json"] +# README.rst added for Sphinx-based projects; API.rst / USAGE.rst +# variants common in the same ecosystem. +files = ["API.md", "API.rst", "docs/api.md", "docs/api.rst", "README.md", "README.rst", "docs/README.md", "USAGE.md", "USAGE.rst", "docs/getting-started.md", "openapi.yaml", "openapi.json", "swagger.yaml", "swagger.json"] pass_if_any = true [controls."OSPS-SA-02.01".passes.pattern.patterns] diff --git a/tests/darnit_baseline/controls/test_pattern_file_variants.py b/tests/darnit_baseline/controls/test_pattern_file_variants.py new file mode 100644 index 0000000..b6915bd --- /dev/null +++ b/tests/darnit_baseline/controls/test_pattern_file_variants.py @@ -0,0 +1,95 @@ +"""Regression guard for the pattern-handler file-list widening. + +Sibling to `test_llm_eval_file_contents.py` (#402). The 5 controls +that go `pattern -> llm_eval` (no `file_exists` between) previously +listed .md-only README candidates in their `pattern.files`. On projects +that ship a .rst / .txt / no-extension README (tqdm, and any Sphinx- +based Python project), the pattern tier resolved INCONCLUSIVE and +llm_eval fired -- which was the trigger for #402's empty-file_contents +bug in the first place. + +Widening the pattern lists to include README.rst / README.txt / README +means the deterministic tier PASSes or FAILs conclusively on those +repos, and the LLM tier fires less often. This test locks the widening +in place per control -- a revert of any entry re-opens the same class +of bug. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +try: + import tomllib +except ImportError: # Python 3.10 + import tomli as tomllib # type: ignore + +_FRAMEWORK_TOML = ( + Path(__file__).resolve().parents[3] + / "packages" + / "darnit-baseline" + / "src" + / "darnit_baseline" + / "openssf-baseline.toml" +) + + +def _load_framework() -> dict: + with open(_FRAMEWORK_TOML, "rb") as f: + return tomllib.load(f) + + +def _pattern_files(framework: dict, control_id: str) -> list[str]: + """Return the `files` list of the first `pattern` pass on control_id.""" + for p in framework["controls"][control_id]["passes"]: + if p.get("handler") in ("pattern", "regex"): + return list(p.get("files", [])) + return [] + + +# Controls whose pattern handler gates an llm_eval pass. Widening their +# file lists reduces how often the LLM tier fires on non-.md repos. +_GATED_CONTROLS = [ + "OSPS-DO-03.02", + "OSPS-DO-04.01", + "OSPS-DO-05.01", + "OSPS-SA-01.01", + "OSPS-SA-02.01", +] + + +@pytest.fixture(scope="module") +def framework() -> dict: + return _load_framework() + + +@pytest.mark.parametrize("control_id", _GATED_CONTROLS) +def test_pattern_list_covers_rst_readme(framework: dict, control_id: str): + """Each gated control's pattern.files MUST include README.rst. + + Without it, the deterministic tier can't resolve on Sphinx-based + projects and the pipeline falls through to `llm_eval`. + """ + files = _pattern_files(framework, control_id) + assert "README.rst" in files, ( + f"{control_id}: pattern handler's `files` list must include " + f"README.rst so the deterministic tier can resolve on rst-based " + f"projects (see #402). Current list: {files}" + ) + + +@pytest.mark.parametrize( + "control_id", + ["OSPS-DO-03.02", "OSPS-DO-04.01", "OSPS-DO-05.01"], +) +def test_pattern_list_covers_txt_and_no_ext_readme( + framework: dict, control_id: str +): + """DO-* controls that scan README should also accept `README.txt` + and the no-extension `README` filename convention some older + projects still use.""" + files = _pattern_files(framework, control_id) + assert "README.txt" in files, f"{control_id}: missing README.txt in {files}" + assert "README" in files, f"{control_id}: missing README in {files}"