From 1fb9732df74da981469c10fea917e3420ec622cf Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Fri, 28 Aug 2026 18:53:34 -0400 Subject: [PATCH 1/2] fix(baseline): enumerate real candidate paths in llm_eval passes (closes #402) Every llm_eval pass in openssf-baseline.toml previously declared `files_to_include = ["$FOUND_FILE"]`. `$FOUND_FILE` is populated only by `file_exists_handler` on PASS; the `pattern`/`regex` handler never sets `gathered_evidence["found_file"]`. When the deterministic tier resolved INCONCLUSIVE (candidate `files` list produced no on-disk matches -- e.g. a .md-only pattern list on an rst-based repo like tqdm), the pipeline continued to `llm_eval` and shipped a consultation with `file_contents = {}`. The LLM reasoned about a list of file paths and dependency statuses but never saw actual file content. Symptom (surveyed across 29 repos): 9 Sonnet calls, 9 non-answers at 0.20-0.30 confidence, one per PENDING_LLM. tqdm produced 4 (README.rst only, no .md files in the pattern lists); renaming flask's README.md to README.rst took it from 0 to 5 PENDING controls. Fix: enumerate real candidate paths per issue #402 option 1 (TOML-only, no framework change). Each `files_to_include` list now mirrors the sibling `pattern` handler's `files` list (for the buggy pattern -> llm_eval controls) or the locator's `discover` list (for the file_exists -> ... -> llm_eval controls that were unreachable but still misleading). Adds .rst variants where the deterministic tier only listed .md files (DO-01.01, DO-03.02, DO-04.01, DO-05.01) so llm_eval can see rst content directly. The llm_eval handler caps at 5 files (builtin_handlers.py:628); every list stays within the cap. Regression guard: tests/darnit_baseline/controls/test_llm_eval_file_contents.py - test_no_pass_ships_lone_found_file: parses the TOML and asserts no llm_eval pass has files_to_include == ["$FOUND_FILE"]. - test_every_llm_eval_declares_files_to_include: files_to_include is mandatory for llm_eval. - test_files_to_include_within_handler_cap: warns if any pass exceeds the handler's 5-entry cap. Not in scope (follow-up): - Framework code changes to remove the 5-entry cap or generalize the handler's file resolution (option 2 or option 3 from the issue). - Widening the sibling `pattern` handler's file lists to accept .rst / .txt / no-extension README variants deterministically. That would make the pattern tier PASS/FAIL conclusively on more repos and reduce how often llm_eval is reached at all, but it changes the pattern handler's verdict distribution and belongs in a separate PR. --- .../src/darnit_baseline/openssf-baseline.toml | 30 ++++-- .../controls/test_llm_eval_file_contents.py | 101 ++++++++++++++++++ 2 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 tests/darnit_baseline/controls/test_llm_eval_file_contents.py diff --git a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml index 3ba794f..c701032 100644 --- a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml +++ b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml @@ -1366,7 +1366,10 @@ prompt = """Evaluate whether this README adequately documents the project: 2. Does it include installation or setup instructions? 3. Does it provide usage examples or getting-started guidance? A README with only TODO placeholders or a single heading MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: enumerate real candidate paths so llm_eval sees content +# even when the deterministic tier resolved INCONCLUSIVE. Handler caps +# at 5 files; mirrors the locator's `discover` list. +files_to_include = ["README.md", "README.rst", "README.txt", "README", "readme.md"] confidence_threshold = 0.7 analysis_hints = [ "Check for project description beyond just the name", @@ -2334,7 +2337,8 @@ prompt = """Evaluate whether this security policy provides actionable vulnerabil 2. Does it describe what information to include in a report? 3. Does it mention an expected response timeline? A policy with only TODO placeholders or just the word 'security' MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: enumerate real candidate paths (locator discover + llm_hints check_files). +files_to_include = ["SECURITY.md", ".github/SECURITY.md", "docs/SECURITY.md", "README.md", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for a concrete contact method, not just 'contact us'", @@ -2476,7 +2480,8 @@ prompt = """Evaluate whether this governance document describes real project gov 2. Does it describe how decisions are made (merge policy, consensus, voting)? 3. Is this substantive governance documentation, not just boilerplate? A file with only TODO placeholders or generic text without named roles MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: enumerate real candidate paths (locator discover + llm_hints check_files). +files_to_include = ["GOVERNANCE.md", "MAINTAINERS.md", "CODEOWNERS", "README.md", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for named maintainers or team references (@username, name)", @@ -3861,7 +3866,9 @@ prompt = """Evaluate whether this document explains how to verify the identity o Look for: GPG/PGP key verification instructions, Sigstore/cosign verification steps, keyserver references, or other cryptographic identity verification methods. A general mention of 'signed releases' without verification instructions is insufficient.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list. Adds README.rst +# so llm_eval sees real content on rst-based repos. +files_to_include = ["SECURITY.md", "README.md", "README.rst", "docs/releases.md", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for specific verification commands or tools (gpg --verify, cosign verify)", @@ -3916,7 +3923,8 @@ handler = "llm_eval" prompt = """Evaluate whether this document defines the scope and expected duration of support. Look for: which versions are supported, how long support lasts, what kind of support is provided (security fixes, bug fixes, feature development), and maintenance windows.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list + rst variant. +files_to_include = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "docs/support.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for version support matrices or timelines", @@ -3979,7 +3987,8 @@ handler = "llm_eval" prompt = """Evaluate whether this document describes the end-of-support policy and process. Look for: deprecation timelines, sunset procedures, migration guidance, end-of-life announcements, or version retirement policies.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list + rst variant. +files_to_include = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for explicit deprecation timelines or sunset dates", @@ -4122,7 +4131,8 @@ prompt = """Evaluate whether this design/architecture document describes real sy 2. Does it describe actors (users, systems) that interact with the software? 3. Does it explain data flows or interactions between components? A document with only TBD/TODO placeholders or generic headings without content MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list. +files_to_include = ["ARCHITECTURE.md", "docs/architecture.md", "docs/design.md", "DESIGN.md", "README.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for named components with descriptions of what they do", @@ -4195,7 +4205,8 @@ prompt = """Evaluate whether this API/interface documentation describes real API 2. Does it include parameter descriptions, request/response formats, or usage examples? 3. Is this substantive documentation, not just a heading with no content? A document with only headings or TODO placeholders MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list (pick top 5 of 10). +files_to_include = ["API.md", "docs/api.md", "README.md", "openapi.yaml", "swagger.json"] confidence_threshold = 0.7 analysis_hints = [ "Look for documented endpoints, methods, or CLI commands", @@ -4329,7 +4340,8 @@ Your job is two-fold: A threat model with only false positives MUST FAIL. A threat model with at least some confirmed findings or honest "no threats found" sections should PASS. Report your verification results structured as: finding ID, verdict, reasoning.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the locator's discover list (top 5 threat-model paths). +files_to_include = ["docs/threatmodel/SUMMARY.md", "THREAT_MODEL.md", "docs/THREAT_MODEL.md", "docs/threat-model.md", "SECURITY.md"] confidence_threshold = 0.7 analysis_hints = [ "Code snippets are embedded in the threat model — use them as primary evidence", diff --git a/tests/darnit_baseline/controls/test_llm_eval_file_contents.py b/tests/darnit_baseline/controls/test_llm_eval_file_contents.py new file mode 100644 index 0000000..6c0098c --- /dev/null +++ b/tests/darnit_baseline/controls/test_llm_eval_file_contents.py @@ -0,0 +1,101 @@ +"""Regression guard for issue #402. + +Every `llm_eval` pass in `openssf-baseline.toml` previously declared +`files_to_include = ["$FOUND_FILE"]`. When the deterministic tier +(pattern/regex) resolved INCONCLUSIVE -- i.e., no candidate file +matched -- the `$FOUND_FILE` variable was never bound, so llm_eval +fired with `file_contents = {}` and the LLM had nothing to reason +about. In a 29-repo survey the LLM tier produced 9 non-answers, one +per PENDING_LLM consultation. + +Fix (issue #402 option 1, TOML-only): every `llm_eval` pass now +enumerates real candidate paths so the handler's on-disk skip logic +can populate `file_contents` even without a preceding `file_exists` +PASS. This test locks that fix in place -- if a new `llm_eval` block +ships with lone `$FOUND_FILE`, we want it to fail here rather than in +a live audit's empty consultation. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +try: + import tomllib +except ImportError: # Python 3.10 + import tomli as tomllib # type: ignore + +_FRAMEWORK_TOML = ( + Path(__file__).resolve().parents[3] + / "packages" + / "darnit-baseline" + / "src" + / "darnit_baseline" + / "openssf-baseline.toml" +) + + +def _iter_llm_eval_passes(): + """Yield (control_id, pass_index, pass_dict) for every llm_eval pass.""" + with open(_FRAMEWORK_TOML, "rb") as f: + framework = tomllib.load(f) + for control_id, control in framework.get("controls", {}).items(): + for i, p in enumerate(control.get("passes", [])): + if p.get("handler") == "llm_eval": + yield control_id, i, p + + +class TestLLMEvalFilesToInclude: + def test_no_pass_ships_lone_found_file(self): + """FR: `files_to_include` MUST enumerate real candidate paths. + + A bare `["$FOUND_FILE"]` produces `file_contents = {}` when the + preceding pass didn't set `gathered_evidence["found_file"]` + (which is the common case for `pattern` -> `llm_eval` shapes + that skip `file_exists`). + """ + offenders: list[str] = [] + for control_id, idx, p in _iter_llm_eval_passes(): + fti = p.get("files_to_include", []) + if fti == ["$FOUND_FILE"]: + offenders.append(f"{control_id} pass[{idx}]") + assert not offenders, ( + "The following llm_eval passes ship with lone $FOUND_FILE, " + "which produces empty file_contents when the deterministic " + "tier resolves INCONCLUSIVE (see issue #402). Enumerate real " + "candidate paths instead.\n\n" + + "\n".join(f" - {o}" for o in offenders) + ) + + def test_every_llm_eval_declares_files_to_include(self): + """No llm_eval pass may omit `files_to_include` entirely.""" + missing: list[str] = [] + for control_id, idx, p in _iter_llm_eval_passes(): + if "files_to_include" not in p: + missing.append(f"{control_id} pass[{idx}]") + assert not missing, ( + "Every llm_eval pass MUST declare `files_to_include` so the " + "handler can gather file content for the consultation. " + "Missing on:\n\n" + "\n".join(f" - {m}" for m in missing) + ) + + def test_files_to_include_within_handler_cap(self): + """Handler caps at `files_to_include[:5]` (`builtin_handlers.py`). + + Declaring more than 5 is not an error but silently drops entries. + Flag it here so an author knows to prune. + """ + overcapped: list[str] = [] + for control_id, idx, p in _iter_llm_eval_passes(): + fti = p.get("files_to_include", []) + if len(fti) > 5: + overcapped.append( + f"{control_id} pass[{idx}] has {len(fti)} entries" + ) + assert not overcapped, ( + "llm_eval handler caps files_to_include at 5 entries; " + "additional entries are silently dropped. Prune:\n\n" + + "\n".join(f" - {o}" for o in overcapped) + ) From dc10bc29d411239a2e5de90987c0fb1fbb5ddf53 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Fri, 28 Aug 2026 18:58:11 -0400 Subject: [PATCH 2/2] style: drop unused pytest import (ruff F401) --- tests/darnit_baseline/controls/test_llm_eval_file_contents.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/darnit_baseline/controls/test_llm_eval_file_contents.py b/tests/darnit_baseline/controls/test_llm_eval_file_contents.py index 6c0098c..09acce0 100644 --- a/tests/darnit_baseline/controls/test_llm_eval_file_contents.py +++ b/tests/darnit_baseline/controls/test_llm_eval_file_contents.py @@ -20,8 +20,6 @@ from pathlib import Path -import pytest - try: import tomllib except ImportError: # Python 3.10