diff --git a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml index 3ba794f..c701032 100644 --- a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml +++ b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml @@ -1366,7 +1366,10 @@ prompt = """Evaluate whether this README adequately documents the project: 2. Does it include installation or setup instructions? 3. Does it provide usage examples or getting-started guidance? A README with only TODO placeholders or a single heading MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: enumerate real candidate paths so llm_eval sees content +# even when the deterministic tier resolved INCONCLUSIVE. Handler caps +# at 5 files; mirrors the locator's `discover` list. +files_to_include = ["README.md", "README.rst", "README.txt", "README", "readme.md"] confidence_threshold = 0.7 analysis_hints = [ "Check for project description beyond just the name", @@ -2334,7 +2337,8 @@ prompt = """Evaluate whether this security policy provides actionable vulnerabil 2. Does it describe what information to include in a report? 3. Does it mention an expected response timeline? A policy with only TODO placeholders or just the word 'security' MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: enumerate real candidate paths (locator discover + llm_hints check_files). +files_to_include = ["SECURITY.md", ".github/SECURITY.md", "docs/SECURITY.md", "README.md", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for a concrete contact method, not just 'contact us'", @@ -2476,7 +2480,8 @@ prompt = """Evaluate whether this governance document describes real project gov 2. Does it describe how decisions are made (merge policy, consensus, voting)? 3. Is this substantive governance documentation, not just boilerplate? A file with only TODO placeholders or generic text without named roles MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: enumerate real candidate paths (locator discover + llm_hints check_files). +files_to_include = ["GOVERNANCE.md", "MAINTAINERS.md", "CODEOWNERS", "README.md", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for named maintainers or team references (@username, name)", @@ -3861,7 +3866,9 @@ prompt = """Evaluate whether this document explains how to verify the identity o Look for: GPG/PGP key verification instructions, Sigstore/cosign verification steps, keyserver references, or other cryptographic identity verification methods. A general mention of 'signed releases' without verification instructions is insufficient.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list. Adds README.rst +# so llm_eval sees real content on rst-based repos. +files_to_include = ["SECURITY.md", "README.md", "README.rst", "docs/releases.md", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for specific verification commands or tools (gpg --verify, cosign verify)", @@ -3916,7 +3923,8 @@ handler = "llm_eval" prompt = """Evaluate whether this document defines the scope and expected duration of support. Look for: which versions are supported, how long support lasts, what kind of support is provided (security fixes, bug fixes, feature development), and maintenance windows.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list + rst variant. +files_to_include = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "docs/support.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for version support matrices or timelines", @@ -3979,7 +3987,8 @@ handler = "llm_eval" prompt = """Evaluate whether this document describes the end-of-support policy and process. Look for: deprecation timelines, sunset procedures, migration guidance, end-of-life announcements, or version retirement policies.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list + rst variant. +files_to_include = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "CONTRIBUTING.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for explicit deprecation timelines or sunset dates", @@ -4122,7 +4131,8 @@ prompt = """Evaluate whether this design/architecture document describes real sy 2. Does it describe actors (users, systems) that interact with the software? 3. Does it explain data flows or interactions between components? A document with only TBD/TODO placeholders or generic headings without content MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list. +files_to_include = ["ARCHITECTURE.md", "docs/architecture.md", "docs/design.md", "DESIGN.md", "README.md"] confidence_threshold = 0.7 analysis_hints = [ "Look for named components with descriptions of what they do", @@ -4195,7 +4205,8 @@ prompt = """Evaluate whether this API/interface documentation describes real API 2. Does it include parameter descriptions, request/response formats, or usage examples? 3. Is this substantive documentation, not just a heading with no content? A document with only headings or TODO placeholders MUST fail.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the sibling pattern's file list (pick top 5 of 10). +files_to_include = ["API.md", "docs/api.md", "README.md", "openapi.yaml", "swagger.json"] confidence_threshold = 0.7 analysis_hints = [ "Look for documented endpoints, methods, or CLI commands", @@ -4329,7 +4340,8 @@ Your job is two-fold: A threat model with only false positives MUST FAIL. A threat model with at least some confirmed findings or honest "no threats found" sections should PASS. Report your verification results structured as: finding ID, verdict, reasoning.""" -files_to_include = ["$FOUND_FILE"] +# Issue #402: mirror the locator's discover list (top 5 threat-model paths). +files_to_include = ["docs/threatmodel/SUMMARY.md", "THREAT_MODEL.md", "docs/THREAT_MODEL.md", "docs/threat-model.md", "SECURITY.md"] confidence_threshold = 0.7 analysis_hints = [ "Code snippets are embedded in the threat model — use them as primary evidence", diff --git a/tests/darnit_baseline/controls/test_llm_eval_file_contents.py b/tests/darnit_baseline/controls/test_llm_eval_file_contents.py new file mode 100644 index 0000000..09acce0 --- /dev/null +++ b/tests/darnit_baseline/controls/test_llm_eval_file_contents.py @@ -0,0 +1,99 @@ +"""Regression guard for issue #402. + +Every `llm_eval` pass in `openssf-baseline.toml` previously declared +`files_to_include = ["$FOUND_FILE"]`. When the deterministic tier +(pattern/regex) resolved INCONCLUSIVE -- i.e., no candidate file +matched -- the `$FOUND_FILE` variable was never bound, so llm_eval +fired with `file_contents = {}` and the LLM had nothing to reason +about. In a 29-repo survey the LLM tier produced 9 non-answers, one +per PENDING_LLM consultation. + +Fix (issue #402 option 1, TOML-only): every `llm_eval` pass now +enumerates real candidate paths so the handler's on-disk skip logic +can populate `file_contents` even without a preceding `file_exists` +PASS. This test locks that fix in place -- if a new `llm_eval` block +ships with lone `$FOUND_FILE`, we want it to fail here rather than in +a live audit's empty consultation. +""" + +from __future__ import annotations + +from pathlib import Path + +try: + import tomllib +except ImportError: # Python 3.10 + import tomli as tomllib # type: ignore + +_FRAMEWORK_TOML = ( + Path(__file__).resolve().parents[3] + / "packages" + / "darnit-baseline" + / "src" + / "darnit_baseline" + / "openssf-baseline.toml" +) + + +def _iter_llm_eval_passes(): + """Yield (control_id, pass_index, pass_dict) for every llm_eval pass.""" + with open(_FRAMEWORK_TOML, "rb") as f: + framework = tomllib.load(f) + for control_id, control in framework.get("controls", {}).items(): + for i, p in enumerate(control.get("passes", [])): + if p.get("handler") == "llm_eval": + yield control_id, i, p + + +class TestLLMEvalFilesToInclude: + def test_no_pass_ships_lone_found_file(self): + """FR: `files_to_include` MUST enumerate real candidate paths. + + A bare `["$FOUND_FILE"]` produces `file_contents = {}` when the + preceding pass didn't set `gathered_evidence["found_file"]` + (which is the common case for `pattern` -> `llm_eval` shapes + that skip `file_exists`). + """ + offenders: list[str] = [] + for control_id, idx, p in _iter_llm_eval_passes(): + fti = p.get("files_to_include", []) + if fti == ["$FOUND_FILE"]: + offenders.append(f"{control_id} pass[{idx}]") + assert not offenders, ( + "The following llm_eval passes ship with lone $FOUND_FILE, " + "which produces empty file_contents when the deterministic " + "tier resolves INCONCLUSIVE (see issue #402). Enumerate real " + "candidate paths instead.\n\n" + + "\n".join(f" - {o}" for o in offenders) + ) + + def test_every_llm_eval_declares_files_to_include(self): + """No llm_eval pass may omit `files_to_include` entirely.""" + missing: list[str] = [] + for control_id, idx, p in _iter_llm_eval_passes(): + if "files_to_include" not in p: + missing.append(f"{control_id} pass[{idx}]") + assert not missing, ( + "Every llm_eval pass MUST declare `files_to_include` so the " + "handler can gather file content for the consultation. " + "Missing on:\n\n" + "\n".join(f" - {m}" for m in missing) + ) + + def test_files_to_include_within_handler_cap(self): + """Handler caps at `files_to_include[:5]` (`builtin_handlers.py`). + + Declaring more than 5 is not an error but silently drops entries. + Flag it here so an author knows to prune. + """ + overcapped: list[str] = [] + for control_id, idx, p in _iter_llm_eval_passes(): + fti = p.get("files_to_include", []) + if len(fti) > 5: + overcapped.append( + f"{control_id} pass[{idx}] has {len(fti)} entries" + ) + assert not overcapped, ( + "llm_eval handler caps files_to_include at 5 entries; " + "additional entries are silently dropped. Prune:\n\n" + + "\n".join(f" - {o}" for o in overcapped) + )