fix(baseline): enumerate real candidate paths in llm_eval passes

packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml

@@ -1366,7 +1366,10 @@ prompt = """Evaluate whether this README adequately documents the project:
 2. Does it include installation or setup instructions?
 3. Does it provide usage examples or getting-started guidance?
 A README with only TODO placeholders or a single heading MUST fail."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: enumerate real candidate paths so llm_eval sees content
+# even when the deterministic tier resolved INCONCLUSIVE. Handler caps
+# at 5 files; mirrors the locator's `discover` list.
+files_to_include = ["README.md", "README.rst", "README.txt", "README", "readme.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Check for project description beyond just the name",
@@ -2334,7 +2337,8 @@ prompt = """Evaluate whether this security policy provides actionable vulnerabil
 2. Does it describe what information to include in a report?
 3. Does it mention an expected response timeline?
 A policy with only TODO placeholders or just the word 'security' MUST fail."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: enumerate real candidate paths (locator discover + llm_hints check_files).
+files_to_include = ["SECURITY.md", ".github/SECURITY.md", "docs/SECURITY.md", "README.md", "CONTRIBUTING.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Look for a concrete contact method, not just 'contact us'",
@@ -2476,7 +2480,8 @@ prompt = """Evaluate whether this governance document describes real project gov
 2. Does it describe how decisions are made (merge policy, consensus, voting)?
 3. Is this substantive governance documentation, not just boilerplate?
 A file with only TODO placeholders or generic text without named roles MUST fail."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: enumerate real candidate paths (locator discover + llm_hints check_files).
+files_to_include = ["GOVERNANCE.md", "MAINTAINERS.md", "CODEOWNERS", "README.md", "CONTRIBUTING.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Look for named maintainers or team references (@username, name)",
@@ -3861,7 +3866,9 @@ prompt = """Evaluate whether this document explains how to verify the identity o
 Look for: GPG/PGP key verification instructions, Sigstore/cosign verification steps,
 keyserver references, or other cryptographic identity verification methods.
 A general mention of 'signed releases' without verification instructions is insufficient."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: mirror the sibling pattern's file list. Adds README.rst
+# so llm_eval sees real content on rst-based repos.
+files_to_include = ["SECURITY.md", "README.md", "README.rst", "docs/releases.md", "CONTRIBUTING.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Look for specific verification commands or tools (gpg --verify, cosign verify)",
@@ -3916,7 +3923,8 @@ handler = "llm_eval"
 prompt = """Evaluate whether this document defines the scope and expected duration of support.
 Look for: which versions are supported, how long support lasts, what kind of support
 is provided (security fixes, bug fixes, feature development), and maintenance windows."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: mirror the sibling pattern's file list + rst variant.
+files_to_include = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "docs/support.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Look for version support matrices or timelines",
@@ -3979,7 +3987,8 @@ handler = "llm_eval"
 prompt = """Evaluate whether this document describes the end-of-support policy and process.
 Look for: deprecation timelines, sunset procedures, migration guidance,
 end-of-life announcements, or version retirement policies."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: mirror the sibling pattern's file list + rst variant.
+files_to_include = ["SUPPORT.md", "SECURITY.md", "README.md", "README.rst", "CONTRIBUTING.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Look for explicit deprecation timelines or sunset dates",
@@ -4122,7 +4131,8 @@ prompt = """Evaluate whether this design/architecture document describes real sy
 2. Does it describe actors (users, systems) that interact with the software?
 3. Does it explain data flows or interactions between components?
 A document with only TBD/TODO placeholders or generic headings without content MUST fail."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: mirror the sibling pattern's file list.
+files_to_include = ["ARCHITECTURE.md", "docs/architecture.md", "docs/design.md", "DESIGN.md", "README.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Look for named components with descriptions of what they do",
@@ -4195,7 +4205,8 @@ prompt = """Evaluate whether this API/interface documentation describes real API
 2. Does it include parameter descriptions, request/response formats, or usage examples?
 3. Is this substantive documentation, not just a heading with no content?
 A document with only headings or TODO placeholders MUST fail."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: mirror the sibling pattern's file list (pick top 5 of 10).
+files_to_include = ["API.md", "docs/api.md", "README.md", "openapi.yaml", "swagger.json"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Look for documented endpoints, methods, or CLI commands",
@@ -4329,7 +4340,8 @@ Your job is two-fold:
 A threat model with only false positives MUST FAIL. A threat model with at least some confirmed
 findings or honest "no threats found" sections should PASS.
 Report your verification results structured as: finding ID, verdict, reasoning."""
-files_to_include = ["$FOUND_FILE"]
+# Issue #402: mirror the locator's discover list (top 5 threat-model paths).
+files_to_include = ["docs/threatmodel/SUMMARY.md", "THREAT_MODEL.md", "docs/THREAT_MODEL.md", "docs/threat-model.md", "SECURITY.md"]
 confidence_threshold = 0.7
 analysis_hints = [
     "Code snippets are embedded in the threat model — use them as primary evidence",

tests/darnit_baseline/controls/test_llm_eval_file_contents.py

@@ -0,0 +1,99 @@
+"""Regression guard for issue #402.
+
+Every `llm_eval` pass in `openssf-baseline.toml` previously declared
+`files_to_include = ["$FOUND_FILE"]`. When the deterministic tier
+(pattern/regex) resolved INCONCLUSIVE -- i.e., no candidate file
+matched -- the `$FOUND_FILE` variable was never bound, so llm_eval
+fired with `file_contents = {}` and the LLM had nothing to reason
+about. In a 29-repo survey the LLM tier produced 9 non-answers, one
+per PENDING_LLM consultation.
+
+Fix (issue #402 option 1, TOML-only): every `llm_eval` pass now
+enumerates real candidate paths so the handler's on-disk skip logic
+can populate `file_contents` even without a preceding `file_exists`
+PASS. This test locks that fix in place -- if a new `llm_eval` block
+ships with lone `$FOUND_FILE`, we want it to fail here rather than in
+a live audit's empty consultation.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+try:
+    import tomllib
+except ImportError:  # Python 3.10
+    import tomli as tomllib  # type: ignore
+
+_FRAMEWORK_TOML = (
+    Path(__file__).resolve().parents[3]
+    / "packages"
+    / "darnit-baseline"
+    / "src"
+    / "darnit_baseline"
+    / "openssf-baseline.toml"
+)
+
+
+def _iter_llm_eval_passes():
+    """Yield (control_id, pass_index, pass_dict) for every llm_eval pass."""
+    with open(_FRAMEWORK_TOML, "rb") as f:
+        framework = tomllib.load(f)
+    for control_id, control in framework.get("controls", {}).items():
+        for i, p in enumerate(control.get("passes", [])):
+            if p.get("handler") == "llm_eval":
+                yield control_id, i, p
+
+
+class TestLLMEvalFilesToInclude:
+    def test_no_pass_ships_lone_found_file(self):
+        """FR: `files_to_include` MUST enumerate real candidate paths.
+
+        A bare `["$FOUND_FILE"]` produces `file_contents = {}` when the
+        preceding pass didn't set `gathered_evidence["found_file"]`
+        (which is the common case for `pattern` -> `llm_eval` shapes
+        that skip `file_exists`).
+        """
+        offenders: list[str] = []
+        for control_id, idx, p in _iter_llm_eval_passes():
+            fti = p.get("files_to_include", [])
+            if fti == ["$FOUND_FILE"]:
+                offenders.append(f"{control_id} pass[{idx}]")
+        assert not offenders, (
+            "The following llm_eval passes ship with lone $FOUND_FILE, "
+            "which produces empty file_contents when the deterministic "
+            "tier resolves INCONCLUSIVE (see issue #402). Enumerate real "
+            "candidate paths instead.\n\n"
+            + "\n".join(f"  - {o}" for o in offenders)
+        )
+
+    def test_every_llm_eval_declares_files_to_include(self):
+        """No llm_eval pass may omit `files_to_include` entirely."""
+        missing: list[str] = []
+        for control_id, idx, p in _iter_llm_eval_passes():
+            if "files_to_include" not in p:
+                missing.append(f"{control_id} pass[{idx}]")
+        assert not missing, (
+            "Every llm_eval pass MUST declare `files_to_include` so the "
+            "handler can gather file content for the consultation. "
+            "Missing on:\n\n" + "\n".join(f"  - {m}" for m in missing)
+        )
+
+    def test_files_to_include_within_handler_cap(self):
+        """Handler caps at `files_to_include[:5]` (`builtin_handlers.py`).
+
+        Declaring more than 5 is not an error but silently drops entries.
+        Flag it here so an author knows to prune.
+        """
+        overcapped: list[str] = []
+        for control_id, idx, p in _iter_llm_eval_passes():
+            fti = p.get("files_to_include", [])
+            if len(fti) > 5:
+                overcapped.append(
+                    f"{control_id} pass[{idx}] has {len(fti)} entries"
+                )
+        assert not overcapped, (
+            "llm_eval handler caps files_to_include at 5 entries; "
+            "additional entries are silently dropped. Prune:\n\n"
+            + "\n".join(f"  - {o}" for o in overcapped)
+        )