From 5f98f1963d30446625922f7826e6a79940b3c578 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Thu, 20 Aug 2026 08:38:53 +0200 Subject: [PATCH] Scope whole-memo claim-quote exemptions to the claims table A hand-edited casefile could whitelist banned words in narrative on render-only reruns. The backstop now applies claim quotes only inside the claims-table region. Co-authored-by: Cursor --- ARCHITECTURE.md | 2 +- DECISIONS.md | 11 ++++ FUTURE.md | 1 - HANDOFF.md | 8 +-- scripts/check_language.py | 16 +++--- src/coldscreen/language.py | 83 ++++++++++++++++++++++++---- src/coldscreen/models.py | 5 +- src/coldscreen/pipeline.py | 24 ++++---- tests/test_exemption_attacks.py | 97 ++++++++++++++++++++++++++++++++- tests/test_golden.py | 11 ++++ tests/test_language.py | 28 +++++++++- 11 files changed, 247 insertions(+), 39 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1b817ad..85cccd1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -221,7 +221,7 @@ Recommendation: **MIT** for the code. Maximum reach, minimum friction, and the s - FCRA: prohibit use for employment, credit, or tenancy decisions in the docs. - GDPR: officer and PSC data is personal data from public registers. Legitimate interest is the usual basis for this kind of processing; ship a short `PRIVACY.md`, and any hosted deployment must handle erasure requests. - Companies House data reuse terms: verify the current terms before launch. -- Defamation posture: memos state record and confidence. Language rules in the synthesis prompt forbid accusatory phrasing; this is a technical control, not just a docs note. CI also scans the tool-authored statement fields on casefile.json. +- Defamation posture: memos state record and confidence. Language rules in the synthesis prompt forbid accusatory phrasing; this is a technical control, not just a docs note. The in-process whole-memo backstop scopes claim-quote exemptions to the rendered claims table. CI also scans the tool-authored statement fields on casefile.json. - No scraping behind authentication, ever. ## 16. Verify before build diff --git a/DECISIONS.md b/DECISIONS.md index de91c20..e65af01 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -204,6 +204,17 @@ The log is written from `write_case` through `write_case_text` (`O_NOFOLLOW`). B No live Companies House, OpenSanctions, Tavily, or model calls in this sprint. Committed fixtures were not given a historical log. `scripts/check_language.py` default targets were not expanded: the log is not tool prose. +### 2026-08-20: whole-memo claim-quote exemptions stay inside the claims table +`language_backstop_failure` used to apply stored claim texts as exemption spans across the entire rendered memo. The per-field synthesis gate already gives model prose zero claim-quote exemption, but `--render-only` (and `rerun` with no model) skips synthesis. A hand-edited `casefile.json` could plant a short claim, or copy a stored claim phrase into narrative, and the backstop would treat every matching span as quoted data. + +The in-process backstop now splits the memo at the template heading. The claims-table region starts at the first line that is exactly `## Claims vs evidence` and ends at the next line that starts with `## ` (that closer is not part of the region). Claim texts plus registry identity names are exempt inside that region. Identity names stay exempt before and after it. If the start heading is missing, or that heading has no following `## ` closer, the region is empty and claim-quote exemptions apply nowhere. If the heading appears more than once, only the first start/closer pair is used. + +The close is region-level, not cell-level. A hand-edited `record_note` that copies a claim's banned wording and still sits inside the table region can still be span-exempt there. That residual is accepted for this sprint. + +`scripts/check_language.py` memo line scanning is unchanged: it may still apply claim exemptions line-by-line across the whole memo. Label-aware CI re-verification stays in FUTURE.md. The synthesis per-field gate is unchanged (identity only). Quotation verification and match-length span advancement are unchanged. Pipe-escape in `_claim_rows` stays fail-closed: a claim that contains both `|` and a banned word is stored as the raw text and rendered escaped, so it no longer span-matches and the backstop fails closed. This sprint does not exempt the escaped form. A minimum-substance rule for stored claims is a different FUTURE bullet and was not added. + +No live Companies House, OpenSanctions, Tavily, or model calls in this sprint. Committed fixture casefiles and snapshot memos were not edited. + ## Section 16 verification log Findings are recorded here as verification completes, each with source URL and retrieval date. diff --git a/FUTURE.md b/FUTURE.md index d9cc738..5fcac60 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -24,7 +24,6 @@ Ideas deferred from the weekend 3 build and review: - OCR fallback for image-only decks; the gap is currently recorded explicitly as a finding. - Fuzzy quotation matching for claim verification, with a strict-verbatim default; per-claim provenance offsets into the extracted text for a stronger audit trail. -- Scope the whole-memo language backstop's exemptions to the rendered claims-table region, closing the hand-tampered-casefile residual on render-only reruns. - A minimum-substance rule for stored claims so single-word quotes cannot become exemption spans. - Sitemap-based about-page discovery and a configurable path token list for site extraction. - Label-aware claim re-verification in the language check script (mapping site evidence back to source labels). diff --git a/HANDOFF.md b/HANDOFF.md index 3f21185..6cfd734 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -8,7 +8,7 @@ The repository is at github.com/samrusani/coldscreen. Branch `main` is current a A CLI that turns a UK company name into a first-pass screening memo built entirely from public sources, with every finding traceable to evidence. It is not due diligence. It is the screen that decides whether due diligence is worth anyone's time. The same pipeline is also an MCP stdio server, so the screen runs inside agent workflows without a second implementation of it. -Current state: 766 tests, 28 modules, roughly 9,600 lines of source, green on Python 3.11 through 3.13. All three milestone success tests passed, two of them against the live Companies House API. Feature-complete for v0.1; not yet published to PyPI. Rubric 0.3 adds a mechanical R4 floor for origin-year contradictions ("operating since 2015" against incorporation in 2019), so that class of claims-bearing case now anchors unconditionally. Cache UX (`--refresh`, `coldscreen cache path|clear|stats`) landed after charges pagination. `--no-write` on `screen` skips the case directory for that run and still uses the HTTP cache; the default persist path is unchanged. A screen writes `fetch_log.json` at the case directory root (URLs, sanitized params, timestamps, cache flags; no bodies, no keys); rerun and `--no-write` do not. The language check now covers tool-authored casefile statements as well as rendered memos and templates. The stage-honesty phrase set grew from observed not-run lies; the gate is still a substring check, still sanctions and media only, and still arms only when those stages are recorded not run or failed. SNI through the pinned backend is proven by a loopback HTTPS handshake; the httpx pool assignment stays fail-closed by choice. +Current state: 774 tests, 28 modules, roughly 9,600 lines of source, green on Python 3.11 through 3.13. All three milestone success tests passed, two of them against the live Companies House API. Feature-complete for v0.1; not yet published to PyPI. Rubric 0.3 adds a mechanical R4 floor for origin-year contradictions ("operating since 2015" against incorporation in 2019), so that class of claims-bearing case now anchors unconditionally. Cache UX (`--refresh`, `coldscreen cache path|clear|stats`) landed after charges pagination. `--no-write` on `screen` skips the case directory for that run and still uses the HTTP cache; the default persist path is unchanged. A screen writes `fetch_log.json` at the case directory root (URLs, sanitized params, timestamps, cache flags; no bodies, no keys); rerun and `--no-write` do not. The language check now covers tool-authored casefile statements as well as rendered memos and templates. The stage-honesty phrase set grew from observed not-run lies; the gate is still a substring check, still sanctions and media only, and still arms only when those stages are recorded not run or failed. SNI through the pinned backend is proven by a loopback HTTPS handshake; the httpx pool assignment stays fail-closed by choice. ## The five non-negotiables @@ -27,7 +27,7 @@ The level is a pure function of the enforced trigger set. There is a property te **4. The language control is a technical control, not a tone preference.** Memos say "contradicted by public record" with a confidence tag; they never state or imply fraud, dishonesty, or intent. Three layers: the prompt states the rules, a per-field gate scans model output with one corrective retry, and a whole-memo scan runs before any memo reaches disk. The banned-word scan strips URLs (source URLs legitimately contain words like fraud in slugs) and exempts provenance-verified quoted data. -That exemption is the subtle part and it has already been attacked successfully once. Claim text is the company's own words, so a deck saying "we fight fraud" must render. But claim text comes from the model, so an unverified exemption is a laundering channel: an early design let a model smuggle arbitrary vocabulary into memos by inventing a "quotation". The fix, which you must not weaken: a claim is stored only if it verifies as a normalized verbatim substring of its declared source section, model prose gets zero exemptions ever, and the CI language script re-verifies exemptions against the committed evidence files. Four attack shapes are permanent regression tests in `tests/test_exemption_attacks.py`. +That exemption is the subtle part and it has already been attacked successfully once. Claim text is the company's own words, so a deck saying "we fight fraud" must render. But claim text comes from the model, so an unverified exemption is a laundering channel: an early design let a model smuggle arbitrary vocabulary into memos by inventing a "quotation". The fix, which you must not weaken: a claim is stored only if it verifies as a normalized verbatim substring of its declared source section, model prose gets zero exemptions ever, and the CI language script re-verifies exemptions against the committed evidence files. Four attack shapes are permanent regression tests in `tests/test_exemption_attacks.py`. The whole-memo backstop applies claim-quote exemptions only inside the rendered claims-table region. **5. No key material anywhere it can leak.** Secrets come from environment variables only. The Companies House key travels in a basic auth header, never in URLs, cache keys, evidence params, or `__repr__`. There is a test that greps every written file for the key. The tool bundles no OpenSanctions key: their terms make rights non-transferable, so every user brings their own key under their own licence. The MCP surface inherits this rather than reopening it: no tool schema has a field that could carry a key, there is a test that walks both schemas asserting so, and keys are read from the server process environment the host sets. @@ -84,9 +84,9 @@ Findings this loop caught that the test suite did not: pagination that silently FUTURE.md holds remaining items. My recommended ordering: -1. Scope the whole-memo language backstop's exemptions to the rendered claims-table region (FUTURE.md weekend-3 residual). +1. A minimum-substance rule for stored claims so single-word quotes cannot become exemption spans (FUTURE.md weekend-3 residual). -The per-run fetch log is done: `fetch_log.json` is written on screen, not on rerun, and not on `--no-write`. The evidence index stays a manifest. Duration and retry rows were not added. The five non-negotiables above are untouched. +The claims-table-region scope is done: the whole-memo backstop applies claim-quote exemptions only between `## Claims vs evidence` and the next `## ` heading, and fails closed when that region cannot be bounded. The per-run fetch log is done: `fetch_log.json` is written on screen, not on rerun, and not on `--no-write`. The evidence index stays a manifest. Duration and retry rows were not added. The five non-negotiables above are untouched. The language check now covers casefile statements: CI parses `casefile.json` and scans the tool-authored fields (finding statements, record notes, verdict rationale and questions, narrative, enforcement notes, skipped reasons). Media titles and claim texts are not scanned. Identity exemptions still require sibling registry evidence; claim-quote exemptions do not apply to those fields. diff --git a/scripts/check_language.py b/scripts/check_language.py index 7caf771..4d66108 100644 --- a/scripts/check_language.py +++ b/scripts/check_language.py @@ -40,13 +40,15 @@ Quoted-data exemption (memos only): a memo's claims table quotes the company's own deck and site words verbatim, and those may legitimately contain banned vocabulary. When a memo.md has a sibling casefile.json, its -stored claim texts are candidates for span-level exemption, the same -exemption the whole-memo backstop applies. But casefile.json is an -editable file, so a claim text is honored ONLY after re-verification -against the sibling evidence: normalized (whitespace, case, unicode quotes -and dashes), it must be a substring of the extracted source text persisted -in evidence/deck_text.json or the evidence/site_*.json records. No -evidence, no exemption: a hand-tampered casefile cannot widen this scan. +stored claim texts are candidates for span-level exemption on this +line-by-line memo scan. The in-process backstop scopes those same +claim texts to the claims-table region; this script does not. But +casefile.json is an editable file, so a claim text is honored ONLY +after re-verification against the sibling evidence: normalized +(whitespace, case, unicode quotes and dashes), it must be a substring +of the extracted source text persisted in evidence/deck_text.json or +the evidence/site_*.json records. No evidence, no exemption: a +hand-tampered casefile cannot widen this scan. Prose outside the exact verified quoted strings stays fully gated. That claim-quote exemption does not apply to the casefile fields above. diff --git a/src/coldscreen/language.py b/src/coldscreen/language.py index 6278faa..60eaf52 100644 --- a/src/coldscreen/language.py +++ b/src/coldscreen/language.py @@ -5,7 +5,7 @@ one place the banned list and the matching rules live. Every enforcement point imports find_banned_terms, so none of them can drift: the mechanical gate over model output (coldscreen.synthesis), the whole-memo backstop that -runs before any memo reaches disk (coldscreen.cli), and the CI gate over +runs before any memo reaches disk (coldscreen.pipeline), and the CI gate over rendered memos and the tool-authored fields of casefile.json (scripts/check_language.py). @@ -29,17 +29,23 @@ per-field gate in coldscreen.synthesis: the model references claims by id and never repeats their wording. That gate does apply the registry identity set, because prose has to be able to name the company and its -people. Only the whole-memo backstop and the CI memo scan take claim -texts, because the code-rendered claims table quotes the stored claim -strings verbatim. The CI casefile-field scan does not: those fields are -tool prose, same polarity as the synthesis per-field gate. Claim texts -themselves are trustworthy only because the claims stage verifies each -one is a real substring of its declared source section (after -normalize_for_match on both sides) before storing it, and -scripts/check_language.py re-verifies stored claims against the sibling -evidence files before honoring them on a memo; the script re-verifies -identity names against the registry evidence files the same way, and -that identity set is the one exemption the casefile-field scan applies. +people. The whole-memo backstop takes claim texts but applies those +exemptions only inside the rendered claims-table region: the first line +that is exactly `## Claims vs evidence` through the next ATX heading +that starts with `## ` (the closer is not part of the region). A missing +start or a missing closer leaves the region empty, so claim-quote +exemptions apply nowhere and identity exemptions still apply to the +whole memo. The CI memo scan still applies claim-quote exemptions +line-by-line across the file. The CI casefile-field scan does not take +claim texts: those fields are tool prose, same polarity as the synthesis +per-field gate. Claim texts themselves are trustworthy only because the +claims stage verifies each one is a real substring of its declared +source section (after normalize_for_match on both sides) before storing +it, and scripts/check_language.py re-verifies stored claims against the +sibling evidence files before honoring them on a memo; the script +re-verifies identity names against the registry evidence files the same +way, and that identity set is the one exemption the casefile-field scan +applies. Occurrence discovery advances by the full match length, so overlapping occurrences of a self-similar quote can never union into coverage of text that was never quoted as a whole. @@ -184,3 +190,56 @@ def find_banned_terms(text: str, exempt_texts: Iterable[str] = ()) -> list[str]: continue hits.append(match.group(0).lower()) return hits + + +# The template heading that opens the claims-vs-evidence table. Exact line +# match only: do not invent markers or HTML comments. +_CLAIMS_TABLE_HEADING = "## Claims vs evidence" + + +def _claims_table_region(memo: str) -> tuple[int, int] | None: + """Character offsets [start, end) of the claims-table region, or None. + + Start is the first line that is exactly the template heading. End is + the next line that starts with `## ` (that closer is excluded). A + missing start or a missing closer yields None: claim-quote exemptions + then apply nowhere. Only the first start/closer pair is used. + """ + start: int | None = None + offset = 0 + for line in memo.splitlines(keepends=True): + content = line.rstrip("\r\n") + if start is None: + if content == _CLAIMS_TABLE_HEADING: + start = offset + elif content.startswith("## "): + return start, offset + offset += len(line) + return None + + +def find_banned_terms_in_memo( + memo: str, + *, + claim_texts: Iterable[str] = (), + identity_names: Iterable[str] = (), +) -> list[str]: + """Banned terms in a rendered memo, with claim quotes region-scoped. + + The memo is scanned as three pieces. Before and after the claims-table + region, only identity_names are exempt. Inside the region, claim_texts + and identity_names are both exempt. Matching rules are find_banned_terms + unchanged. If the region cannot be bounded, the whole memo is scanned + with identity_names only. + """ + identity = tuple(identity_names) + bounds = _claims_table_region(memo) + if bounds is None: + return find_banned_terms(memo, identity) + start, end = bounds + claims_and_identity = (*claim_texts, *identity) + return ( + find_banned_terms(memo[:start], identity) + + find_banned_terms(memo[start:end], claims_and_identity) + + find_banned_terms(memo[end:], identity) + ) diff --git a/src/coldscreen/models.py b/src/coldscreen/models.py index 11cc81b..e60c36c 100644 --- a/src/coldscreen/models.py +++ b/src/coldscreen/models.py @@ -65,8 +65,9 @@ class Claim(BaseModel): (CLM-001 style), text is the company's own words as extracted from the deck or site, source names where they appeared ("deck p.4", "site /about"). Unfalsifiable puffery is kept with checkable False, never - dropped. Because text is quoted data, it is the one kind of string the - language gate exempts (span-level, exact match); see coldscreen.language. + dropped. Because text is quoted data, the whole-memo backstop exempts + it (span-level, exact match) only inside the rendered claims-table + region; see coldscreen.language. """ id: str diff --git a/src/coldscreen/pipeline.py b/src/coldscreen/pipeline.py index b116bf6..061f76d 100644 --- a/src/coldscreen/pipeline.py +++ b/src/coldscreen/pipeline.py @@ -63,7 +63,7 @@ from .deck import DeckError, DeckExtraction, extract_deck from .findings import build_findings from .http_cache import HttpCache -from .language import find_banned_terms, registry_identity_names +from .language import find_banned_terms_in_memo, registry_identity_names from .media import MediaStageResult, TavilyProvider, run_media from .models import CaseFile, CompanyCandidate, Officer from .providers import ModelProvider, ProviderError, get_provider, parse_model_spec @@ -207,17 +207,21 @@ def language_backstop_failure(memo: str, casefile: CaseFile) -> str | None: and the synthesis-failure memo). The per-field gate on model output already ran; this catches banned vocabulary arriving through any other channel. Two exemptions, both spans of code-verified data: the - casefile's stored claim texts (the company's own quoted words in the - claims table) and the registry identity names (registered name, - previous names, officer and PSC names), which are registry data the - memo must be able to spell out. On a hit the caller writes nothing and - the returned message reports a count only: the terms themselves never - reach the output either. + casefile's stored claim texts (the company's own quoted words, exempt + only inside the rendered claims-table region) and the registry + identity names (registered name, previous names, officer and PSC + names), which are registry data the memo must be able to spell out + anywhere. On a hit the caller writes nothing and the returned message + reports a count only: the terms themselves never reach the output + either. """ - exempt_texts = tuple(claim.text for claim in casefile.claims) + registry_identity_names( - casefile + count = len( + find_banned_terms_in_memo( + memo, + claim_texts=tuple(claim.text for claim in casefile.claims), + identity_names=registry_identity_names(casefile), + ) ) - count = len(find_banned_terms(memo, exempt_texts)) if not count: return None return ( diff --git a/tests/test_exemption_attacks.py b/tests/test_exemption_attacks.py index 8edbc7c..72e46df 100644 --- a/tests/test_exemption_attacks.py +++ b/tests/test_exemption_attacks.py @@ -18,27 +18,48 @@ from __future__ import annotations import importlib.util +import shutil from pathlib import Path from types import ModuleType import httpx import pytest import respx -from typer.testing import CliRunner +from typer.testing import CliRunner, Result from coldscreen.cli import app from coldscreen.language import find_banned_terms from coldscreen.models import CaseFile +from coldscreen.pipeline import language_backstop_failure from .conftest import FIXTURES_DIR, mock_company_routes from .fakes import assessment_json, claim_json, claims_json, synthesis_json DECK_PATH = FIXTURES_DIR / "deck_fabricated_widgets.pdf" +GOLDEN_DIR = FIXTURES_DIR / "golden" OLLAMA_CHAT_URL = "http://localhost:11434/api/chat" SCRIPT_PATH = Path(__file__).parent.parent / "scripts" / "check_language.py" +PUFFERY_QUOTE = "Our platform eliminates fraud in widget procurement" runner = CliRunner() + +def all_output(result: Result) -> str: + """stdout plus stderr, tolerant of click versions that separate them.""" + text = result.output + try: + text += result.stderr + except (ValueError, AttributeError): + pass + return text + + +def _write_casefile(case_dir: Path, casefile: CaseFile) -> None: + (case_dir / "casefile.json").write_text( + casefile.model_dump_json(indent=2) + "\n", encoding="utf-8" + ) + + # Genuine deck quotations (verbatim from the fixture deck) for the happy # parts of each attack scenario. GENUINE = claim_json("Operating since 2015 with a national footprint", "deck p.2", "history", True) @@ -269,3 +290,77 @@ def test_attack_4_planted_phrase_reused_in_narrative_fails_closed( memo = (case_dir(attack_env) / "memo.md").read_text(encoding="utf-8") assert find_banned_terms(memo) == [] assert load_check_language().main([str(case_dir(attack_env) / "memo.md")]) == 0 + + +# -- render-only hand-tamper: claim-quote exemptions stay in the table ------------ + + +def test_hand_tamper_short_claim_fails_render_only_backstop( + tmp_path: Path, respx_mock: respx.MockRouter +) -> None: + """A hand-edited short claim cannot whitelist the same word in narrative.""" + case_dir = tmp_path / "hand-tampered-short" + shutil.copytree(GOLDEN_DIR, case_dir) + casefile = CaseFile.model_validate_json( + (case_dir / "casefile.json").read_text(encoding="utf-8") + ) + claims = list(casefile.claims) + claims[0] = claims[0].model_copy(update={"text": "fraud"}) + _write_casefile( + case_dir, + casefile.model_copy( + update={ + "claims": claims, + "narrative": "The public record shows fraud in the filing pattern.", + } + ), + ) + (case_dir / "memo.md").unlink() + result = runner.invoke(app, ["rerun", str(case_dir), "--render-only"]) + combined = all_output(result) + assert result.exit_code == 1 + assert "language backstop" in combined + assert not (case_dir / "memo.md").exists() + assert "fraud" not in combined.lower() + + +def test_hand_tamper_full_phrase_in_prose_fails_render_only_backstop( + tmp_path: Path, respx_mock: respx.MockRouter +) -> None: + """Copying a stored claim phrase into narrative is no longer a whole-memo exempt span.""" + case_dir = tmp_path / "hand-tampered-phrase" + shutil.copytree(GOLDEN_DIR, case_dir) + casefile = CaseFile.model_validate_json( + (case_dir / "casefile.json").read_text(encoding="utf-8") + ) + assert PUFFERY_QUOTE in [c.text for c in casefile.claims] + _write_casefile(case_dir, casefile.model_copy(update={"narrative": PUFFERY_QUOTE})) + before_memo = (case_dir / "memo.md").read_bytes() + result = runner.invoke(app, ["rerun", str(case_dir), "--render-only"]) + combined = all_output(result) + assert result.exit_code == 1 + assert "language backstop" in combined + assert (case_dir / "memo.md").read_bytes() == before_memo + assert "fraud" not in combined.lower() + + +def test_hand_tamper_short_claim_passes_when_prose_is_clean( + tmp_path: Path, respx_mock: respx.MockRouter +) -> None: + """The short claim may render in the table when narrative stays clean.""" + case_dir = tmp_path / "hand-tampered-positive" + shutil.copytree(GOLDEN_DIR, case_dir) + casefile = CaseFile.model_validate_json( + (case_dir / "casefile.json").read_text(encoding="utf-8") + ) + claims = list(casefile.claims) + claims[0] = claims[0].model_copy(update={"text": "fraud"}) + tampered = casefile.model_copy(update={"claims": claims}) + _write_casefile(case_dir, tampered) + (case_dir / "memo.md").unlink() + result = runner.invoke(app, ["rerun", str(case_dir), "--render-only"]) + assert result.exit_code == 0, result.output + memo = (case_dir / "memo.md").read_text(encoding="utf-8") + assert '"fraud"' in memo + written = CaseFile.model_validate_json((case_dir / "casefile.json").read_text(encoding="utf-8")) + assert language_backstop_failure(memo, written) is None diff --git a/tests/test_golden.py b/tests/test_golden.py index 0a88082..4bac44a 100644 --- a/tests/test_golden.py +++ b/tests/test_golden.py @@ -35,6 +35,7 @@ from coldscreen.cli import app from coldscreen.language import find_banned_terms from coldscreen.models import CaseFile +from coldscreen.pipeline import language_backstop_failure from coldscreen.render import render_memo from coldscreen.rubric import detect_candidates from coldscreen.synthesis import ( @@ -228,6 +229,16 @@ def test_direction_three_same_word_in_a_record_note_fails() -> None: assert "fraud" in find_banned_terms(memo, claim_texts(poisoned)) +def test_golden_memo_passes_the_region_scoped_language_backstop() -> None: + """The committed golden memo (and a fresh render) stay backstop-clean.""" + casefile = load_casefile(GOLDEN_DIR) + memo = (GOLDEN_DIR / "memo.md").read_text(encoding="utf-8") + quoted = "Our platform eliminates fraud in widget procurement" + assert quoted in memo + assert language_backstop_failure(memo, casefile) is None + assert language_backstop_failure(render_memo(casefile), casefile) is None + + # -- green: GREEN with a supported claim ---------------------------------------- diff --git a/tests/test_language.py b/tests/test_language.py index 1998f93..93d7638 100644 --- a/tests/test_language.py +++ b/tests/test_language.py @@ -22,7 +22,7 @@ import pytest from coldscreen.casedir import load_casefile -from coldscreen.language import find_banned_terms, normalize_for_match +from coldscreen.language import find_banned_terms, find_banned_terms_in_memo, normalize_for_match from coldscreen.models import MediaItem, MediaScreening from coldscreen.render import render_memo @@ -171,6 +171,32 @@ def test_exemption_only_covers_the_named_terms_span() -> None: assert find_banned_terms(text, two_claims) == ["sham"] +# -- claims-table region scope on the whole-memo helper ------------------------- + + +def test_memo_helper_hits_when_the_claims_table_heading_is_missing() -> None: + memo = "The filing pattern is fraud.\n" + assert find_banned_terms_in_memo(memo, claim_texts=("fraud",)) == ["fraud"] + + +def test_memo_helper_hits_when_the_claims_table_heading_has_no_closer() -> None: + memo = '## Claims vs evidence\n| "fraud" |\nNo following heading.\n' + assert find_banned_terms_in_memo(memo, claim_texts=("fraud",)) == ["fraud"] + + +def test_memo_helper_exempts_a_claim_only_inside_the_table_region() -> None: + memo = '## Claims vs evidence\n| "fraud" |\n## Findings\nThe filing pattern is fraud.\n' + assert find_banned_terms_in_memo(memo, claim_texts=("fraud",)) == ["fraud"] + + +def test_memo_helper_uses_only_the_first_heading_pair() -> None: + memo = ( + '## Claims vs evidence\n| "fraud" |\n## Narrative\nclean\n' + '## Claims vs evidence\n| "fraud" |\n## Findings\n' + ) + assert find_banned_terms_in_memo(memo, claim_texts=("fraud",)) == ["fraud"] + + def _write_evidence(case_dir: Path, deck_pages: dict[str, str]) -> None: evidence_dir = case_dir / "evidence" evidence_dir.mkdir(exist_ok=True)