From bc4c3193f9517316415c52e2d6945895f00b93c1 Mon Sep 17 00:00:00 2001 From: Workbench Date: Mon, 25 May 2026 00:22:36 -0700 Subject: [PATCH 1/5] heartbeat test From fb2435b7f9d5b3abd0454f0c77df331835455954 Mon Sep 17 00:00:00 2001 From: Fable Run 2 Date: Wed, 8 Jul 2026 23:02:09 -0700 Subject: [PATCH 2/5] Piranha Lab: model-driven engine + deterministic offline verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D1: ModelEngine (portfolio/model_engine.py) — a model-driven Engine that delegates candidate generation to a pluggable proposer (in-process deterministic for the verifier; real LLM CLI via ClaudeCodeEngine for production). Sits below the sandbox/safety guards; writes only under ctx.workdir; never raises. D2: make_engine('model', ...) + --engine choices {model,codex,claude-code} in tools/piranha.py. No safety override added; non-shell engines require an explicit --engine-command (no fake default). D3: tools/piranha_model_verify.py — a deterministic, offline end-to-end verifier (no sys.executable, no network) proving the planted improvement wins the champion, a null variant is demoted (never champion) under the improvement gate, and S1-S4 + blast-radius parity + source immutability + report-always-written all hold. Emits 'PIRANHA-MODEL-ENGINE-VERIFY: PASS'. Non-vacuous (sensitivity-probed: an escaping write, a raising proposer, and a false-champion are all caught). Also fixed the historical env-flaky shell-engine tests toward determinism by quoting the (space-containing) interpreter path — no assertion weakened. Piranha suite now 20/20 (+ verifier) green. Co-Authored-By: Claude Opus 4.8 --- code_covenant/portfolio/engines.py | 21 ++ code_covenant/portfolio/model_engine.py | 202 ++++++++++++ code_covenant/tools/piranha.py | 19 +- code_covenant/tools/piranha_model_verify.py | 341 ++++++++++++++++++++ tests/test_piranha.py | 17 +- 5 files changed, 593 insertions(+), 7 deletions(-) create mode 100644 code_covenant/portfolio/model_engine.py create mode 100644 code_covenant/tools/piranha_model_verify.py diff --git a/code_covenant/portfolio/engines.py b/code_covenant/portfolio/engines.py index d6976bf..80337be 100644 --- a/code_covenant/portfolio/engines.py +++ b/code_covenant/portfolio/engines.py @@ -291,6 +291,27 @@ def make_engine( return ClaudeCodeEngine( command_template=command_template, output_mode=mode ) + if kind == "model": + # The model-driven engine. From the factory/CLI it is backed by a real + # LLM CLI (via ClaudeCodeEngine) — no fake default template. For the + # offline/deterministic verifier, construct ModelEngine directly with an + # in-process proposer (see portfolio.model_engine). + if not command_template: + raise ValueError( + "model engine requires a command_template (the LLM CLI that " + "drives proposals); for an offline/deterministic proposer, " + "construct ModelEngine(proposer=...) directly" + ) + from code_covenant.portfolio.model_engine import ( + ModelEngine, + llm_cli_proposer, + ) + + mode = output_mode or "stdout" + return ModelEngine( + proposer=llm_cli_proposer(command_template, output_mode=mode), + prompt_label="model-cli", + ) raise ValueError(f"unknown engine kind: {kind!r}") diff --git a/code_covenant/portfolio/model_engine.py b/code_covenant/portfolio/model_engine.py new file mode 100644 index 0000000..a2b3548 --- /dev/null +++ b/code_covenant/portfolio/model_engine.py @@ -0,0 +1,202 @@ +""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: code_covenant.portfolio.model_engine ║ +# ║ purpose: A model-driven Engine for the portfolio runner.║ +# ║ Delegates candidate generation to a `proposer` ║ +# ║ (ctx -> after_source). Two backings: an ║ +# ║ in-process deterministic proposer (offline, ║ +# ║ for the verifier) and a real LLM CLI reached ║ +# ║ through the existing ClaudeCodeEngine. ║ +# ║ ║ +# ║ inputs: ║ +# ║ - EngineContext describing the proposal request ║ +# ║ ║ +# ║ outputs: ║ +# ║ - EngineResult with after_source or an error message ║ +# ║ ║ +# ║ constraints: ║ +# ║ - Must never raise; failures go into EngineResult.error ║ +# ║ - Must write ONLY under ctx.workdir (blast-radius parity) ║ +# ║ - Must respect the configured timeout ║ +# ║ ║ +# ║ invariants: ║ +# ║ - result.ok is True iff after_source is a non-empty str ║ +# ║ - The engine proposes; it never selects or merges ║ +# ║ ║ +# ║ side_effects: ║ +# ║ - Writes scratch (before/cpc/program/after) under workdir ║ +# ║ - The LLM-CLI proposer spawns a subprocess (via ShellEngine)║ +# ║ ║ +# ║ forbidden_changes: ║ +# ║ - Do not let the engine touch the gate or champion table ║ +# ║ - Do not write outside ctx.workdir ║ +# ║ - Do not ship a fake default command_template ║ +# ║ ║ +# ║ optimization_targets: ║ +# ║ - [robustness] every failure becomes EngineResult.error ║ +# ║ - [clarity] a pluggable proposer seam ║ +# ║ ║ +# ║ risk_level: high ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: portfolio ║ +# ║ super_cohort: code_covenant_core ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +from __future__ import annotations + +import json +import time +from dataclasses import dataclass, field +from typing import Callable + +from code_covenant.portfolio.engines import ( + EngineContext, + EngineResult, + _prompt_id, +) + +# A proposer turns a proposal request into candidate source. It MAY raise — the +# ModelEngine converts any exception into EngineResult(ok=False, error=...), so the +# safety invariant "the engine never raises" is preserved no matter what the +# proposer does. +Proposer = Callable[[EngineContext], str] + + +@dataclass +class ModelEngine: + """A model-driven Engine: it *thinks* (via `proposer`) about a mutation of + `ctx.before_source` and returns it as a candidate. It sits strictly BELOW the + lab's sandbox/safety guards (run_lab → run_portfolio → engine.propose) and + inherits S1–S4 for free by (a) writing scratch only under ``ctx.workdir`` and + (b) never raising. It proposes; the portfolio gate decides merge/reject/review + and the champion table records the winner — the engine never touches either. + """ + + proposer: Proposer + name: str = "model" + prompt_label: str = "model" + + def propose(self, ctx: EngineContext) -> EngineResult: + start = time.perf_counter() + # Stable per-strategy prompt_id so lineage/ledger analytics can attribute + # proposals (the lineage walk keys on this downstream). + pid = _prompt_id(self.prompt_label or self.name) + try: + ctx.workdir.mkdir(parents=True, exist_ok=True) + # Blast-radius parity with ShellEngine: the SAME scratch surface, all + # under ctx.workdir — before.py, cpc.json, program.md (and after.py + # below). No path outside the workdir is ever touched. + (ctx.workdir / "before.py").write_text(ctx.before_source, encoding="utf-8") + (ctx.workdir / "cpc.json").write_text( + json.dumps(ctx.cpc, indent=2), encoding="utf-8") + (ctx.workdir / "program.md").write_text(ctx.program, encoding="utf-8") + after = self.proposer(ctx) + except Exception as exc: # noqa: BLE001 — never raise past here (S3 depends on it) + return EngineResult( + ok=False, after_source=None, + duration_s=time.perf_counter() - start, + error=f"model engine failed: {exc!r}", prompt_id=pid, + ) + if not isinstance(after, str) or not after: + return EngineResult( + ok=False, after_source=None, + duration_s=time.perf_counter() - start, + error="model engine returned empty or non-string output", + prompt_id=pid, + ) + try: + (ctx.workdir / "after.py").write_text(after, encoding="utf-8") + except OSError: + pass # scratch write is best-effort; the return value is authoritative + return EngineResult( + ok=True, after_source=after, + duration_s=time.perf_counter() - start, prompt_id=pid, + ) + + +# --- in-process deterministic proposers (offline; for the verifier + embedding) -- + +def strip_lines_proposer(marker: str) -> Proposer: + """A conservative, deterministic proposer: drop every source line containing + ``marker`` (e.g. a redundant-comment tag). Behavior-preserving when the marked + lines are inert — the perfect *plantable* improvement for the verifier.""" + + def _propose(ctx: EngineContext) -> str: + kept = [ln for ln in ctx.before_source.splitlines(keepends=True) + if marker not in ln] + return "".join(kept) + + return _propose + + +def append_comment_proposer(comment: str) -> Proposer: + """A deterministic *non-improving* proposer: append a harmless comment. Valid + and CPC-preserving, but moves no metric — used to prove the improvement gate + demotes a null variant (it must NOT become champion).""" + + def _propose(ctx: EngineContext) -> str: + src = ctx.before_source + if not src.endswith("\n"): + src += "\n" + return src + f"# {comment}\n" + + return _propose + + +@dataclass +class SequenceProposer: + """Cycle through several proposers, one per call — so a single round yields + multiple *model-generated variants* and champion selection runs over them.""" + + proposers: list # list[Proposer] + index: int = 0 + + def __call__(self, ctx: EngineContext) -> str: + if not self.proposers: + raise ValueError("SequenceProposer needs at least one proposer") + proposer = self.proposers[self.index % len(self.proposers)] + self.index += 1 + return proposer(ctx) + + +# --- production proposer: a real LLM CLI via the existing adapters --------------- + +def llm_cli_proposer( + command_template: str, + *, + output_mode: str = "stdout", + subdir: str = "llm", +) -> Proposer: + """Back the model engine with a real LLM CLI through the existing + ``ClaudeCodeEngine`` delegation (the preferred production path). The inner + engine writes ONLY under ``ctx.workdir/`` (containment preserved) and + its subprocess enforces ``ctx.timeout_s``. Any failure raises, which the + ModelEngine converts into ``EngineResult.error``. No fake default template — + the caller must supply the CLI command.""" + if not command_template: + raise ValueError("llm_cli_proposer requires a command_template") + + def _propose(ctx: EngineContext) -> str: + from code_covenant.portfolio.llm_engines import ClaudeCodeEngine + + inner = ClaudeCodeEngine( + command_template=command_template, output_mode=output_mode) + sub_ctx = EngineContext( + target_path=ctx.target_path, + before_source=ctx.before_source, + cpc=ctx.cpc, + program=ctx.program, + workdir=ctx.workdir / subdir, # stays under ctx.workdir + timeout_s=ctx.timeout_s, + ) + result = inner.propose(sub_ctx) + if not result.ok or not result.after_source: + raise RuntimeError(result.error or "llm cli produced no output") + return result.after_source + + return _propose diff --git a/code_covenant/tools/piranha.py b/code_covenant/tools/piranha.py index 946da81..05f3fa2 100644 --- a/code_covenant/tools/piranha.py +++ b/code_covenant/tools/piranha.py @@ -19,7 +19,8 @@ # ║ constraints: ║ # ║ - Must refuse to run without the confirmation flag ║ # ║ - Must reject sandbox roots that overlap the source ║ -# ║ - Must only support shell engines from the CLI today ║ +# ║ - Non-'shell' engines require an explicit --engine-command;║ +# ║ no engine bypasses the gate or the confirmation flag ║ # ║ ║ # ║ invariants: ║ # ║ - piranha_lab_report.md is always written on success ║ @@ -83,8 +84,20 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Allow the source to be a git repo root (default: refuse).", ) - parser.add_argument("--engine", choices=["shell"], default="shell") - parser.add_argument("--engine-command", default="", help="Shell engine template.") + parser.add_argument( + "--engine", + choices=["shell", "model", "codex", "claude-code"], + default="shell", + help="Proposal engine. 'model' is the model-driven engine (backed by an " + "LLM CLI given via --engine-command). No engine bypasses the gate or the " + "confirmation flag.", + ) + parser.add_argument( + "--engine-command", + default="", + help="Command template for the shell/model/codex/claude-code engines " + "(required for all non-'shell' engines too). No fake default is shipped.", + ) parser.add_argument("--rounds", type=int, default=3) parser.add_argument("--duration", type=float, default=60.0) parser.add_argument("--max-attempts-per-round", type=int, default=None) diff --git a/code_covenant/tools/piranha_model_verify.py b/code_covenant/tools/piranha_model_verify.py new file mode 100644 index 0000000..247e749 --- /dev/null +++ b/code_covenant/tools/piranha_model_verify.py @@ -0,0 +1,341 @@ +""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: code_covenant.tools.piranha_model_verify ║ +# ║ purpose: The runnable end-to-end verifier (D3) for the ║ +# ║ model-driven engine. Deterministic + offline. ║ +# ║ Proves a planted improvement wins the champion ║ +# ║ while every safety flag provably holds, and ║ +# ║ that a null variant is demoted (never champion)║ +# ║ under the improvement gate. Prints one ║ +# ║ greppable verdict line. ║ +# ║ ║ +# ║ inputs: ║ +# ║ - None (materialises its own throwaway source tree) ║ +# ║ ║ +# ║ outputs: ║ +# ║ - 'PIRANHA-MODEL-ENGINE-VERIFY: PASS' (exit 0) or ║ +# ║ 'PIRANHA-MODEL-ENGINE-VERIFY: FAIL: ' (exit 1) ║ +# ║ ║ +# ║ constraints: ║ +# ║ - Fully offline and deterministic (no network, ║ +# ║ no sys.executable in any spawned command) ║ +# ║ - Re-asserts S1–S4 with the new engine installed ║ +# ║ ║ +# ║ invariants: ║ +# ║ - A green run means the model engine widened no blast ║ +# ║ radius beyond the shell engine ║ +# ║ ║ +# ║ side_effects: ║ +# ║ - Creates + removes a temporary directory tree ║ +# ║ ║ +# ║ forbidden_changes: ║ +# ║ - Do not weaken a safety assertion to make the run pass ║ +# ║ ║ +# ║ optimization_targets: ║ +# ║ - [robustness] each safety flag asserted, not assumed ║ +# ║ ║ +# ║ risk_level: high ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: tools ║ +# ║ super_cohort: code_covenant_core ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +from __future__ import annotations + +import hashlib +import shutil +import sys +import tempfile +from pathlib import Path + +import pytest + +from code_covenant.piranha.lab import LabConfig, run_lab +from code_covenant.piranha.sandbox import SandboxError, validate_contained +from code_covenant.portfolio.model_engine import ( + ModelEngine, + SequenceProposer, + append_comment_proposer, + strip_lines_proposer, +) + +_MARK = "SLOWMARK" + + +def _cpc_block(module: str) -> str: + return f'''""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: {module:<48.48s} ║ +# ║ purpose: Compute doubles of the input. ║ +# ║ ║ +# ║ inputs: ║ +# ║ - value: int ║ +# ║ ║ +# ║ outputs: ║ +# ║ - result: int ║ +# ║ ║ +# ║ constraints: ║ +# ║ - Must not raise on int input. ║ +# ║ ║ +# ║ invariants: ║ +# ║ - Output deterministic. ║ +# ║ ║ +# ║ side_effects: ║ +# ║ - None ║ +# ║ ║ +# ║ forbidden_changes: ║ +# ║ - Do not change signature. ║ +# ║ ║ +# ║ optimization_targets: ║ +# ║ - [clarity] keep it small ║ +# ║ ║ +# ║ risk_level: low ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: demo ║ +# ║ super_cohort: demo_suite ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" +''' + + +# The seed target carries two inert marker-comment lines (the plantable "cost"). +# The planted improvement removes them → cost drops from 2 to 0, behaviour and +# signature unchanged (CPC preserved). +_SEED_BODY = ( + "def double(value: int) -> int:\n" + f" # {_MARK} redundant note one\n" + f" # {_MARK} redundant note two\n" + " return value * 2\n" +) + + +def _seed_source(tmp: Path, name: str = "source") -> Path: + source = tmp / name + source.mkdir(parents=True) + (source / "alpha.py").write_text(_cpc_block("demo.alpha") + _SEED_BODY) + return source + + +def _benchmark_for(sandbox_root: Path) -> str: + """A deterministic, offline benchmark: count the cost markers in the APPLIED + target and emit a METRIC line. Uses only /bin/sh + grep + printf — no + sys.executable (the exact dependence that made the shell e2e flaky), no + network. Smaller cost is better (minimize).""" + target = (sandbox_root.resolve() / "src" / "alpha.py") + script = ( + f'n=$(grep -c {_MARK} "{target}"); ' + 'printf "METRIC: cost %s\\n" "${n:-0}"' + ) + return f"/bin/sh -c '{script}'" + + +def _tree_hash(root: Path) -> str: + h = hashlib.sha256() + for p in sorted(root.rglob("*")): + if p.is_file(): + h.update(p.relative_to(root).as_posix().encode()) + h.update(b"\0") + h.update(p.read_bytes()) + h.update(b"\0") + return h.hexdigest() + + +def _all_files(root: Path) -> set[str]: + return {str(p.resolve()) for p in root.rglob("*") if p.is_file()} + + +class _Fail(AssertionError): + pass + + +def _check(cond: bool, msg: str) -> None: + if not cond: + raise _Fail(msg) + + +def _engine_workdirs(layout_root: Path) -> list[Path]: + return [p for p in layout_root.rglob("engine_workdirs/*") if p.is_dir()] + + +def _scenario_planted_wins(tmp: Path) -> None: + """Criteria 1, 3, 4: the planted improvement wins the champion; source + untouched; every output contained; blast radius not widened.""" + source = _seed_source(tmp, "srcA") + sandbox = tmp / "sbxA" + before_tree = _tree_hash(source) + before_files = _all_files(tmp) + + engine = ModelEngine(strip_lines_proposer(_MARK), prompt_label="strip") + config = LabConfig( + source_root=source, sandbox_root=sandbox, engine=engine, + confirm_sandbox_destructive=True, rounds=1, max_attempts_per_round=1, + duration_s=30.0, benchmark_command=_benchmark_for(sandbox), + require_metric_improvement=True, minimize_metrics=frozenset({"cost"}), + ) + result = run_lab(config) + layout_root = result.layout.root + + # (1) planted improvement wins the champion — matched to the planted variant, + # not merely "some champion exists". + merged = [e for e in result.ledger_entries if e.get("merged")] + _check(len(merged) == 1, f"expected exactly one merge, got {len(merged)}") + champ_id = merged[0].get("proposal_id") + entry = result.champion_table.entries.get("demo.alpha") + _check(entry is not None, "no champion entry for demo.alpha") + _check(entry.champion_proposal_id == champ_id, + f"champion {entry.champion_proposal_id!r} != merged {champ_id!r}") + _check(merged[0].get("metric_after") == {"cost": 0.0}, + f"champion metric is not the planted (cost=0): {merged[0].get('metric_after')}") + + # (3) source untouched — byte-for-byte full-tree hash. + _check(_tree_hash(source) == before_tree, "source tree changed during run") + _check((source / "alpha.py").read_text().count(_MARK) == 2, + "seed markers were mutated") + + # (3) every lab output path is contained inside the sandbox root. + for label, path in [ + ("report_path", result.report_path), + ("champion_state_path", result.champion_state_path), + ("ledger_path", result.layout.ledger_path), + ("proposals_dir", result.layout.proposals_dir), + ]: + validate_contained(path, layout_root) # raises SandboxError if it escapes + workdirs = _engine_workdirs(layout_root) + _check(len(workdirs) >= 1, "model engine wrote no workdir") + for wd in workdirs: + validate_contained(wd, layout_root) + for scratch in wd.rglob("*"): + validate_contained(scratch, layout_root) + + # (3) report always written with the required sections. + body = result.report_path.read_text() + for section in ("# Piranha Lab Report", "Champion state", "Best lineage", + "Worst lineage"): + _check(section in body, f"report missing section {section!r}") + + # (4) blast radius not widened: every NEW file created during the run lives + # inside the sandbox root — the model engine touched nothing the shell + # engine could not. + new_files = _all_files(tmp) - before_files + escaped = [f for f in new_files + if not f.startswith(str(layout_root.resolve()) + "/")] + _check(not escaped, f"files written outside the sandbox root: {escaped[:5]}") + + +def _scenario_null_demoted(tmp: Path) -> None: + """Criterion 2: with require_metric_improvement=True, a non-improving (null) + variant is demoted merge→review and does NOT occupy the champion slot.""" + source = _seed_source(tmp, "srcB") + sandbox = tmp / "sbxB" + engine = ModelEngine( + SequenceProposer([ + strip_lines_proposer(_MARK), # attempt 1: real improvement + append_comment_proposer("noop variant"), # attempt 2: null (ties) + ]), + prompt_label="seq", + ) + config = LabConfig( + source_root=source, sandbox_root=sandbox, engine=engine, + confirm_sandbox_destructive=True, rounds=1, max_attempts_per_round=2, + duration_s=30.0, benchmark_command=_benchmark_for(sandbox), + require_metric_improvement=True, minimize_metrics=frozenset({"cost"}), + ) + result = run_lab(config) + + merged = [e for e in result.ledger_entries if e.get("merged")] + reviewed = [e for e in result.ledger_entries + if _decision_of(e) == "review"] + _check(len(merged) == 1, f"expected exactly one merge (the improver), got {len(merged)}") + _check(len(reviewed) >= 1, "null variant was not demoted to review") + entry = result.champion_table.entries.get("demo.alpha") + _check(entry is not None, "no champion entry for demo.alpha") + _check(entry.champion_proposal_id == merged[0].get("proposal_id"), + "the null variant took the champion slot (must not)") + # the null variant is NOT the champion + null_ids = {e.get("proposal_id") for e in reviewed} + _check(entry.champion_proposal_id not in null_ids, + "champion id matches a demoted (review) proposal") + + +def _decision_of(entry: dict) -> str: + d = entry.get("decision") + if isinstance(d, dict): + return str(d.get("outcome", "")) + return str(d or "") + + +def _scenario_safety_flags(tmp: Path) -> None: + """S1 confirmation + S2 no-overlap re-asserted WITH the new engine, and the + zero-target report is still written.""" + source = _seed_source(tmp, "srcC") + engine = ModelEngine(strip_lines_proposer(_MARK), prompt_label="strip") + + # S1 — refuses without confirmation, with the model engine installed. + try: + run_lab(LabConfig(source_root=source, sandbox_root=tmp / "sbxC1", + engine=engine, rounds=1)) + raise _Fail("run_lab did not refuse without confirm_sandbox_destructive") + except SandboxError: + pass + + # S2 — rejects overlapping source/sandbox roots, with the model engine. + try: + run_lab(LabConfig(source_root=source, sandbox_root=source / "inside", + engine=engine, confirm_sandbox_destructive=True, rounds=1)) + raise _Fail("run_lab did not reject an overlapping sandbox root") + except SandboxError: + pass + + # S3 — report is written even with ZERO targets (empty source dir). + empty = tmp / "emptysrc" + empty.mkdir() + zresult = run_lab(LabConfig( + source_root=empty, sandbox_root=tmp / "sbxC2", engine=engine, + confirm_sandbox_destructive=True, rounds=1, duration_s=5.0)) + _check(zresult.report_path.exists(), "zero-target report not written") + zbody = zresult.report_path.read_text() + for section in ("# Piranha Lab Report", "Best lineage", "Worst lineage"): + _check(section in zbody, f"zero-target report missing {section!r}") + + +def verify() -> tuple[bool, str]: + """Run all scenarios. Returns (passed, reason).""" + tmp = Path(tempfile.mkdtemp(prefix="piranha_model_verify_")) + try: + _scenario_planted_wins(tmp) + _scenario_null_demoted(tmp) + _scenario_safety_flags(tmp) + return True, "planted improvement wins; null demoted; S1–S4 hold" + except _Fail as exc: + return False, str(exc) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def main(argv: list[str] | None = None) -> int: + passed, reason = verify() + if passed: + print("PIRANHA-MODEL-ENGINE-VERIFY: PASS") + return 0 + print(f"PIRANHA-MODEL-ENGINE-VERIFY: FAIL: {reason}") + return 1 + + +# --- pytest surface (runs alongside the existing suite) ------------------------ + +def test_piranha_model_engine_verify(): + passed, reason = verify() + assert passed, reason + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_piranha.py b/tests/test_piranha.py index c47844b..2fe2f68 100644 --- a/tests/test_piranha.py +++ b/tests/test_piranha.py @@ -3,9 +3,18 @@ from __future__ import annotations import json +import shlex import sys from pathlib import Path +# The interpreter path can contain spaces (e.g. a venv under +# "Application Support/"). ShellEngine shlex.split()s its command template, so an +# unquoted `sys.executable` gets torn into two argv entries and the engine +# subprocess never runs — the sole source of this suite's historical +# environment-dependent flakiness. Quoting it is a determinism fix, not a +# behaviour change: the assertions below are unchanged. +_PY = shlex.quote(sys.executable) + import pytest from code_covenant.piranha.champion import ChampionState, ChampionTable @@ -221,7 +230,7 @@ def test_run_lab_end_to_end(tmp_path: Path): optimizer = _write_optimizer(tmp_path) engine = ShellEngine( command_template=( - f"{sys.executable} {optimizer} --before {{before}} --after {{after}}" + f"{_PY} {optimizer} --before {{before}} --after {{after}}" ) ) config = LabConfig( @@ -247,7 +256,7 @@ def test_run_lab_writes_lab_report_content(tmp_path: Path): optimizer = _write_optimizer(tmp_path) engine = ShellEngine( command_template=( - f"{sys.executable} {optimizer} --before {{before}} --after {{after}}" + f"{_PY} {optimizer} --before {{before}} --after {{after}}" ) ) config = LabConfig( @@ -272,7 +281,7 @@ def test_run_lab_champion_state_reflects_merges(tmp_path: Path): optimizer = _write_optimizer(tmp_path) engine = ShellEngine( command_template=( - f"{sys.executable} {optimizer} --before {{before}} --after {{after}}" + f"{_PY} {optimizer} --before {{before}} --after {{after}}" ) ) config = LabConfig( @@ -310,7 +319,7 @@ def test_cli_piranha_runs_with_confirmation(tmp_path: Path, capsys): "--sandbox-root", str(tmp_path / "sbx"), "--i-understand-piranha-is-destructive", "--engine-command", - f"{sys.executable} {optimizer} --before {{before}} --after {{after}}", + f"{_PY} {optimizer} --before {{before}} --after {{after}}", "--rounds", "1", "--duration", "5", "--max-attempts-per-round", "1", From 366e89487c42a0c5efcb27b1a985b5069d5ef30b Mon Sep 17 00:00:00 2001 From: Cryptosourus Tex <268834908+cryptosourusTex@users.noreply.github.com> Date: Tue, 21 Jul 2026 00:19:18 -0700 Subject: [PATCH 3/5] Harden Piranha utility mutation campaigns --- README.md | 25 +- code_covenant/gate/folder.py | 126 ++- code_covenant/gate/gates.py | 46 +- code_covenant/gate/grading.py | 8 +- code_covenant/gate/improvement.py | 85 +- code_covenant/gate/pipeline.py | 257 +++++- code_covenant/gate/proposal_config.py | 4 + code_covenant/gate/utility.py | 684 +++++++++++++++ code_covenant/piranha/execution.py | 739 +++++++++++++++++ code_covenant/piranha/lab.py | 82 +- code_covenant/piranha/policy.py | 88 ++ code_covenant/piranha/sandbox.py | 134 ++- code_covenant/piranha/utility_engine.py | 784 ++++++++++++++++++ code_covenant/portfolio/engines.py | 141 +++- code_covenant/portfolio/model_engine.py | 47 +- code_covenant/portfolio/runner.py | 287 ++++++- code_covenant/portfolio/scope.py | 14 +- code_covenant/tools/piranha.py | 72 +- code_covenant/tools/piranha_model_verify.py | 130 ++- code_covenant/tools/piranha_utility_verify.py | 366 ++++++++ docs/PIRANHA_UTILITY_CAMPAIGNS.md | 179 ++++ docs/TUTORIAL.md | 28 +- tests/piranha_logging_oracle.py | 19 + tests/piranha_scope_oracle.py | 40 + tests/test_gate_improvement.py | 272 ++++-- tests/test_gate_pipeline.py | 132 +-- tests/test_piranha.py | 216 +++-- tests/test_piranha_security.py | 320 +++++++ tests/test_piranha_utility_engine.py | 357 ++++++++ tests/test_piranha_utility_gate.py | 631 ++++++++++++++ tests/test_portfolio_engines_and_program.py | 35 +- tests/test_portfolio_runner.py | 157 +++- 32 files changed, 6000 insertions(+), 505 deletions(-) create mode 100644 code_covenant/gate/utility.py create mode 100644 code_covenant/piranha/execution.py create mode 100644 code_covenant/piranha/policy.py create mode 100644 code_covenant/piranha/utility_engine.py create mode 100644 code_covenant/tools/piranha_utility_verify.py create mode 100644 docs/PIRANHA_UTILITY_CAMPAIGNS.md create mode 100644 tests/piranha_logging_oracle.py create mode 100644 tests/piranha_scope_oracle.py create mode 100644 tests/test_piranha_security.py create mode 100644 tests/test_piranha_utility_engine.py create mode 100644 tests/test_piranha_utility_gate.py diff --git a/README.md b/README.md index dad3081..28a02ee 100644 --- a/README.md +++ b/README.md @@ -254,17 +254,30 @@ observation hooks proposed for the safe levels. python -m code_covenant.cli piranha path/to/project \ --sandbox-root /tmp/piranha-sandbox \ --i-understand-piranha-is-destructive \ - --engine shell \ - --engine-command "python optimizer.py --before {before} --after {after}" \ + --engine utility \ + --test-command "python -m pytest -q -p no:cacheprovider" \ --rounds 5 \ --duration 300 \ --max-attempts-per-round 10 ``` -Copies the source into an isolated workspace, then runs aggressive -auto-merge optimisation rounds **inside the sandbox only**. The original -source is byte-identical before and after. Writes `piranha_lab_report.md` -with champion state and best / worst lineage. +The default utility engine proposes conservative AST-derived rewrites (Boolean +and guard-return collapse, `any()` reductions, and filtered comprehensions). Promotion is +fail-closed: exact CPC bytes and public signatures must survive, capabilities +cannot expand, immutable tests must pass in an OS-confined process, and every +declared trusted metric must be present and strictly Pareto-improve over the +current champion. The original source remains byte-identical. + +Piranha refuses existing unowned sandbox roots, source symlinks, unsafe CPC +module paths, missing behavior tests, and unsupported confinement hosts. The +built-in host backend is macOS `sandbox-exec`; use a dedicated VM/container on +other hosts. Run the adversarial check-of-the-check with: + +```bash +python -m code_covenant.tools.piranha_utility_verify +``` + +See `docs/PIRANHA_UTILITY_CAMPAIGNS.md` for model-engine and campaign guidance. ## Try the portfolio runner diff --git a/code_covenant/gate/folder.py b/code_covenant/gate/folder.py index 43111cb..09e7de4 100644 --- a/code_covenant/gate/folder.py +++ b/code_covenant/gate/folder.py @@ -22,6 +22,8 @@ # ║ - Must create all parent directories ║ # ║ - Must not overwrite an existing proposal folder ║ # ║ - Must write JSON with stable key ordering ║ +# ║ - Module and proposal id must be safe single path names ║ +# ║ - Proposal paths must stay under a real proposals_dir ║ # ║ ║ # ║ invariants: ║ # ║ - The returned path exists and contains the seven files ║ @@ -31,6 +33,7 @@ # ║ ║ # ║ forbidden_changes: ║ # ║ - Do not permit overwriting an existing proposal ║ +# ║ - Do not follow proposal or artifact symlinks ║ # ║ ║ # ║ optimization_targets: ║ # ║ - [clarity] obvious, stable on-disk layout ║ @@ -47,6 +50,7 @@ from __future__ import annotations import json +import re from pathlib import Path from typing import Any @@ -59,17 +63,38 @@ "decision.json", "rationale.md", ) +_MODULE_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*") +_PROPOSAL_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") -def build_proposal_folder( - proposals_dir: Path, module: str, proposal_id: str -) -> Path: +def build_proposal_folder(proposals_dir: Path, module: str, proposal_id: str) -> Path: """Create the per-proposal directory path and return it. Raises on conflict.""" - safe_module = module or "unknown_module" - folder = proposals_dir / safe_module / proposal_id - if folder.exists(): + _validate_module(module) + _validate_proposal_id(proposal_id) + + raw_base = proposals_dir.expanduser().absolute() + if raw_base.is_symlink(): + raise ValueError(f"proposals_dir must not be a symlink: {raw_base}") + raw_base.mkdir(parents=True, exist_ok=True) + if raw_base.is_symlink() or not raw_base.is_dir(): + raise NotADirectoryError(f"proposals_dir is not a real directory: {raw_base}") + base = raw_base.resolve() + + module_dir = base / module + _validate_contained(module_dir, base) + if module_dir.is_symlink(): + raise ValueError(f"proposal module directory must not be a symlink: {module_dir}") + if module_dir.exists() and not module_dir.is_dir(): + raise NotADirectoryError(f"proposal module path is not a directory: {module_dir}") + module_dir.mkdir(exist_ok=True) + _validate_real_contained_directory(module_dir, base) + + folder = module_dir / proposal_id + _validate_contained(folder, base) + if folder.exists() or folder.is_symlink(): raise FileExistsError(f"proposal folder already exists: {folder}") - folder.mkdir(parents=True, exist_ok=False) + folder.mkdir(exist_ok=False) + _validate_real_contained_directory(folder, base) return folder @@ -85,23 +110,88 @@ def write_proposal_artifacts( rationale: str, ) -> None: """Write every standard proposal artifact into folder.""" - (folder / "before.py").write_text(before_source, encoding="utf-8") - (folder / "after.py").write_text(after_source, encoding="utf-8") - (folder / "diff.patch").write_text(diff_text, encoding="utf-8") - (folder / "metrics.json").write_text( - json.dumps(metrics, indent=2, sort_keys=True) + "\n", encoding="utf-8" + raw_folder = folder.expanduser().absolute() + if raw_folder.is_symlink() or raw_folder.resolve() != raw_folder: + raise ValueError(f"proposal folder must not traverse symlinks: {raw_folder}") + if not raw_folder.is_dir(): + raise NotADirectoryError(f"proposal folder is not a directory: {raw_folder}") + _write_new_text(raw_folder / "before.py", before_source) + _write_new_text(raw_folder / "after.py", after_source) + _write_new_text(raw_folder / "diff.patch", diff_text) + _write_new_text( + raw_folder / "metrics.json", + json.dumps(metrics, indent=2, sort_keys=True) + "\n", ) - (folder / "constraint_grade.json").write_text( + _write_new_text( + raw_folder / "constraint_grade.json", json.dumps(constraint_grade, indent=2, sort_keys=True) + "\n", - encoding="utf-8", ) - (folder / "decision.json").write_text( - json.dumps(decision, indent=2, sort_keys=True) + "\n", encoding="utf-8" + _write_new_text( + raw_folder / "decision.json", + json.dumps(decision, indent=2, sort_keys=True) + "\n", ) - (folder / "rationale.md").write_text(rationale or "(no rationale supplied)\n", - encoding="utf-8") + _write_new_text(raw_folder / "rationale.md", rationale or "(no rationale supplied)\n") def required_artifact_files() -> tuple[str, ...]: """Return the tuple of filenames every proposal folder is expected to hold.""" return _ARTIFACT_FILES + + +def existing_proposal_folder(proposals_dir: Path, module: str, proposal_id: str) -> Path: + """Resolve an existing folder without permitting traversal or symlinks.""" + _validate_module(module) + _validate_proposal_id(proposal_id) + raw_base = proposals_dir.expanduser().absolute() + if raw_base.is_symlink(): + raise ValueError(f"proposals_dir is not a real directory: {raw_base}") + if not raw_base.exists(): + raise FileNotFoundError(f"proposals_dir does not exist: {raw_base}") + if not raw_base.is_dir(): + raise NotADirectoryError(f"proposals_dir is not a directory: {raw_base}") + base = raw_base.resolve() + folder = base / module / proposal_id + _validate_contained(folder, base) + _validate_real_contained_directory(folder, base) + return folder + + +def _validate_module(module: str) -> None: + if not isinstance(module, str) or len(module) > 255 or not _MODULE_RE.fullmatch(module): + raise ValueError(f"invalid CPC module for proposal folder: {module!r}") + + +def _validate_proposal_id(proposal_id: str) -> None: + if ( + not isinstance(proposal_id, str) + or not _PROPOSAL_ID_RE.fullmatch(proposal_id) + or proposal_id in {".", ".."} + ): + raise ValueError(f"invalid proposal_id for proposal folder: {proposal_id!r}") + + +def _validate_contained(path: Path, root: Path) -> None: + try: + path.resolve().relative_to(root.resolve()) + except ValueError as exc: + raise ValueError(f"proposal path escapes proposals_dir: {path}") from exc + + +def _validate_real_contained_directory(path: Path, root: Path) -> None: + if path.is_symlink(): + raise ValueError(f"proposal path is not a real directory: {path}") + if not path.exists(): + raise FileNotFoundError(f"proposal path does not exist: {path}") + if not path.is_dir(): + raise NotADirectoryError(f"proposal path is not a directory: {path}") + resolved = path.resolve() + _validate_contained(resolved, root) + if resolved != path: + raise ValueError(f"proposal path traverses a symlink: {path}") + + +def _write_new_text(path: Path, content: str) -> None: + # Exclusive creation prevents a pre-planted artifact symlink from being + # followed and also preserves the no-overwrite proposal invariant. + with path.open("x", encoding="utf-8") as stream: + stream.write(content) diff --git a/code_covenant/gate/gates.py b/code_covenant/gate/gates.py index 5504829..4805323 100644 --- a/code_covenant/gate/gates.py +++ b/code_covenant/gate/gates.py @@ -51,6 +51,10 @@ import time from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from code_covenant.piranha.execution import ConfinementPolicy @dataclass @@ -90,9 +94,10 @@ def test_gate( command: str, cwd: Path | None = None, timeout_s: float = 120.0, + confinement: "ConfinementPolicy | None" = None, ) -> GateResult: """Run a test command under a timeout and return a GateResult.""" - return _run_subprocess_gate("tests", command, cwd, timeout_s) + return _run_subprocess_gate("tests", command, cwd, timeout_s, confinement) test_gate.__test__ = False # type: ignore[attr-defined] # not a pytest test case despite the name @@ -102,9 +107,10 @@ def benchmark_gate( command: str, cwd: Path | None = None, timeout_s: float = 120.0, + confinement: "ConfinementPolicy | None" = None, ) -> GateResult: """Run a benchmark command and return a GateResult (no comparison).""" - return _run_subprocess_gate("benchmark", command, cwd, timeout_s) + return _run_subprocess_gate("benchmark", command, cwd, timeout_s, confinement) def _decode_io(value: bytes | str | None) -> str: @@ -120,6 +126,7 @@ def _run_subprocess_gate( command: str, cwd: Path | None, timeout_s: float, + confinement: "ConfinementPolicy | None" = None, ) -> GateResult: """Shared subprocess runner for test and benchmark gates.""" start = time.perf_counter() @@ -133,14 +140,26 @@ def _run_subprocess_gate( error=f"could not parse command: {exc}", ) try: - completed = subprocess.run( # noqa: S603 (caller-provided command, by design) - argv, - cwd=str(cwd) if cwd else None, - capture_output=True, - text=True, - timeout=timeout_s, - check=False, - ) + if confinement is not None: + if cwd is None: + raise ValueError("confined gates require an explicit cwd") + from code_covenant.piranha.execution import run_confined + + completed = run_confined( + argv, + cwd=cwd, + timeout_s=timeout_s, + policy=confinement, + ) + else: + completed = subprocess.run( # noqa: S603 (caller-provided command, by design) + argv, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) except FileNotFoundError as exc: return GateResult( name=name, @@ -157,6 +176,13 @@ def _run_subprocess_gate( stdout=_decode_io(exc.stdout), stderr=_decode_io(exc.stderr), ) + except (OSError, RuntimeError, ValueError) as exc: + return GateResult( + name=name, + ok=False, + duration_s=time.perf_counter() - start, + error=f"confinement failure: {exc}", + ) return GateResult( name=name, ok=completed.returncode == 0, diff --git a/code_covenant/gate/grading.py b/code_covenant/gate/grading.py index 4bf6be2..9aacd90 100644 --- a/code_covenant/gate/grading.py +++ b/code_covenant/gate/grading.py @@ -133,9 +133,11 @@ def decide( if not compile_result.ok: return Decision("reject", f"compile gate failed: {compile_result.error}") if test_result is not None and not test_result.ok: - return Decision("reject", "test gate failed") - if benchmark_result is not None and benchmark_result.error: - return Decision("reject", f"benchmark gate error: {benchmark_result.error}") + detail = test_result.error or test_result.stderr[-200:] or f"exit {test_result.exit_code}" + return Decision("reject", f"test gate failed: {detail}") + if benchmark_result is not None and not benchmark_result.ok: + detail = benchmark_result.error or f"exit {benchmark_result.exit_code}" + return Decision("reject", f"benchmark gate failed: {detail}") if after_validator_issues: return Decision( "reject", diff --git a/code_covenant/gate/improvement.py b/code_covenant/gate/improvement.py index 33dd275..eaca471 100644 --- a/code_covenant/gate/improvement.py +++ b/code_covenant/gate/improvement.py @@ -51,6 +51,7 @@ from __future__ import annotations import json +import math from dataclasses import dataclass from pathlib import Path @@ -66,13 +67,19 @@ class ImprovementVerdict: def find_baseline_metrics( - ledger_path: Path, module: str + ledger_path: Path, + module: str, + *, + metric_field: str = "metric_after", + fallback_field: str | None = None, ) -> dict[str, float] | None: - """Return metric_after of the most recent merged ledger entry for module.""" + """Return requested metrics from the most recent merged module entry.""" if not ledger_path.exists(): return None - most_recent: dict[str, float] | None = None - most_recent_ts: str = "" + primary: dict[str, float] | None = None + primary_ts = "" + fallback: dict[str, float] | None = None + fallback_ts = "" text = ledger_path.read_text(encoding="utf-8") for raw in text.splitlines(): stripped = raw.strip() @@ -86,18 +93,29 @@ def find_baseline_metrics( continue if not entry.get("merged"): continue - metrics = entry.get("metric_after") - if not isinstance(metrics, dict): - continue ts = str(entry.get("timestamp", "")) - if ts >= most_recent_ts: - most_recent_ts = ts - most_recent = { - k: float(v) - for k, v in metrics.items() - if isinstance(v, (int, float)) + metrics = entry.get(metric_field) + if isinstance(metrics, dict): + if ts >= primary_ts: + primary_ts = ts + primary = { + key: float(value) + for key, value in metrics.items() + if isinstance(value, (int, float)) + } + continue + metrics = entry.get(fallback_field) if fallback_field is not None else None + if isinstance(metrics, dict) and ts >= fallback_ts: + fallback_ts = ts + fallback = { + key: float(value) + for key, value in metrics.items() + if isinstance(value, (int, float)) } - return most_recent + # The fallback exists only for ledgers written before the dedicated field. + # Once any dedicated baseline exists, later utility-only entries must not + # silently replace it with a different metric namespace. + return primary if primary is not None else fallback def evaluate_improvement( @@ -105,8 +123,31 @@ def evaluate_improvement( candidate_metrics: dict[str, float], baseline_metrics: dict[str, float] | None, minimize: frozenset[str] = frozenset(), + required_metrics: frozenset[str] = frozenset(), ) -> ImprovementVerdict: """Strict-Pareto comparison of candidate vs baseline.""" + if not candidate_metrics: + return ImprovementVerdict( + improved=False, + reason="candidate has no metric_after; cannot evaluate improvement", + baseline=dict(baseline_metrics) if baseline_metrics is not None else None, + candidate=None, + ) + missing_candidate = required_metrics - set(candidate_metrics) + if missing_candidate: + return ImprovementVerdict( + improved=False, + reason=f"candidate missing required metrics: {sorted(missing_candidate)}", + baseline=dict(baseline_metrics) if baseline_metrics is not None else None, + candidate=dict(candidate_metrics) if candidate_metrics else None, + ) + if any(not math.isfinite(float(value)) for value in candidate_metrics.values()): + return ImprovementVerdict( + improved=False, + reason="candidate contains a non-finite metric", + baseline=dict(baseline_metrics) if baseline_metrics is not None else None, + candidate=dict(candidate_metrics), + ) if baseline_metrics is None: return ImprovementVerdict( improved=True, @@ -114,14 +155,22 @@ def evaluate_improvement( baseline=None, candidate=dict(candidate_metrics) if candidate_metrics else None, ) - if not candidate_metrics: + missing_baseline = required_metrics - set(baseline_metrics) + if missing_baseline: return ImprovementVerdict( improved=False, - reason="candidate has no metric_after; cannot evaluate improvement", + reason=f"baseline missing required metrics: {sorted(missing_baseline)}", baseline=dict(baseline_metrics), - candidate=None, + candidate=dict(candidate_metrics) if candidate_metrics else None, + ) + if any(not math.isfinite(float(value)) for value in baseline_metrics.values()): + return ImprovementVerdict( + improved=False, + reason="baseline contains a non-finite metric", + baseline=dict(baseline_metrics), + candidate=dict(candidate_metrics) if candidate_metrics else None, ) - shared = set(candidate_metrics) & set(baseline_metrics) + shared = required_metrics or (set(candidate_metrics) & set(baseline_metrics)) if not shared: return ImprovementVerdict( improved=False, diff --git a/code_covenant/gate/pipeline.py b/code_covenant/gate/pipeline.py index 3b23a75..7987164 100644 --- a/code_covenant/gate/pipeline.py +++ b/code_covenant/gate/pipeline.py @@ -51,7 +51,11 @@ from code_covenant.contracts.validator import validate_cpc from code_covenant.gate.checkpoint import restore_checkpoint, save_checkpoint from code_covenant.gate.diff import unified_diff -from code_covenant.gate.folder import build_proposal_folder, write_proposal_artifacts +from code_covenant.gate.folder import ( + build_proposal_folder, + existing_proposal_folder, + write_proposal_artifacts, +) from code_covenant.gate.gates import ( GateResult, benchmark_gate, @@ -65,6 +69,7 @@ select_grades, ) from code_covenant.gate.history import append_benchmark_row +from code_covenant.gate.improvement import ImprovementVerdict from code_covenant.gate.ledger import append_ledger_entry from code_covenant.gate.proposal_config import ProposalConfig, ProposalResult from code_covenant.gate.reports import ( @@ -86,13 +91,21 @@ def evaluate_proposal(config: ProposalConfig) -> ProposalResult: from code_covenant.prediction_loop import predict_or_skip - prediction = predict_or_skip( - config.predictor, before_cpc, before_source, after_source - ) + prediction = predict_or_skip(config.predictor, before_cpc, before_source, after_source) compile_result = compile_gate(after_source, filename=str(config.target_path)) + utility_evaluation, utility_result = _evaluate_utility( + config, before_source, after_source, compile_result + ) + risk = grade_risk(before_cpc or after_cpc) + utility_risk_allowed = config.utility_policy is None or config.utility_policy.allows_risk(risk) test_result, benchmark_result = _run_runtime_gates( - config, after_source, folder, compile_result.ok + config, + after_source, + folder, + compile_result.ok + and (utility_result is None or utility_result.ok) + and utility_risk_allowed, ) after_issues = _validator_issues(after_cpc) @@ -103,7 +116,6 @@ def evaluate_proposal(config: ProposalConfig) -> ProposalResult: after_cpc=after_cpc, semantic_grader=config.semantic_grader, ) - risk = grade_risk(before_cpc or after_cpc) decision = decide( compile_result, test_result, @@ -113,8 +125,45 @@ def evaluate_proposal(config: ProposalConfig) -> ProposalResult: config.merge_mode, after_issues, ) - metric_after = _extract_benchmark_metrics(benchmark_result) - decision = _apply_improvement_gate(config, decision, module, metric_after) + if utility_result is not None and not utility_result.ok: + decision = Decision("reject", f"utility safety gate failed: {utility_result.error}") + elif not utility_risk_allowed and config.utility_policy is not None: + decision = Decision( + "reject", + f"utility risk policy rejected {risk!r}; maximum is {config.utility_policy.max_risk!r}", + ) + elif ( + utility_evaluation is not None + and decision.outcome == "merge" + and not (utility_evaluation.improvement.improved) + ): + decision = Decision( + "review", + "utility gate demoted merge: " + utility_evaluation.improvement.reason, + ) + benchmark_metrics = _extract_benchmark_metrics(benchmark_result) + metric_before = ( + { + key: utility_evaluation.before_metrics[key] + for key in config.utility_policy.required_metrics + } + if utility_evaluation is not None and config.utility_policy is not None + else None + ) + metric_after = ( + { + key: utility_evaluation.after_metrics[key] + for key in config.utility_policy.required_metrics + } + if utility_evaluation is not None and config.utility_policy is not None + else benchmark_metrics + ) + decision, benchmark_improvement = _apply_improvement_gate( + config, + decision, + module, + benchmark_metrics, + ) diff_text = unified_diff(before_source, after_source) metrics = build_metrics_payload( proposal_id=proposal_id, @@ -126,6 +175,17 @@ def evaluate_proposal(config: ProposalConfig) -> ProposalResult: risk=risk, after_issues=after_issues, ) + if utility_evaluation is not None: + from code_covenant.gate.utility import evaluation_to_dict + + metrics["utility"] = evaluation_to_dict(utility_evaluation) + if benchmark_improvement is not None: + metrics["benchmark_improvement"] = { + "improved": benchmark_improvement.improved, + "reason": benchmark_improvement.reason, + "baseline": benchmark_improvement.baseline, + "candidate": benchmark_improvement.candidate, + } write_proposal_artifacts( folder, before_source=before_source, @@ -159,8 +219,25 @@ def evaluate_proposal(config: ProposalConfig) -> ProposalResult: engine_name=config.engine_name, prompt_template=config.prompt_template, ) + if metric_before: + entry["metric_before"] = dict(metric_before) if metric_after: entry["metric_after"] = dict(metric_after) + if benchmark_metrics: + entry["benchmark_metric_after"] = dict(benchmark_metrics) + if benchmark_improvement is not None: + if benchmark_improvement.baseline is not None: + entry["benchmark_metric_before"] = dict(benchmark_improvement.baseline) + entry["benchmark_improvement"] = { + "improved": benchmark_improvement.improved, + "reason": benchmark_improvement.reason, + } + if utility_evaluation is not None: + from code_covenant.gate.utility import evaluation_to_dict + + entry["utility_evaluation"] = evaluation_to_dict(utility_evaluation) + if config.mutation_metadata: + entry["mutation"] = dict(config.mutation_metadata) from code_covenant.prediction_loop import resolve_ledger_fields prediction_dict, attribution_dict = resolve_ledger_fields( @@ -204,14 +281,14 @@ def evaluate_proposal(config: ProposalConfig) -> ProposalResult: ) -def rollback_proposal( - proposals_dir: Path, module: str, proposal_id: str, target: Path -) -> None: +def rollback_proposal(proposals_dir: Path, module: str, proposal_id: str, target: Path) -> None: """Restore target from the before.py snapshot in the proposal folder.""" - folder = proposals_dir / module / proposal_id + folder = existing_proposal_folder(proposals_dir, module, proposal_id) before_path = folder / "before.py" - if not before_path.exists(): + if before_path.is_symlink() or not before_path.is_file(): raise FileNotFoundError(f"missing before.py for rollback: {before_path}") + if target.is_symlink() or not target.is_file(): + raise ValueError(f"rollback target is not a regular file: {target}") target.write_bytes(before_path.read_bytes()) @@ -228,18 +305,62 @@ def _run_runtime_gates( return test_result, benchmark_result if not (config.test_command or config.benchmark_command): return test_result, benchmark_result + if config.target_path.is_symlink() or not config.target_path.is_file(): + failure = GateResult( + name="tests", + ok=False, + duration_s=0.0, + error="target is not a regular non-symlink file", + ) + return failure, benchmark_result checkpoint_path = folder / "checkpoint.py" save_checkpoint(config.target_path, checkpoint_path) try: config.target_path.write_bytes(after_source.encode("utf-8")) if config.test_command: - test_result = test_gate( - config.test_command, config.cwd, config.timeout_s - ) + try: + test_confinement = _runtime_confinement( + config, + folder, + gate_name="tests", + require_completion_receipt=True, + ) + except (OSError, RuntimeError, ValueError) as exc: + test_result = GateResult( + name="tests", + ok=False, + duration_s=0.0, + error=f"confinement setup failed: {exc}", + ) + else: + test_result = test_gate( + config.test_command, + config.cwd, + config.timeout_s, + confinement=test_confinement, + ) if config.benchmark_command: - benchmark_result = benchmark_gate( - config.benchmark_command, config.cwd, config.timeout_s - ) + try: + benchmark_confinement = _runtime_confinement( + config, + folder, + gate_name="benchmark", + require_completion_receipt=False, + ) + except (OSError, RuntimeError, ValueError) as exc: + benchmark_result = GateResult( + name="benchmark", + ok=False, + duration_s=0.0, + error=f"confinement setup failed: {exc}", + ) + else: + benchmark_result = benchmark_gate( + config.benchmark_command, + config.cwd, + config.timeout_s, + confinement=benchmark_confinement, + ) finally: restore_checkpoint(config.target_path, checkpoint_path) # The intra-run checkpoint is redundant after restore — `before.py` @@ -251,9 +372,7 @@ def _run_runtime_gates( return test_result, benchmark_result -def _maybe_merge( - config: ProposalConfig, decision: Decision, after_source: str -) -> bool: +def _maybe_merge(config: ProposalConfig, decision: Decision, after_source: str) -> bool: if decision.outcome != "merge": return False config.target_path.write_bytes(after_source.encode("utf-8")) @@ -293,28 +412,100 @@ def _apply_improvement_gate( decision: Decision, module: str, metric_after: dict[str, float], -) -> Decision: +) -> tuple[Decision, ImprovementVerdict | None]: """Demote merge -> review when metric_after fails to beat the prior baseline.""" if not config.require_metric_improvement: - return decision - if decision.outcome != "merge": - return decision + return decision, None from code_covenant.gate.improvement import ( evaluate_improvement, find_baseline_metrics, ) - baseline = find_baseline_metrics(config.ledger_path, module) - verdict = evaluate_improvement( + baseline = find_baseline_metrics( + config.ledger_path, + module, + metric_field="benchmark_metric_after", + fallback_field="metric_after", + ) + verdict: ImprovementVerdict = evaluate_improvement( candidate_metrics=metric_after, baseline_metrics=baseline, minimize=config.minimize_metrics, + required_metrics=config.minimize_metrics, ) - if verdict.improved: - return decision - return Decision( - outcome="review", - reason=f"improvement gate demoted merge: {verdict.reason}", + if verdict.improved or decision.outcome != "merge": + return decision, verdict + return ( + Decision( + outcome="review", + reason=f"improvement gate demoted merge: {verdict.reason}", + ), + verdict, ) +def _evaluate_utility( + config: ProposalConfig, + before_source: str, + after_source: str, + compile_result: GateResult, +): + policy = config.utility_policy + if policy is None: + return None, None + if policy.require_tests and not config.test_command: + return None, GateResult( + name="utility-safety", + ok=False, + duration_s=0.0, + error="utility policy requires an immutable test command", + ) + if not compile_result.ok: + return None, None + try: + from code_covenant.gate.utility import evaluate_static_utility + + evaluation = evaluate_static_utility(before_source, after_source, policy) + except (SyntaxError, TypeError, ValueError) as exc: + return None, GateResult( + name="utility-safety", + ok=False, + duration_s=0.0, + error=f"static utility analysis failed: {exc}", + ) + return evaluation, GateResult( + name="utility-safety", + ok=evaluation.safe, + duration_s=0.0, + error="; ".join(evaluation.safety_reasons), + extra={"improved": evaluation.improvement.improved}, + ) + + +def _runtime_confinement( + config: ProposalConfig, + folder: Path, + *, + gate_name: str, + require_completion_receipt: bool, +): + if not _process_confinement_required(config): + return None + if config.cwd is None: + from code_covenant.piranha.execution import ConfinementError + + raise ConfinementError("confined runtime gates require project cwd") + from code_covenant.piranha.execution import ConfinementPolicy + + return ConfinementPolicy( + read_roots=(config.cwd,), + write_root=folder / "runtime" / gate_name, + allow_network=False, + require_completion_receipt=require_completion_receipt, + ) + + +def _process_confinement_required(config: ProposalConfig) -> bool: + return config.require_process_confinement or bool( + config.utility_policy and config.utility_policy.require_process_confinement + ) diff --git a/code_covenant/gate/proposal_config.py b/code_covenant/gate/proposal_config.py index 81f07e3..77f4ce8 100644 --- a/code_covenant/gate/proposal_config.py +++ b/code_covenant/gate/proposal_config.py @@ -49,6 +49,7 @@ if TYPE_CHECKING: from code_covenant.llm import LLMAdapter + from code_covenant.piranha.policy import UtilityPolicy from code_covenant.prediction import Predictor @@ -77,6 +78,9 @@ class ProposalConfig: predictor: "Predictor | None" = None require_metric_improvement: bool = False minimize_metrics: frozenset[str] = field(default_factory=frozenset) + utility_policy: "UtilityPolicy | None" = None + require_process_confinement: bool = False + mutation_metadata: dict[str, Any] = field(default_factory=dict) @dataclass diff --git a/code_covenant/gate/utility.py b/code_covenant/gate/utility.py new file mode 100644 index 0000000..573a827 --- /dev/null +++ b/code_covenant/gate/utility.py @@ -0,0 +1,684 @@ +""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: code_covenant.gate.utility ║ +# ║ purpose: Statically prove structural safety and ║ +# ║ deterministic code utility before execution. ║ +# ║ inputs: ║ +# ║ - before/after Python source, UtilityPolicy ║ +# ║ outputs: ║ +# ║ - UtilityEvaluation ║ +# ║ constraints: ║ +# ║ - Never executes candidate code ║ +# ║ - New capabilities and public-API changes fail closed ║ +# ║ invariants: ║ +# ║ - Metrics are calculated by trusted code, not candidates ║ +# ║ side_effects: ║ +# ║ - None ║ +# ║ forbidden_changes: ║ +# ║ - Do not accept candidate-supplied metric values ║ +# ║ optimization_targets: ║ +# ║ - [security] pre-execution capability gate ║ +# ║ - [accuracy] complete strict-Pareto utility score ║ +# ║ risk_level: critical ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: gate ║ +# ║ super_cohort: code_covenant_core ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +from __future__ import annotations + +import ast +import dis +from collections import Counter +from dataclasses import dataclass +from difflib import SequenceMatcher +from types import CodeType +from typing import Any + +from code_covenant.contracts.parser import find_cpc_blocks +from code_covenant.gate.improvement import ImprovementVerdict, evaluate_improvement +from code_covenant.piranha.policy import UtilityPolicy + + +@dataclass(frozen=True) +class UtilityEvaluation: + """Trusted static evidence used before any candidate execution.""" + + safe: bool + safety_reasons: tuple[str, ...] + before_metrics: dict[str, float] + after_metrics: dict[str, float] + improvement: ImprovementVerdict + before_capabilities: frozenset[str] + after_capabilities: frozenset[str] + changed_lines: int + + @property + def promotable(self) -> bool: + return self.safe and self.improvement.improved + + +def evaluate_static_utility( + before_source: str, + after_source: str, + policy: UtilityPolicy, +) -> UtilityEvaluation: + """Evaluate a bounded candidate without importing or executing it.""" + reasons: list[str] = [] + before_tree = ast.parse(before_source) + after_tree = ast.parse(after_source) + before_bytes = len(before_source.encode("utf-8")) + after_bytes = len(after_source.encode("utf-8")) + changed_lines = _changed_lines(before_source, after_source) + + if after_bytes > policy.max_candidate_bytes: + reasons.append( + f"candidate exceeds byte limit: {after_bytes} > {policy.max_candidate_bytes}" + ) + if before_bytes and after_bytes / before_bytes > policy.max_size_ratio: + reasons.append(f"candidate size ratio exceeds {policy.max_size_ratio:.2f}") + if changed_lines > policy.max_changed_lines: + reasons.append( + f"changed-line budget exceeded: {changed_lines} > {policy.max_changed_lines}" + ) + + if policy.preserve_cpc_bytes and _module_docstring_literal(before_source, before_tree) != ( + _module_docstring_literal(after_source, after_tree) + ): + reasons.append("CPC/module docstring bytes changed") + if policy.preserve_cpc_bytes and _cpc_payloads(before_source) != _cpc_payloads(after_source): + reasons.append("one or more CPC block bytes changed") + + if policy.preserve_public_api: + before_api = _public_api(before_tree) + after_api = _public_api(after_tree) + if before_api != after_api: + reasons.append("public API or signature changed") + + before_capabilities = _capabilities(before_tree) + after_capabilities = _capabilities(after_tree) + added = after_capabilities - before_capabilities + if policy.forbid_capability_expansion and added: + reasons.append(f"new code capabilities: {', '.join(sorted(added))}") + if policy.forbid_capability_expansion: + authority_changes = _authority_surface_changes(before_tree, after_tree) + if authority_changes: + reasons.append("authority-sensitive code changed: " + ", ".join(authority_changes)) + + before_metrics = _metrics(before_source, before_tree) + after_metrics = _metrics(after_source, after_tree) + required_before = {key: before_metrics[key] for key in policy.required_metrics} + required_after = {key: after_metrics[key] for key in policy.required_metrics} + improvement = evaluate_improvement( + candidate_metrics=required_after, + baseline_metrics=required_before, + minimize=policy.minimize_metrics, + required_metrics=policy.required_metrics, + ) + return UtilityEvaluation( + safe=not reasons, + safety_reasons=tuple(reasons), + before_metrics=before_metrics, + after_metrics=after_metrics, + improvement=improvement, + before_capabilities=before_capabilities, + after_capabilities=after_capabilities, + changed_lines=changed_lines, + ) + + +def _metrics(source: str, tree: ast.AST) -> dict[str, float]: + code = compile(source, "", "exec") + return { + "source_bytes": float(len(source.encode("utf-8"))), + "ast_nodes": float(sum(1 for _ in ast.walk(tree))), + "branch_nodes": float(sum(1 for node in ast.walk(tree) if _is_branch(node))), + "bytecode_instructions": float(_instruction_count(code)), + } + + +def _instruction_count(code: CodeType) -> int: + count = sum(1 for _ in dis.get_instructions(code)) + for value in code.co_consts: + if isinstance(value, CodeType): + count += _instruction_count(value) + return count + + +def _is_branch(node: ast.AST) -> bool: + return isinstance( + node, + ( + ast.If, + ast.IfExp, + ast.For, + ast.AsyncFor, + ast.While, + ast.Try, + ast.TryStar, + ast.BoolOp, + ast.Match, + ast.comprehension, + ), + ) + + +def _module_docstring_literal(source: str, tree: ast.AST) -> str | None: + if not isinstance(tree, ast.Module) or not tree.body: + return None + first = tree.body[0] + if not ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + return None + return ast.get_source_segment(source, first) + + +def _cpc_payloads(source: str) -> tuple[str, ...]: + lines = source.splitlines(keepends=True) + return tuple("".join(lines[start - 1 : end]) for start, end in find_cpc_blocks(source)) + + +def _public_api(tree: ast.AST) -> dict[str, str]: + if not isinstance(tree, ast.Module): + return {} + result: dict[str, str] = {} + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and not node.name.startswith( + "_" + ): + result[node.name] = _function_signature(node) + elif isinstance(node, ast.ClassDef) and not node.name.startswith("_"): + bases = ",".join(ast.dump(item, include_attributes=False) for item in node.bases) + keywords = ",".join(ast.dump(item, include_attributes=False) for item in node.keywords) + result[node.name] = f"class:{bases}:{keywords}" + for child in node.body: + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and not ( + child.name.startswith("_") and child.name not in {"__init__", "__call__"} + ): + result[f"{node.name}.{child.name}"] = _function_signature(child) + return result + + +def _function_signature(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str: + kind = "async" if isinstance(node, ast.AsyncFunctionDef) else "sync" + returns = ast.dump(node.returns, include_attributes=False) if node.returns else "" + decorators = ",".join(ast.dump(item, include_attributes=False) for item in node.decorator_list) + return f"{kind}:{ast.dump(node.args, include_attributes=False)}:{returns}:{decorators}" + + +_CAPABILITY_MODULES: dict[str, str] = { + "atexit": "interpreter-control", + "builtins": "dynamic-code", + "socket": "network", + "requests": "network", + "urllib": "network", + "httpx": "network", + "aiohttp": "network", + "subprocess": "process", + "multiprocessing": "process", + "ctypes": "native-code", + "importlib": "dynamic-code", + "inspect": "reflection", + "io": "filesystem", + "os": "system", + "pathlib": "filesystem", + "posix": "system", + "resource": "process", + "shutil": "filesystem-write", + "signal": "process", + "sys": "interpreter-control", + "tempfile": "filesystem-write", + "pickle": "unsafe-deserialization", + "marshal": "unsafe-deserialization", +} +_CAPABILITY_CALLS: dict[str, str] = { + "open": "filesystem", + "eval": "dynamic-code", + "exec": "dynamic-code", + "compile": "dynamic-code", + "__import__": "dynamic-code", + "system": "process", + "popen": "process", + "run": "process", + "call": "process", + "check_call": "process", + "check_output": "process", + "unlink": "filesystem-write", + "remove": "filesystem-write", + "rmtree": "filesystem-write", + "write_text": "filesystem-write", + "write_bytes": "filesystem-write", + "rename": "filesystem-write", + "replace": "filesystem-write", + "connect": "network", + "urlopen": "network", + "getenv": "environment", +} + +# Capability categories intentionally remain useful, compact evidence, but they +# are too coarse to guard an authority boundary by themselves: adding a second +# filesystem call to a module that already opens one file would not add a new +# category. These surfaces and their authority-bearing containers are therefore +# compared as exact AST multisets. A proposal may not add, remove, move, or alter +# one, nor change a local function that directly or transitively reaches one. +_AUTHORITY_MODULES: frozenset[str] = frozenset( + { + *_CAPABILITY_MODULES, + "builtins", + "atexit", + "importlib", + "inspect", + "io", + "os", + "pathlib", + "posix", + "resource", + "shutil", + "signal", + "sitecustomize", + "sys", + "tempfile", + } +) +_AUTHORITY_CALL_NAMES: frozenset[str] = frozenset( + { + *_CAPABILITY_CALLS, + "_exit", + "abort", + "breakpoint", + "chroot", + "delattr", + "execl", + "execle", + "execlp", + "execlpe", + "execv", + "execve", + "execvp", + "execvpe", + "exit", + "fork", + "forkpty", + "FileIO", + "getattr", + "globals", + "import_module", + "kill", + "killpg", + "locals", + "pthread_kill", + "posix_spawn", + "posix_spawnp", + "quit", + "raise_signal", + "reload", + "send_signal", + "setattr", + "setgid", + "setpgid", + "setsid", + "setuid", + "spawnl", + "spawnle", + "spawnlp", + "spawnlpe", + "spawnv", + "spawnve", + "spawnvp", + "spawnvpe", + "startfile", + "terminate", + "unregister", + "vars", + } +) +_AUTHORITY_ATTRIBUTES: frozenset[str] = frozenset( + { + "__bases__", + "__builtins__", + "__class__", + "__closure__", + "__code__", + "__dict__", + "__func__", + "__getattribute__", + "__globals__", + "__import__", + "__loader__", + "__mro__", + "__setattr__", + "__spec__", + "__subclasses__", + "_exit", + "modules", + } +) +_DYNAMIC_AUTHORITY_NAMES: frozenset[str] = frozenset( + { + "__builtins__", + "__import__", + "_exit", + "compile", + "eval", + "exec", + "exit", + "quit", + } +) +_TERMINATION_EXCEPTIONS: frozenset[str] = frozenset({"SystemExit"}) + + +def _capabilities(tree: ast.AST) -> frozenset[str]: + found: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + root = alias.name.split(".", 1)[0] + capability = _CAPABILITY_MODULES.get(root) + if capability: + found.add(capability) + elif isinstance(node, ast.ImportFrom) and node.module: + root = node.module.split(".", 1)[0] + capability = _CAPABILITY_MODULES.get(root) + if capability: + found.add(capability) + elif isinstance(node, ast.Call): + name = _call_name(node.func) + capability = _CAPABILITY_CALLS.get(name) + if capability: + found.add(capability) + elif isinstance(node, ast.Attribute) and node.attr == "environ": + found.add("environment") + return frozenset(found) + + +def _call_name(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return "" + + +def _authority_surface_changes(before_tree: ast.AST, after_tree: ast.AST) -> list[str]: + """Describe any structural change to termination or dynamic authority.""" + before = _authority_surfaces(before_tree) + after = _authority_surfaces(after_tree) + additions = after - before + removals = before - after + changes: list[str] = [] + for prefix, delta in (("added", additions), ("removed/altered", removals)): + for (label, _fingerprint), count in sorted(delta.items()): + suffix = f" x{count}" if count > 1 else "" + changes.append(f"{prefix} {label}{suffix}") + return changes + + +def _authority_surfaces(tree: ast.AST) -> Counter[tuple[str, str]]: + """Return exact, non-executing fingerprints for authority-bearing syntax.""" + aliases = _import_aliases(tree) + sensitive_functions = _transitively_sensitive_functions(tree, aliases) + parents = _parent_nodes(tree) + surfaces: Counter[tuple[str, str]] = Counter() + for function in ast.walk(tree): + if isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( + function.name in sensitive_functions + ): + _record_surface( + surfaces, + f"authority-function:{function.name}", + function, + ) + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)) and _sensitive_import(node): + _record_surface(surfaces, "import", node) + _record_non_function_container(surfaces, node, parents) + continue + if isinstance(node, ast.Call): + qualified = _qualified_name(node.func, aliases) + terminal = qualified.rsplit(".", maxsplit=1)[-1] + if _authority_call(node.func, aliases, sensitive_functions): + _record_surface(surfaces, f"call:{qualified or terminal}", node) + _record_non_function_container(surfaces, node, parents) + continue + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + qualified = aliases.get(node.id, node.id) + if qualified.rsplit(".", maxsplit=1)[-1] in sensitive_functions: + _record_surface(surfaces, f"authority-reference:{qualified}", node) + _record_non_function_container(surfaces, node, parents) + continue + if isinstance(node, ast.Raise) and _raises_termination(node, aliases): + _record_surface(surfaces, "raise:SystemExit", node) + _record_non_function_container(surfaces, node, parents) + continue + if isinstance(node, ast.Subscript) and _contains_dynamic_attribute(node.value): + _record_surface(surfaces, "dynamic-attribute-subscript", node) + _record_non_function_container(surfaces, node, parents) + continue + if isinstance(node, ast.Attribute): + qualified = _qualified_name(node, aliases) + if node.attr in _AUTHORITY_ATTRIBUTES: + _record_surface(surfaces, f"attribute:{qualified}", node) + _record_non_function_container(surfaces, node, parents) + return surfaces + + +def _record_surface( + surfaces: Counter[tuple[str, str]], + label: str, + node: ast.AST, +) -> None: + fingerprint = ast.dump(node, include_attributes=False) + surfaces[(label, fingerprint)] += 1 + + +def _parent_nodes(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} + + +def _record_non_function_container( + surfaces: Counter[tuple[str, str]], + node: ast.AST, + parents: dict[ast.AST, ast.AST], +) -> None: + """Bind module/class authority to its containing statement for integrity.""" + current = node + while current in parents: + parent = parents[current] + if isinstance(parent, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + return + if isinstance(parent, ast.Module): + _record_surface(surfaces, "authority-container:module", current) + return + current = parent + + +def _import_aliases(tree: ast.AST) -> dict[str, str]: + aliases: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + bound = alias.asname or alias.name.split(".", maxsplit=1)[0] + aliases[bound] = alias.name + elif isinstance(node, ast.ImportFrom) and node.module: + for alias in node.names: + if alias.name == "*": + continue + aliases[alias.asname or alias.name] = f"{node.module}.{alias.name}" + assignments: list[tuple[str, ast.AST]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + assignments.append((target.id, node.value)) + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if node.value is not None: + assignments.append((node.target.id, node.value)) + for _ in range(len(assignments) + 1): + changed = False + for target, value in assignments: + if isinstance(value, (ast.Name, ast.Attribute)): + resolved = _qualified_name(value, aliases) + elif isinstance(value, (ast.Call, ast.Lambda)): + resolved = f"__dynamic_callable__.{target}" + else: + continue + if resolved and aliases.get(target) != resolved: + aliases[target] = resolved + changed = True + if not changed: + break + return aliases + + +def _sensitive_import(node: ast.Import | ast.ImportFrom) -> bool: + if isinstance(node, ast.Import): + return any( + alias.name.split(".", maxsplit=1)[0] in _AUTHORITY_MODULES for alias in node.names + ) + if not node.module: + return False + return node.module.split(".", maxsplit=1)[0] in _AUTHORITY_MODULES + + +def _qualified_name(node: ast.AST, aliases: dict[str, str]) -> str: + if isinstance(node, ast.Name): + return aliases.get(node.id, node.id) + if isinstance(node, ast.Attribute): + prefix = _qualified_name(node.value, aliases) + return f"{prefix}.{node.attr}" if prefix else node.attr + return "" + + +def _raises_termination(node: ast.Raise, aliases: dict[str, str]) -> bool: + exception = node.exc + if isinstance(exception, ast.Call): + exception = exception.func + qualified = _qualified_name(exception, aliases) if exception is not None else "" + return qualified.rsplit(".", maxsplit=1)[-1] in _TERMINATION_EXCEPTIONS + + +def _contains_dynamic_attribute(node: ast.AST) -> bool: + return any( + (isinstance(item, ast.Attribute) and item.attr in _AUTHORITY_ATTRIBUTES) + or (isinstance(item, ast.Name) and item.id in _DYNAMIC_AUTHORITY_NAMES) + or ( + isinstance(item, ast.Constant) + and isinstance(item.value, str) + and item.value in (_DYNAMIC_AUTHORITY_NAMES | _AUTHORITY_ATTRIBUTES) + ) + for item in ast.walk(node) + ) + + +def _transitively_sensitive_functions( + tree: ast.AST, + aliases: dict[str, str], +) -> frozenset[str]: + """Find local functions that directly or transitively exercise authority.""" + functions = [ + node for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + direct: set[str] = set() + calls: dict[str, set[str]] = {} + references: dict[str, set[str]] = {} + for function in functions: + function_calls: set[str] = set() + function_references: set[str] = set() + for node in _owned_function_nodes(function): + if _direct_authority_node(node, aliases): + direct.add(function.name) + if isinstance(node, ast.Call): + qualified = _qualified_name(node.func, aliases) + function_calls.add(qualified.rsplit(".", maxsplit=1)[-1]) + elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + qualified = aliases.get(node.id, node.id) + function_references.add(qualified.rsplit(".", maxsplit=1)[-1]) + calls.setdefault(function.name, set()).update(function_calls) + references.setdefault(function.name, set()).update(function_references) + + sensitive = set(direct) + changed = True + while changed: + changed = False + for function_name, callees in calls.items(): + if function_name not in sensitive and ( + callees & sensitive or references.get(function_name, set()) & sensitive + ): + sensitive.add(function_name) + changed = True + return frozenset(sensitive) + + +def _owned_function_nodes( + function: ast.FunctionDef | ast.AsyncFunctionDef, +): + """Walk a function body without attributing nested definitions to it.""" + pending: list[ast.AST] = list(function.body) + while pending: + node = pending.pop() + yield node + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + pending.extend(ast.iter_child_nodes(node)) + + +def _direct_authority_node(node: ast.AST, aliases: dict[str, str]) -> bool: + if isinstance(node, (ast.Import, ast.ImportFrom)): + return _sensitive_import(node) + if isinstance(node, ast.Call): + return _authority_call(node.func, aliases, frozenset()) + if isinstance(node, ast.Raise): + return _raises_termination(node, aliases) + if isinstance(node, ast.Subscript): + return _contains_dynamic_attribute(node.value) + return isinstance(node, ast.Attribute) and node.attr in _AUTHORITY_ATTRIBUTES + + +def _authority_call( + function: ast.AST, + aliases: dict[str, str], + sensitive_functions: frozenset[str], +) -> bool: + qualified = _qualified_name(function, aliases) + terminal = qualified.rsplit(".", maxsplit=1)[-1] + root = qualified.split(".", maxsplit=1)[0] + return ( + terminal in _AUTHORITY_CALL_NAMES + or terminal in sensitive_functions + or root in _AUTHORITY_MODULES + or root == "__dynamic_callable__" + or not isinstance(function, (ast.Name, ast.Attribute)) + ) + + +def _changed_lines(before: str, after: str) -> int: + matcher = SequenceMatcher(a=before.splitlines(), b=after.splitlines(), autojunk=False) + total = 0 + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag != "equal": + total += max(i2 - i1, j2 - j1) + return total + + +def evaluation_to_dict(evaluation: UtilityEvaluation) -> dict[str, Any]: + """Return a stable JSON-ready evidence payload.""" + return { + "safe": evaluation.safe, + "safety_reasons": list(evaluation.safety_reasons), + "before_metrics": dict(evaluation.before_metrics), + "after_metrics": dict(evaluation.after_metrics), + "improved": evaluation.improvement.improved, + "improvement_reason": evaluation.improvement.reason, + "before_capabilities": sorted(evaluation.before_capabilities), + "after_capabilities": sorted(evaluation.after_capabilities), + "changed_lines": evaluation.changed_lines, + } diff --git a/code_covenant/piranha/execution.py b/code_covenant/piranha/execution.py new file mode 100644 index 0000000..21ad165 --- /dev/null +++ b/code_covenant/piranha/execution.py @@ -0,0 +1,739 @@ +""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: code_covenant.piranha.execution ║ +# ║ purpose: Run untrusted Piranha subprocesses behind a ║ +# ║ fail-closed operating-system confinement ║ +# ║ boundary with a scrubbed environment. ║ +# ║ inputs: ║ +# ║ - argv, cwd, timeout, ConfinementPolicy ║ +# ║ outputs: ║ +# ║ - subprocess.CompletedProcess ║ +# ║ constraints: ║ +# ║ - No supported backend means no execution ║ +# ║ - Network is denied unless explicitly enabled ║ +# ║ - Writes are limited to one declared run directory ║ +# ║ invariants: ║ +# ║ - The command receives no ambient credential environment ║ +# ║ side_effects: ║ +# ║ - Spawns and supervises a confined process group ║ +# ║ forbidden_changes: ║ +# ║ - Do not add a silent unconfined fallback ║ +# ║ optimization_targets: ║ +# ║ - [security] least-authority subprocess execution ║ +# ║ risk_level: critical ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: piranha ║ +# ║ super_cohort: code_covenant_core ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +from __future__ import annotations + +import json +import os +import resource +import secrets +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Sequence + + +class ConfinementError(RuntimeError): + """Raised when an untrusted process cannot be safely confined.""" + + +@dataclass(frozen=True) +class ConfinementPolicy: + """Least-authority policy for one generator or evaluator subprocess.""" + + read_roots: tuple[Path, ...] + write_root: Path + allow_network: bool = False + extra_env: Mapping[str, str] = field(default_factory=dict) + # CPython on macOS maps a large virtual address space before user code; a + # 1 GiB RLIMIT_AS can abort the interpreter even when resident memory is + # tiny. Four GiB remains bounded without producing that false failure. + max_memory_mb: int = 1024 + max_file_bytes: int = 32 * 1024 * 1024 + max_total_write_bytes: int = 64 * 1024 * 1024 + max_write_files: int = 1024 + max_open_files: int = 128 + max_output_bytes: int = 2 * 1024 * 1024 + require_completion_receipt: bool = False + + +@dataclass(frozen=True) +class _CompletionReceipt: + control_dir: Path + path: Path + token: str + runner: Path + + +def run_confined( + argv: Sequence[str], + *, + cwd: Path, + timeout_s: float, + policy: ConfinementPolicy, +) -> subprocess.CompletedProcess[str]: + """Run argv under the supported OS backend or fail without launching it.""" + if not argv: + raise ConfinementError("refusing to execute an empty command") + _validate_policy(policy, timeout_s) + require_confinement_backend() + + cwd = _safe_existing_directory(cwd, "cwd") + write_root = _safe_write_root(policy.write_root) + requested_executable = _locate_executable(argv[0]) + executable, launcher_env = _launch_executable(requested_executable) + read_roots = _read_roots( + policy.read_roots, + cwd, + executable, + requested_executable=requested_executable, + ) + if policy.require_completion_receipt and not _is_python_executable( + requested_executable, executable + ): + raise ConfinementError("completion receipts require an explicit Python test command") + receipt = _prepare_completion_receipt(write_root) if policy.require_completion_receipt else None + try: + profile = _sandbox_profile( + read_roots=read_roots, + metadata_read_paths=(cwd.parent,), + write_root=write_root, + allow_network=policy.allow_network, + executable=executable, + receipt=receipt, + ) + payload_args = [str(item) for item in argv[1:]] + if receipt is not None: + payload_args.insert(0, str(receipt.runner)) + command = [ + "/usr/bin/sandbox-exec", + "-p", + profile, + str(executable), + *payload_args, + ] + internal_env = dict(launcher_env) + env = _sanitized_env(write_root, policy.extra_env, internal_env) + limiter = _resource_limiter(policy, timeout_s) + with ( + tempfile.TemporaryFile(mode="w+b", dir=write_root) as stdout_file, + tempfile.TemporaryFile(mode="w+b", dir=write_root) as stderr_file, + ): + try: + process = subprocess.Popen( # noqa: S603 - wrapped by sandbox-exec + command, + cwd=str(cwd), + env=env, + stdin=subprocess.DEVNULL, + stdout=stdout_file, + stderr=stderr_file, + text=False, + close_fds=True, + start_new_session=True, + preexec_fn=limiter, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ConfinementError(f"could not start confined process: {exc}") from exc + _supervise_process( + process, + command=command, + timeout_s=timeout_s, + write_root=write_root, + stdout_file=stdout_file, + stderr_file=stderr_file, + policy=policy, + ) + stdout = _read_bounded_output(stdout_file, policy.max_output_bytes) + remaining = policy.max_output_bytes - len(stdout.encode("utf-8")) + stderr = _read_bounded_output(stderr_file, max(0, remaining)) + if receipt is not None: + _verify_completion_receipt(receipt, stderr=stderr) + return subprocess.CompletedProcess( + args=command, + returncode=process.returncode, + stdout=stdout, + stderr=stderr, + ) + finally: + if receipt is not None: + _cleanup_completion_receipt(receipt) + + +def require_confinement_backend() -> None: + """Fail before campaign setup when no reviewed process backend exists.""" + if sys.platform != "darwin" or not Path("/usr/bin/sandbox-exec").is_file(): + raise ConfinementError( + "no supported Piranha confinement backend; run inside a dedicated " + "VM/container or on macOS with /usr/bin/sandbox-exec" + ) + + +def _validate_policy(policy: ConfinementPolicy, timeout_s: float) -> None: + if timeout_s <= 0: + raise ConfinementError("confinement timeout must be positive") + limits = { + "max_memory_mb": policy.max_memory_mb, + "max_file_bytes": policy.max_file_bytes, + "max_total_write_bytes": policy.max_total_write_bytes, + "max_write_files": policy.max_write_files, + "max_open_files": policy.max_open_files, + "max_output_bytes": policy.max_output_bytes, + } + invalid = [name for name, value in limits.items() if value <= 0] + if invalid: + raise ConfinementError("confinement limits must be positive: " + ", ".join(sorted(invalid))) + if policy.max_memory_mb < 64: + raise ConfinementError("max_memory_mb must be at least 64") + + +def _safe_existing_directory(path: Path, label: str) -> Path: + raw = path.expanduser().absolute() + if raw.is_symlink(): + raise ConfinementError(f"{label} must not be a symlink: {raw}") + resolved = raw.resolve() + if not resolved.is_dir(): + raise ConfinementError(f"{label} is not a directory: {resolved}") + return resolved + + +def _safe_write_root(path: Path) -> Path: + raw = path.expanduser().absolute() + if raw.exists() and raw.is_symlink(): + raise ConfinementError(f"write_root must not be a symlink: {raw}") + raw.mkdir(parents=True, exist_ok=True) + root = raw.resolve() + if root in {Path("/"), Path.home().resolve()}: + raise ConfinementError(f"unsafe write_root: {root}") + (root / "home").mkdir(exist_ok=True) + (root / "tmp").mkdir(exist_ok=True) + return root + + +def _locate_executable(value: str) -> Path: + candidate = Path(value).expanduser() + resolved = shutil.which(value) if not candidate.is_absolute() else str(candidate) + if not resolved: + raise FileNotFoundError(value) + path = Path(resolved).absolute() + if not path.is_file(): + raise FileNotFoundError(value) + return path + + +def _launch_executable(requested: Path) -> tuple[Path, dict[str, str]]: + """Avoid macOS framework launchers that posix_spawn a second binary.""" + resolved = requested.resolve() + if sys.platform != "darwin": + return resolved, {} + version_root = resolved.parent.parent + inner = version_root / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python" + if "Python.framework" in resolved.parts and inner.is_file(): + return inner.resolve(), {"__PYVENV_LAUNCHER__": str(requested)} + return resolved, {} + + +def _is_python_executable(requested: Path, launched: Path) -> bool: + return requested.name.lower().startswith("python") or "Python.framework" in launched.parts + + +def _read_roots( + declared: tuple[Path, ...], + cwd: Path, + executable: Path, + *, + requested_executable: Path, +) -> tuple[Path, ...]: + roots: set[Path] = {cwd, executable.parent, Path(sys.prefix).resolve()} + roots.add(Path(sys.base_prefix).resolve()) + roots.update(path.expanduser().resolve() for path in declared) + # A selected venv launcher resolves to the base interpreter, but its + # packages remain under the venv root. Treat that root as an explicit + # executable dependency only when the ordinary ``bin/python`` layout and + # pyvenv marker both prove it is a virtual environment. + requested_parent = requested_executable.parent + requested_venv = requested_parent.parent + if requested_parent.name == "bin" and (requested_venv / "pyvenv.cfg").is_file(): + roots.add(requested_venv.resolve()) + for system_root in ( + Path("/bin"), + Path("/sbin"), + Path("/usr"), + Path("/System"), + Path("/Library/Apple"), + Path("/opt/homebrew"), + Path("/dev"), + ): + if system_root.exists(): + roots.add(system_root.resolve()) + return tuple(sorted(roots, key=str)) + + +def _sandbox_profile( + *, + read_roots: tuple[Path, ...], + metadata_read_paths: tuple[Path, ...], + write_root: Path, + allow_network: bool, + executable: Path, + receipt: _CompletionReceipt | None, +) -> str: + read_rules = " ".join(f"(subpath {_sb(path)})" for path in read_roots) + metadata_rules = " ".join(f"(literal {_sb(path)})" for path in metadata_read_paths) + receipt_read = f"(subpath {_sb(receipt.control_dir)})" if receipt is not None else "" + receipt_write = f"(literal {_sb(receipt.path)})" if receipt is not None else "" + network_rule = "(allow network*)" if allow_network else "(deny network*)" + return " ".join( + ( + "(version 1)", + # Let the language runtime perform ordinary non-I/O operations, + # then take filesystem, network, and child-process authority away + # explicitly. This is materially narrower than cwd-only execution + # while remaining usable by CPython on supported macOS releases. + "(allow default)", + "(deny process-fork)", + "(deny process-exec)", + f"(allow process-exec (literal {_sb(executable)}))", + "(deny signal (target others))", + "(deny appleevent-send)", + "(deny authorization-right-obtain)", + "(deny darwin-notification-post)", + "(deny distributed-notification-post)", + "(deny dynamic-code-generation)", + "(deny generic-issue-extension)", + "(deny iokit*)", + "(deny ipc-posix*)", + "(deny job-creation)", + "(deny lsopen)", + "(deny mach-cross-domain-lookup)", + "(deny mach-issue-extension)", + "(deny mach-lookup)", + "(deny mach-per-user-lookup)", + "(deny mach-priv-host-port)", + "(deny mach-register)", + "(deny mach-task-name)", + "(deny mach-task-read)", + "(deny mach-task-special-port*)", + "(deny managed-preference-read)", + "(deny nvram*)", + "(deny process-info*)", + "(deny pseudo-tty)", + "(deny qtn-user)", + "(deny system-kext*)", + "(deny system-privilege)", + "(deny system-suspend-resume)", + "(deny user-preference*)", + "(deny file-clone)", + "(deny file-link)", + # Deny ambient user homes, mounted volumes, and neighboring temp + # runs. macOS/CPython needs assorted system reads at startup, so + # declared roots are specifically re-allowed instead of attempting + # an unusable global read deny. + "(deny file-read*)", + # Homebrew's macOS framework launcher reads the root directory + # itself while looking for pyvenv.cfg. A literal rule grants that + # one directory entry without granting any descendant path. + f'(allow file-read* (literal "/") {read_rules} ' + f"(subpath {_sb(write_root)}) {receipt_read})", + f"(allow file-read-metadata {metadata_rules})", + "(deny file-write*)", + '(allow file-write-data (literal "/dev/null"))', + f"(allow file-write* (subpath {_sb(write_root)}) {receipt_write})", + network_rule, + ) + ) + + +def _sb(path: Path) -> str: + return json.dumps(str(path)) + + +def _sanitized_env( + write_root: Path, + extra: Mapping[str, str], + internal: Mapping[str, str] | None = None, +) -> dict[str, str]: + env = { + "HOME": str(write_root / "home"), + "TMPDIR": str(write_root / "tmp"), + "PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin", + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONNOUSERSITE": "1", + } + reserved = { + "HOME", + "PATH", + "PYTHONDONTWRITEBYTECODE", + "PYTHONHOME", + "PYTHONNOUSERSITE", + "PYTHONPATH", + "TMPDIR", + "__PYVENV_LAUNCHER__", + } + for key, value in extra.items(): + if not key or "\x00" in key or "=" in key: + raise ConfinementError(f"invalid environment key: {key!r}") + if key in reserved or key.startswith(("DYLD_", "LD_", "PYTHON")): + raise ConfinementError(f"reserved environment key: {key}") + env[str(key)] = str(value) + env.update({str(key): str(value) for key, value in (internal or {}).items()}) + return env + + +def _prepare_completion_receipt(write_root: Path) -> _CompletionReceipt: + """Install a trusted wrapper that acknowledges regained test-runner control.""" + token = secrets.token_hex(32) + control_dir = write_root.parent / (f".piranha-control-{write_root.name}-{secrets.token_hex(8)}") + try: + control_dir.mkdir(mode=0o700) + except FileExistsError as exc: + raise ConfinementError( + f"completion control directory already exists: {control_dir}" + ) from exc + receipt_path = control_dir / "receipt.txt" + runner = control_dir / "runner.py" + runner_source = ( + "import os as _os\n" + "import runpy as _runpy\n" + "import sys as _sys\n" + f"_PATH = {str(receipt_path)!r}\n" + f"_TOKEN = {token!r}\n" + "def _emit():\n" + " flags = _os.O_WRONLY | _os.O_TRUNC\n" + " if hasattr(_os, 'O_NOFOLLOW'):\n" + " flags |= _os.O_NOFOLLOW\n" + " fd = _os.open(_PATH, flags, 0o600)\n" + " try:\n" + " _os.write(fd, _TOKEN.encode('ascii'))\n" + " finally:\n" + " _os.close(fd)\n" + "def _execute(args):\n" + " if not args:\n" + " raise RuntimeError('missing Python test target')\n" + " _sys.path.insert(0, _os.getcwd())\n" + " if args[0] == '-m' and len(args) >= 2:\n" + " if args[1] == 'pytest':\n" + " import pytest as _pytest\n" + " code = _pytest.main(args[2:])\n" + " if code:\n" + " raise SystemExit(code)\n" + " return\n" + " _sys.argv = [args[1], *args[2:]]\n" + " _runpy.run_module(args[1], run_name='__main__', alter_sys=True)\n" + " return\n" + " if args[0] == '-c' and len(args) >= 2:\n" + " _sys.argv = ['-c', *args[2:]]\n" + " exec(compile(args[1], '', 'exec'), {'__name__': '__main__'})\n" + " return\n" + " if args[0].startswith('-'):\n" + " raise RuntimeError('unsupported Python test-runner option')\n" + " _sys.argv = list(args)\n" + " _runpy.run_path(args[0], run_name='__main__')\n" + "try:\n" + " _execute(_sys.argv[1:])\n" + "except SystemExit as _exit:\n" + " if _exit.code not in (None, 0):\n" + " _emit()\n" + " raise\n" + "except BaseException:\n" + " _emit()\n" + " raise\n" + "else:\n" + " _emit()\n" + ) + try: + receipt_path.touch(mode=0o600, exist_ok=False) + runner.write_text(runner_source, encoding="utf-8") + runner.chmod(0o400) + control_dir.chmod(0o500) + except OSError as exc: + raise ConfinementError(f"could not prepare completion receipt: {exc}") from exc + return _CompletionReceipt( + control_dir=control_dir, + path=receipt_path, + token=token, + runner=runner, + ) + + +def _verify_completion_receipt(receipt: _CompletionReceipt, *, stderr: str = "") -> None: + try: + info = receipt.path.lstat() + except FileNotFoundError as exc: + diagnostic = stderr.strip()[-500:] + suffix = f"; runtime stderr: {diagnostic}" if diagnostic else "" + raise ConfinementError( + "runtime exited without the independent completion receipt" + suffix + ) from exc + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise ConfinementError("runtime completion receipt is not a single regular file") + try: + value = receipt.path.read_text(encoding="ascii") + except (OSError, UnicodeError) as exc: + raise ConfinementError(f"could not verify runtime completion receipt: {exc}") from exc + if value != receipt.token: + raise ConfinementError("runtime completion receipt did not authenticate") + + +def _cleanup_completion_receipt(receipt: _CompletionReceipt) -> None: + """Best-effort removal of the per-run control artifacts after supervision.""" + try: + receipt.control_dir.chmod(0o700) + except OSError: + return + for path in (receipt.runner, receipt.path): + try: + path.unlink() + except FileNotFoundError: + pass + except OSError: + return + try: + receipt.control_dir.rmdir() + except OSError: + pass + + +def _supervise_process( + process: subprocess.Popen[bytes], + *, + command: list[str], + timeout_s: float, + write_root: Path, + stdout_file, + stderr_file, + policy: ConfinementPolicy, +) -> None: + try: + _supervise_process_inner( + process, + command=command, + timeout_s=timeout_s, + write_root=write_root, + stdout_file=stdout_file, + stderr_file=stderr_file, + policy=policy, + ) + except BaseException: + if process.poll() is None: + _terminate_and_wait(process) + raise + + +def _supervise_process_inner( + process: subprocess.Popen[bytes], + *, + command: list[str], + timeout_s: float, + write_root: Path, + stdout_file, + stderr_file, + policy: ConfinementPolicy, +) -> None: + deadline = time.monotonic() + timeout_s + next_rss_check = 0.0 + while process.poll() is None: + now = time.monotonic() + if now >= deadline: + _terminate_and_wait(process) + output = _read_bounded_output(stdout_file, policy.max_output_bytes) + error = _read_bounded_output(stderr_file, policy.max_output_bytes) + raise subprocess.TimeoutExpired( + command, + timeout_s, + output=output, + stderr=error, + ) + output_bytes = ( + os.fstat(stdout_file.fileno()).st_size + os.fstat(stderr_file.fileno()).st_size + ) + if output_bytes > policy.max_output_bytes: + _terminate_and_wait(process) + raise ConfinementError(f"confined output exceeded {policy.max_output_bytes} bytes") + written_bytes, written_files = _tree_usage( + write_root, + max_bytes=policy.max_total_write_bytes, + max_entries=policy.max_write_files, + deadline=deadline, + ) + if written_bytes > policy.max_total_write_bytes: + _terminate_and_wait(process) + raise ConfinementError( + f"confined writable tree exceeded {policy.max_total_write_bytes} bytes" + ) + if written_files > policy.max_write_files: + _terminate_and_wait(process) + raise ConfinementError( + f"confined writable tree exceeded {policy.max_write_files} entries" + ) + if now >= next_rss_check: + rss = _rss_bytes(process.pid) + if rss is not None and rss > policy.max_memory_mb * 1024 * 1024: + _terminate_and_wait(process) + raise ConfinementError( + f"confined resident memory exceeded {policy.max_memory_mb} MiB" + ) + next_rss_check = now + 0.02 + time.sleep(0.02) + process.wait() + output_bytes = os.fstat(stdout_file.fileno()).st_size + os.fstat(stderr_file.fileno()).st_size + if output_bytes > policy.max_output_bytes: + raise ConfinementError(f"confined output exceeded {policy.max_output_bytes} bytes") + written_bytes, written_files = _tree_usage( + write_root, + max_bytes=policy.max_total_write_bytes, + max_entries=policy.max_write_files, + ) + if written_bytes > policy.max_total_write_bytes: + raise ConfinementError( + f"confined writable tree exceeded {policy.max_total_write_bytes} bytes" + ) + if written_files > policy.max_write_files: + raise ConfinementError(f"confined writable tree exceeded {policy.max_write_files} entries") + + +def _terminate_and_wait(process: subprocess.Popen[bytes]) -> None: + _kill_process_group(process.pid) + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def _read_bounded_output(stream, max_bytes: int) -> str: + stream.flush() + stream.seek(0) + data = stream.read(max_bytes + 1) + if len(data) > max_bytes: + raise ConfinementError(f"confined output exceeded {max_bytes} bytes") + return data.decode("utf-8", errors="replace") + + +def _tree_usage( + root: Path, + *, + max_bytes: int | None = None, + max_entries: int | None = None, + deadline: float | None = None, +) -> tuple[int, int]: + """Boundedly count bytes and entries without following symlinks.""" + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + total_bytes = 0 + total_entries = 0 + pending = [root] + while pending: + if deadline is not None and time.monotonic() >= deadline: + raise ConfinementError("writable-tree inspection reached the runtime deadline") + directory = pending.pop() + try: + directory_fd = os.open(directory, flags) + except FileNotFoundError: + continue + except OSError as exc: + raise ConfinementError(f"unsafe directory in confined write root: {directory}") from exc + try: + with os.scandir(directory_fd) as entries: + for entry in entries: + if deadline is not None and time.monotonic() >= deadline: + raise ConfinementError( + "writable-tree inspection reached the runtime deadline" + ) + try: + info = entry.stat(follow_symlinks=False) + except FileNotFoundError: + continue + total_entries += 1 + if stat.S_ISDIR(info.st_mode): + pending.append(directory / entry.name) + elif stat.S_ISREG(info.st_mode): + total_bytes += info.st_size + if max_entries is not None and total_entries > max_entries: + return total_bytes, total_entries + if max_bytes is not None and total_bytes > max_bytes: + return total_bytes, total_entries + except ConfinementError: + raise + except OSError as exc: + raise ConfinementError(f"could not inspect confined writable tree: {exc}") from exc + finally: + os.close(directory_fd) + return total_bytes, total_entries + + +def _rss_bytes(pid: int) -> int | None: + """Read resident memory from the reviewed macOS system process table.""" + try: + completed = subprocess.run( # noqa: S603 + ["/bin/ps", "-o", "rss=", "-p", str(pid)], + capture_output=True, + text=True, + timeout=0.5, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ConfinementError(f"could not supervise confined resident memory: {exc}") from exc + if completed.returncode != 0: + return None + value = completed.stdout.strip() + if not value: + return None + try: + return int(value.splitlines()[-1].strip()) * 1024 + except ValueError as exc: + raise ConfinementError("could not parse confined resident memory") from exc + + +def _resource_limiter(policy: ConfinementPolicy, timeout_s: float): + def _limit() -> None: + memory = max(64, policy.max_memory_mb) * 1024 * 1024 + file_size = max(1024, policy.max_file_bytes) + open_files = max(16, policy.max_open_files) + cpu_seconds = max(1, int(timeout_s) + 1) + for name, value in ( + ("RLIMIT_AS", memory), + ("RLIMIT_DATA", memory), + ("RLIMIT_RSS", memory), + ("RLIMIT_FSIZE", min(file_size, max(1024, policy.max_output_bytes))), + ("RLIMIT_NOFILE", open_files), + ("RLIMIT_CPU", cpu_seconds), + ): + if sys.platform == "darwin" and name in { + "RLIMIT_AS", + "RLIMIT_DATA", + "RLIMIT_RSS", + }: + # macOS either accounts CPython's large reserved VM regions or + # rejects these three memory-limit updates. Resident memory is + # instead sampled and enforced by the supervising parent. + continue + limit = getattr(resource, name, None) + if limit is None: + continue + resource.setrlimit(limit, (value, value)) + + return _limit + + +def _kill_process_group(pid: int) -> None: + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass diff --git a/code_covenant/piranha/lab.py b/code_covenant/piranha/lab.py index 86e0117..c16af89 100644 --- a/code_covenant/piranha/lab.py +++ b/code_covenant/piranha/lab.py @@ -69,6 +69,8 @@ from code_covenant.portfolio.runner import PortfolioConfig, run_portfolio from code_covenant.portfolio.scope import ScopeFilter +from .policy import RISK_ORDER, UtilityPolicy + @dataclass class LabConfig: @@ -89,6 +91,9 @@ class LabConfig: allow_repo_root: bool = False require_metric_improvement: bool = False minimize_metrics: frozenset[str] = field(default_factory=frozenset) + utility_policy: UtilityPolicy | None = None + require_process_confinement: bool = True + allow_engine_network: bool = False @dataclass @@ -111,9 +116,23 @@ class LabResult: def run_lab(config: LabConfig) -> LabResult: """Execute every lab round against a freshly-materialised sandbox.""" if not config.confirm_sandbox_destructive: - raise SandboxError( - "Piranha Lab refused to run: set confirm_sandbox_destructive=True" + raise SandboxError("Piranha Lab refused to run: set confirm_sandbox_destructive=True") + if ( + config.utility_policy is not None + and config.utility_policy.require_tests + and not config.test_command + ): + raise SandboxError("Piranha utility mode requires an immutable --test-command") + if _effective_confinement(config): + from code_covenant.piranha.execution import ( + ConfinementError, + require_confinement_backend, ) + + try: + require_confinement_backend() + except ConfinementError as exc: + raise SandboxError(str(exc)) from exc layout = materialize_sandbox( SandboxConfig( source_root=config.source_root, @@ -136,9 +155,14 @@ def run_lab(config: LabConfig) -> LabResult: total_attempts = len(ledger) total_merges = sum(1 for row in ledger if row.get("merged")) report_path = _write_report( - layout, config, champions, lineage, - started=started, ended=ended, - total_attempts=total_attempts, total_merges=total_merges, + layout, + config, + champions, + lineage, + started=started, + ended=ended, + total_attempts=total_attempts, + total_merges=total_merges, ) return LabResult( layout=layout, @@ -170,7 +194,7 @@ def _run_round( duration_s=config.duration_s, max_attempts=config.max_attempts_per_round, merge_mode="auto", - scope=config.scope, + scope=_effective_scope(config), test_command=config.test_command, benchmark_command=config.benchmark_command, require_metric_improvement=config.require_metric_improvement, @@ -178,6 +202,10 @@ def _run_round( gate_timeout_s=config.gate_timeout_s, engine_timeout_s=config.engine_timeout_s, benchmark_history_path=layout.benchmark_history_path, + project_cwd=layout.src, + utility_policy=config.utility_policy, + require_process_confinement=_effective_confinement(config), + allow_engine_network=config.allow_engine_network, ) validate_contained(portfolio_config.output_dir, layout.root) result = run_portfolio(portfolio_config) @@ -212,6 +240,10 @@ def _write_report( f"- **Rounds:** {config.rounds}", f"- **Attempts:** {total_attempts}", f"- **Merges:** {total_merges}", + f"- **Utility policy:** {'enabled' if config.utility_policy else 'legacy'}", + f"- **Utility max risk:** {_utility_policy_value(config, 'max_risk')}", + f"- **Required utility metrics:** {_required_metrics(config)}", + f"- **Process confinement:** {_effective_confinement(config)}", "", "## Champion state", f"- targets tracked: {summary.get('targets', 0)}", @@ -255,3 +287,41 @@ def _per_target_lines(per_target: dict[str, int]) -> list[str]: def _iso_now() -> str: return datetime.now(UTC).isoformat() + + +def _effective_confinement(config: LabConfig) -> bool: + return config.require_process_confinement or bool( + config.utility_policy and config.utility_policy.require_process_confinement + ) + + +def _utility_policy_value(config: LabConfig, name: str) -> str: + if config.utility_policy is None: + return "n/a" + return str(getattr(config.utility_policy, name)) + + +def _required_metrics(config: LabConfig) -> str: + if config.utility_policy is None: + return "n/a" + return ", ".join(sorted(config.utility_policy.required_metrics)) + + +def _effective_scope(config: LabConfig) -> ScopeFilter | None: + if config.utility_policy is None: + return config.scope + current = config.scope or ScopeFilter() + requested_max = current.max_risk + if requested_max is not None and requested_max not in RISK_ORDER: + raise SandboxError(f"unknown maximum risk: {requested_max!r}") + max_risk = config.utility_policy.max_risk + if requested_max is not None and RISK_ORDER[requested_max] < RISK_ORDER[max_risk]: + max_risk = requested_max + return ScopeFilter( + modules=list(current.modules), + cohorts=list(current.cohorts), + super_cohorts=list(current.super_cohorts), + authorities=list(current.authorities), + max_risk=max_risk, + min_risk=current.min_risk, + ) diff --git a/code_covenant/piranha/policy.py b/code_covenant/piranha/policy.py new file mode 100644 index 0000000..abd7a0d --- /dev/null +++ b/code_covenant/piranha/policy.py @@ -0,0 +1,88 @@ +""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: code_covenant.piranha.policy ║ +# ║ purpose: Declare the immutable safety and utility gate ║ +# ║ used by beneficial-mutation campaigns. ║ +# ║ inputs: ║ +# ║ - operator-selected policy fields ║ +# ║ outputs: ║ +# ║ - UtilityPolicy ║ +# ║ constraints: ║ +# ║ - Defaults are fail-closed and require behavior tests ║ +# ║ invariants: ║ +# ║ - Required built-in metrics are always explicitly named ║ +# ║ side_effects: ║ +# ║ - None ║ +# ║ forbidden_changes: ║ +# ║ - Do not make security gates model-configurable ║ +# ║ optimization_targets: ║ +# ║ - [clarity] one serialisable promotion policy ║ +# ║ risk_level: critical ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: piranha ║ +# ║ super_cohort: code_covenant_core ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Mapping + + +BUILTIN_UTILITY_METRICS: frozenset[str] = frozenset( + {"source_bytes", "ast_nodes", "branch_nodes", "bytecode_instructions"} +) +RISK_ORDER: Mapping[str, int] = MappingProxyType({"low": 0, "medium": 1, "high": 2, "critical": 3}) + + +@dataclass(frozen=True) +class UtilityPolicy: + """External policy that a mutation engine can neither edit nor bypass.""" + + required_metrics: frozenset[str] = field( + default_factory=lambda: frozenset({"source_bytes", "branch_nodes", "bytecode_instructions"}) + ) + minimize_metrics: frozenset[str] = field( + default_factory=lambda: frozenset(BUILTIN_UTILITY_METRICS) + ) + require_tests: bool = True + require_process_confinement: bool = True + preserve_cpc_bytes: bool = True + preserve_public_api: bool = True + forbid_capability_expansion: bool = True + max_changed_lines: int = 80 + max_candidate_bytes: int = 2 * 1024 * 1024 + max_size_ratio: float = 1.25 + max_risk: str = "medium" + + def __post_init__(self) -> None: + unknown = self.required_metrics - BUILTIN_UTILITY_METRICS + if unknown: + raise ValueError(f"unknown built-in utility metrics: {sorted(unknown)}") + unknown_minimize = self.minimize_metrics - BUILTIN_UTILITY_METRICS + if unknown_minimize: + raise ValueError(f"unknown minimized utility metrics: {sorted(unknown_minimize)}") + if not self.required_metrics: + raise ValueError("utility policy requires at least one metric") + wrong_direction = self.required_metrics - self.minimize_metrics + if wrong_direction: + raise ValueError(f"built-in count metrics must be minimized: {sorted(wrong_direction)}") + if self.max_changed_lines < 1: + raise ValueError("max_changed_lines must be positive") + if self.max_candidate_bytes < 1: + raise ValueError("max_candidate_bytes must be positive") + if self.max_size_ratio < 1.0: + raise ValueError("max_size_ratio must be at least 1.0") + if self.max_risk not in RISK_ORDER: + raise ValueError(f"unknown utility max_risk: {self.max_risk!r}") + + def allows_risk(self, risk: str) -> bool: + """Return whether a CPC risk is within the immutable campaign ceiling.""" + rank = RISK_ORDER.get(risk, RISK_ORDER["critical"]) + return rank <= RISK_ORDER[self.max_risk] diff --git a/code_covenant/piranha/sandbox.py b/code_covenant/piranha/sandbox.py index 8752a2d..4ce7015 100644 --- a/code_covenant/piranha/sandbox.py +++ b/code_covenant/piranha/sandbox.py @@ -20,12 +20,15 @@ # ║ - sandbox_root must not overlap source ║ # ║ - Every lab output path must be inside sandbox_root ║ # ║ - materialize_sandbox must copy files only; no symlinks ║ +# ║ - Existing roots require the exact Piranha owner marker ║ +# ║ - Source entries must be regular files or directories ║ # ║ - Must refuse to run if the source looks like the repo ║ # ║ root (presence of .git) unless explicitly allowed ║ # ║ ║ # ║ invariants: ║ # ║ - SandboxLayout.src is a child of SandboxLayout.root ║ # ║ - All helper paths validate via validate_contained ║ +# ║ - A materialised root carries the Piranha owner marker ║ # ║ ║ # ║ side_effects: ║ # ║ - Creates the sandbox tree on disk ║ @@ -34,6 +37,7 @@ # ║ forbidden_changes: ║ # ║ - Do not follow symlinks when copying ║ # ║ - Do not return layouts whose paths escape sandbox_root ║ +# ║ - Do not delete an existing unowned root ║ # ║ ║ # ║ optimization_targets: ║ # ║ - [robustness] strict path containment ║ @@ -50,11 +54,19 @@ from __future__ import annotations +import os import shutil +import stat +import tempfile from dataclasses import dataclass from pathlib import Path +_OWNERSHIP_MARKER = ".piranha-owned-v1" +_OWNERSHIP_MARKER_CONTENT = "code-covenant:piranha-sandbox:v1\n" +_NOISE_NAMES = frozenset({".git", "__pycache__", ".venv", "venv", ".pytest_cache", "node_modules"}) + + class SandboxError(RuntimeError): """Raised when a Piranha Lab operation would escape the sandbox.""" @@ -84,12 +96,17 @@ class SandboxLayout: def materialize_sandbox(config: SandboxConfig) -> SandboxLayout: """Create an isolated sandbox copy of source_root and return its layout.""" - source = config.source_root.resolve() - root = config.sandbox_root.resolve() + source_input = config.source_root.expanduser().absolute() + root_input = config.sandbox_root.expanduser().absolute() + if source_input.is_symlink(): + raise SandboxError(f"source_root must not be a symlink: {source_input}") + if root_input.is_symlink(): + raise SandboxError(f"sandbox_root must not be a symlink: {root_input}") + source = source_input.resolve() + root = root_input.resolve() _validate_roots(source, root, allow_repo_root=config.allow_repo_root) - if root.exists(): - shutil.rmtree(root) - root.mkdir(parents=True, exist_ok=True) + _validate_source_tree(source) + _prepare_owned_root(root) layout = _build_layout(root) _copy_source(source, layout.src) layout.proposals_dir.mkdir(parents=True, exist_ok=True) @@ -104,27 +121,21 @@ def validate_contained(path: Path, sandbox_root: Path) -> None: try: resolved.relative_to(root_resolved) except ValueError as exc: - raise SandboxError( - f"path {resolved} escapes sandbox root {root_resolved}" - ) from exc + raise SandboxError(f"path {resolved} escapes sandbox root {root_resolved}") from exc -def _validate_roots( - source: Path, sandbox: Path, *, allow_repo_root: bool -) -> None: +def _validate_roots(source: Path, sandbox: Path, *, allow_repo_root: bool) -> None: if not source.exists() or not source.is_dir(): raise SandboxError(f"source is not a directory: {source}") if sandbox == source: raise SandboxError("sandbox_root must differ from source_root") if _is_ancestor(source, sandbox): - raise SandboxError( - f"sandbox_root {sandbox} is inside source {source}" - ) + raise SandboxError(f"sandbox_root {sandbox} is inside source {source}") if _is_ancestor(sandbox, source): - raise SandboxError( - f"source {source} is inside sandbox_root {sandbox}" - ) - if not allow_repo_root and (source / ".git").is_dir(): + raise SandboxError(f"source {source} is inside sandbox_root {sandbox}") + _reject_dangerous_root(sandbox) + git_marker = source / ".git" + if not allow_repo_root and (git_marker.exists() or git_marker.is_symlink()): raise SandboxError( "source appears to be a git repo root; pass allow_repo_root=True " "to confirm you really want to materialise it." @@ -155,9 +166,90 @@ def _build_layout(root: Path) -> SandboxLayout: def _copy_source(source: Path, destination: Path) -> None: if destination.exists(): shutil.rmtree(destination) - shutil.copytree(source, destination, symlinks=False, ignore=_ignore_noise) + # Preserve (rather than follow) a link introduced by a source-tree race, then + # reject it in the copied tree. ``symlinks=False`` would dereference it and + # could copy data from outside source_root. + shutil.copytree(source, destination, symlinks=True, ignore=_ignore_noise) + _validate_source_tree(destination) def _ignore_noise(directory: str, names: list[str]) -> list[str]: - skip = {".git", "__pycache__", ".venv", "venv", ".pytest_cache", "node_modules"} - return [name for name in names if name in skip] + return [name for name in names if name in _NOISE_NAMES] + + +def _prepare_owned_root(root: Path) -> None: + """Create root, destructively refreshing it only when Piranha owns it.""" + if root.exists(): + if not root.is_dir(): + raise SandboxError(f"sandbox_root is not a directory: {root}") + marker = root / _OWNERSHIP_MARKER + if marker.is_symlink() or not marker.is_file(): + raise SandboxError(f"refusing to replace existing unowned sandbox_root: {root}") + try: + marker_content = marker.read_text(encoding="utf-8") + except OSError as exc: + raise SandboxError(f"cannot verify sandbox ownership marker: {marker}") from exc + if marker_content != _OWNERSHIP_MARKER_CONTENT: + raise SandboxError(f"refusing sandbox_root with invalid ownership marker: {root}") + _make_owned_tree_removable(root) + shutil.rmtree(root) + root.mkdir(parents=True, exist_ok=False) + (root / _OWNERSHIP_MARKER).write_text(_OWNERSHIP_MARKER_CONTENT, encoding="utf-8") + + +def _make_owned_tree_removable(root: Path) -> None: + """Restore owner access to directories in a verified Piranha-owned tree.""" + pending = [root] + while pending: + directory = pending.pop() + try: + directory.chmod(stat.S_IRWXU) + entries = list(os.scandir(directory)) + except OSError as exc: + raise SandboxError(f"cannot prepare owned sandbox for refresh: {directory}") from exc + for entry in entries: + try: + if entry.is_dir(follow_symlinks=False): + pending.append(Path(entry.path)) + except OSError as exc: + raise SandboxError(f"cannot inspect owned sandbox entry: {entry.path}") from exc + + +def _reject_dangerous_root(root: Path) -> None: + protected = { + Path(root.anchor).resolve(), + Path.home().resolve(), + Path.cwd().resolve(), + Path(tempfile.gettempdir()).resolve(), + Path("/tmp").resolve(), + Path("/var/tmp").resolve(), + Path("/private/tmp").resolve(), + } + for protected_root in protected: + if root == protected_root or _is_ancestor(root, protected_root): + raise SandboxError(f"refusing dangerous sandbox_root: {root}") + + +def _validate_source_tree(source: Path) -> None: + """Accept only real directories and regular files in the copied tree.""" + pending = [source] + while pending: + directory = pending.pop() + try: + entries = list(os.scandir(directory)) + except OSError as exc: + raise SandboxError(f"cannot inspect source directory: {directory}") from exc + for entry in entries: + path = Path(entry.path) + try: + mode = entry.stat(follow_symlinks=False).st_mode + except OSError as exc: + raise SandboxError(f"cannot inspect source entry: {path}") from exc + if stat.S_ISLNK(mode): + raise SandboxError(f"source contains symlink: {path}") + if stat.S_ISDIR(mode): + if entry.name not in _NOISE_NAMES: + pending.append(path) + continue + if not stat.S_ISREG(mode): + raise SandboxError(f"source contains special file: {path}") diff --git a/code_covenant/piranha/utility_engine.py b/code_covenant/piranha/utility_engine.py new file mode 100644 index 0000000..226df9d --- /dev/null +++ b/code_covenant/piranha/utility_engine.py @@ -0,0 +1,784 @@ +""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: code_covenant.piranha.utility_engine ║ +# ║ purpose: Propose deterministic, utility-oriented Python║ +# ║ mutations from trusted AST patterns while ║ +# ║ preserving governed source headers exactly. ║ +# ║ ║ +# ║ inputs: ║ +# ║ - EngineContext containing Python source and feedback ║ +# ║ ║ +# ║ outputs: ║ +# ║ - EngineResult containing one full-file replacement ║ +# ║ - MutationCandidate metadata for the selected change ║ +# ║ ║ +# ║ constraints: ║ +# ║ - Must derive mutations only from conservative AST shapes ║ +# ║ - Must preserve every CPC block byte for byte ║ +# ║ - Must not execute target code or launch subprocesses ║ +# ║ - Must not erase comments within a changed line range ║ +# ║ ║ +# ║ invariants: ║ +# ║ - Identical source and feedback produce identical metadata║ +# ║ - Every successful result parses as Python ║ +# ║ ║ +# ║ side_effects: ║ +# ║ - None ║ +# ║ ║ +# ║ forbidden_changes: ║ +# ║ - Do not mutate files directly ║ +# ║ - Do not bypass Code Covenant proposal gates ║ +# ║ ║ +# ║ optimization_targets: ║ +# ║ - [utility] reduce incidental control-flow complexity ║ +# ║ - [security] keep proposal generation non-executing ║ +# ║ - [determinism] emit stable hashes and strategy labels ║ +# ║ ║ +# ║ risk_level: high ║ +# ║ authority: draft ║ +# ║ source_basis: implementation_draft ║ +# ║ language: python ║ +# ║ cohort: piranha ║ +# ║ super_cohort: code_covenant_core ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +from __future__ import annotations + +import ast +import hashlib +import io +import json +import time +import tokenize +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, cast + +from code_covenant.contracts.parser import find_cpc_blocks +from code_covenant.portfolio.engines import EngineContext, EngineResult + +BRANCH_TO_BOOL = "branch-to-bool-expression" +GUARD_RETURN_TO_CONDITIONAL = "guard-return-to-conditional" +LOOP_TO_ANY = "loop-to-any" +LIST_ACCUMULATOR_TO_COMPREHENSION = "list-accumulator-to-comprehension" + +_STRATEGY_ORDER = { + BRANCH_TO_BOOL: 0, + GUARD_RETURN_TO_CONDITIONAL: 1, + LOOP_TO_ANY: 2, + LIST_ACCUMULATOR_TO_COMPREHENSION: 3, +} +_SCOPE_SENSITIVE_NODES = ( + ast.Await, + ast.Lambda, + ast.NamedExpr, + ast.Starred, + ast.Yield, + ast.YieldFrom, +) +_SCOPE_OBSERVERS = frozenset({"dir", "eval", "exec", "globals", "locals", "super", "vars"}) + + +@dataclass(frozen=True, slots=True) +class MutationCandidate: + """One auditable whole-file proposal derived from a bounded source patch.""" + + strategy: str + candidate_hash: str + source_hash: str + line_start: int + line_end: int + replacement: str + after_source: str + novelty_tags: tuple[str, ...] + estimated_line_reduction: int + + +@dataclass(frozen=True, slots=True) +class _Patch: + strategy: str + line_start: int + line_end: int + code: str + novelty_tags: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class UtilityMutationEngine: + """Trusted deterministic engine for a small set of semantics-aware rewrites.""" + + name: str = "piranha-utility" + + def candidates(self, ctx: EngineContext) -> tuple[MutationCandidate, ...]: + """Return unseen candidates in deterministic selection order.""" + try: + tree = ast.parse(ctx.before_source, filename=str(ctx.target_path)) + except SyntaxError: + return () + return _discover_candidates( + ctx.before_source, + tree, + seen_hashes=_seen_candidate_hashes(ctx), + ) + + def propose(self, ctx: EngineContext) -> EngineResult: + """Select the first unseen safe mutation and return its full source text.""" + started = time.perf_counter() + try: + tree = ast.parse(ctx.before_source, filename=str(ctx.target_path)) + except SyntaxError as exc: + return EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - started, + error=f"source is not valid Python: {exc.msg} (line {exc.lineno})", + ) + + try: + candidates = _discover_candidates( + ctx.before_source, + tree, + seen_hashes=_seen_candidate_hashes(ctx), + ) + except Exception as exc: # noqa: BLE001 - Engine failures are result data + return EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - started, + error=f"utility mutation discovery failed: {exc!r}", + ) + if not candidates: + return EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - started, + error="no unseen utility-preserving mutation candidate found", + ) + + candidate = candidates[0] + metadata: dict[str, Any] = { + "candidate_hash": candidate.candidate_hash, + "source_hash": candidate.source_hash, + "strategy": candidate.strategy, + "novelty_tags": list(candidate.novelty_tags), + "target_lines": [candidate.line_start, candidate.line_end], + "estimated_line_reduction": candidate.estimated_line_reduction, + } + return EngineResult( + ok=True, + after_source=candidate.after_source, + duration_s=time.perf_counter() - started, + prompt_id=f"{self.name}:{candidate.candidate_hash[:16]}", + extra=metadata, + ) + + +def _discover_candidates( + source: str, + tree: ast.Module, + *, + seen_hashes: frozenset[str], +) -> tuple[MutationCandidate, ...]: + patches: list[_Patch] = [] + parents = _parent_nodes(tree) + patches.extend(_branch_patches(source, tree)) + for statements in _statement_lists(tree): + patches.extend(_adjacent_branch_patches(source, tree, statements)) + patches.extend(_guard_return_patches(source, statements)) + patches.extend(_loop_to_any_patches(source, tree, statements, parents)) + patches.extend(_list_comprehension_patches(source, tree, statements, parents)) + + candidates: dict[str, MutationCandidate] = {} + for patch in patches: + candidate = _candidate_from_patch(source, patch) + if candidate is None or candidate.candidate_hash in seen_hashes: + continue + candidates[candidate.candidate_hash] = candidate + return tuple( + sorted( + candidates.values(), + key=lambda candidate: ( + candidate.line_start, + _STRATEGY_ORDER[candidate.strategy], + candidate.line_end, + candidate.candidate_hash, + ), + ) + ) + + +def _branch_patches(source: str, tree: ast.Module) -> list[_Patch]: + if _binds_name(tree, "bool"): + return [] + patches: list[_Patch] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + if len(node.body) != 1 or len(node.orelse) != 1: + continue + truthy = _returned_bool(node.body[0]) + falsy = _returned_bool(node.orelse[0]) + if truthy is None or falsy is None or truthy == falsy: + continue + if not _safe_expression(node.test): + continue + line_start, line_end = _node_lines(node) + if _range_has_protected_tokens(source, line_start, line_end): + continue + test = ast.unparse(node.test) + code = _boolean_return_code(test, truthy) + patches.append( + _Patch( + strategy=BRANCH_TO_BOOL, + line_start=line_start, + line_end=line_end, + code=code, + novelty_tags=( + "ast-derived", + "boolean-normalization", + "control-flow-collapse", + "cpc-byte-preserving", + ), + ) + ) + return patches + + +def _adjacent_branch_patches( + source: str, + tree: ast.Module, + statements: list[ast.stmt], +) -> list[_Patch]: + """Collapse an if-return followed by its opposite Boolean fallback.""" + if _binds_name(tree, "bool"): + return [] + patches: list[_Patch] = [] + for index in range(len(statements) - 1): + branch = statements[index] + fallback = statements[index + 1] + if not isinstance(branch, ast.If) or branch.orelse or len(branch.body) != 1: + continue + truthy = _returned_bool(branch.body[0]) + falsy = _returned_bool(fallback) + if truthy is None or falsy is None or truthy == falsy: + continue + if not _safe_expression(branch.test): + continue + line_start = branch.lineno + line_end = _node_lines(fallback)[1] + if _range_has_protected_tokens(source, line_start, line_end): + continue + test = ast.unparse(branch.test) + code = _boolean_return_code(test, truthy) + patches.append( + _Patch( + strategy=BRANCH_TO_BOOL, + line_start=line_start, + line_end=line_end, + code=code, + novelty_tags=( + "ast-derived", + "boolean-normalization", + "guard-clause-collapse", + "cpc-byte-preserving", + ), + ) + ) + return patches + + +def _boolean_return_code(test: str, truthy: bool) -> str: + # `not` already normalizes arbitrary truthiness to an exact bool. Avoiding + # an extra bool() call makes the inverse form smaller in source and bytecode. + return f"return bool({test})" if truthy else f"return not ({test})" + + +def _guard_return_patches(source: str, statements: list[ast.stmt]) -> list[_Patch]: + """Collapse a simple guard return and its fallback into one conditional.""" + patches: list[_Patch] = [] + for index in range(len(statements) - 1): + branch = statements[index] + fallback = statements[index + 1] + if not isinstance(branch, ast.If) or branch.orelse or len(branch.body) != 1: + continue + guarded = branch.body[0] + if not isinstance(guarded, ast.Return) or not isinstance(fallback, ast.Return): + continue + if guarded.value is None or fallback.value is None: + continue + guarded_bool = _returned_bool(guarded) + fallback_bool = _returned_bool(fallback) + if guarded_bool is not None and fallback_bool is not None: + continue + if not all( + _safe_expression(expression) + for expression in (branch.test, guarded.value, fallback.value) + ): + continue + line_start = branch.lineno + line_end = _node_lines(fallback)[1] + if _range_has_protected_tokens(source, line_start, line_end): + continue + conditional = ast.IfExp( + test=branch.test, + body=guarded.value, + orelse=fallback.value, + ) + code = ast.unparse(ast.Return(value=conditional)) + if not _replacement_fits(source, line_start, code): + continue + patches.append( + _Patch( + strategy=GUARD_RETURN_TO_CONDITIONAL, + line_start=line_start, + line_end=line_end, + code=code, + novelty_tags=( + "ast-derived", + "guard-clause-collapse", + "lazy-arm-preserving", + "cpc-byte-preserving", + ), + ) + ) + return patches + + +def _loop_to_any_patches( + source: str, + tree: ast.Module, + statements: list[ast.stmt], + parents: dict[ast.AST, ast.AST], +) -> list[_Patch]: + if _binds_name(tree, "any"): + return [] + patches: list[_Patch] = [] + for index in range(len(statements) - 1): + loop = statements[index] + fallback = statements[index + 1] + if not isinstance(loop, ast.For) or loop.orelse: + continue + if not isinstance(loop.target, ast.Name) or len(loop.body) != 1: + continue + condition = loop.body[0] + if not isinstance(condition, ast.If) or condition.orelse: + continue + if len(condition.body) != 1 or _returned_bool(condition.body[0]) is not True: + continue + if _returned_bool(fallback) is not False: + continue + if not _safe_expression(loop.iter) or not _safe_expression(condition.test): + continue + line_start = loop.lineno + line_end = _node_lines(fallback)[1] + if _removed_bindings_are_observable( + tree, + loop, + parents, + names=frozenset({loop.target.id}), + line_start=line_start, + line_end=line_end, + ): + continue + if _range_has_protected_tokens(source, line_start, line_end): + continue + target = ast.unparse(loop.target) + iterable = ast.unparse(loop.iter) + predicate = ast.unparse(condition.test) + patches.append( + _Patch( + strategy=LOOP_TO_ANY, + line_start=line_start, + line_end=line_end, + code=f"return any({predicate} for {target} in {iterable})", + novelty_tags=( + "ast-derived", + "short-circuit-preserving", + "iterator-reduction", + "cpc-byte-preserving", + ), + ) + ) + return patches + + +def _list_comprehension_patches( + source: str, + tree: ast.Module, + statements: list[ast.stmt], + parents: dict[ast.AST, ast.AST], +) -> list[_Patch]: + patches: list[_Patch] = [] + for index in range(len(statements) - 2): + assignment, loop, result = statements[index : index + 3] + accumulator = _empty_list_target(assignment) + if accumulator is None or not isinstance(loop, ast.For) or loop.orelse: + continue + if not isinstance(loop.target, ast.Name) or loop.target.id == accumulator: + continue + if _returned_name(result) != accumulator: + continue + append_value, predicate = _append_pattern(loop, accumulator) + if append_value is None: + continue + expressions = [loop.iter, append_value] + if predicate is not None: + expressions.append(predicate) + if not all(_safe_expression(expression) for expression in expressions): + continue + if any(_loads_name(expression, accumulator) for expression in expressions): + continue + line_start = assignment.lineno + line_end = _node_lines(result)[1] + if _removed_bindings_are_observable( + tree, + assignment, + parents, + names=frozenset({accumulator, loop.target.id}), + line_start=line_start, + line_end=line_end, + ): + continue + if _range_has_protected_tokens(source, line_start, line_end): + continue + target = ast.unparse(loop.target) + iterable = ast.unparse(loop.iter) + value = ast.unparse(append_value) + filter_clause = "" + novelty = [ + "ast-derived", + "declarative-collection", + "allocation-preserving", + "cpc-byte-preserving", + ] + if predicate is not None: + filter_clause = f" if {ast.unparse(predicate)}" + novelty.append("filter-fusion") + patches.append( + _Patch( + strategy=LIST_ACCUMULATOR_TO_COMPREHENSION, + line_start=line_start, + line_end=line_end, + code=(f"return [{value} for {target} in {iterable}{filter_clause}]"), + novelty_tags=tuple(novelty), + ) + ) + return patches + + +def _append_pattern( + loop: ast.For, + accumulator: str, +) -> tuple[ast.expr | None, ast.expr | None]: + if len(loop.body) != 1: + return None, None + statement = loop.body[0] + direct = _appended_value(statement, accumulator) + if direct is not None: + return direct, None + if not isinstance(statement, ast.If) or statement.orelse: + return None, None + if len(statement.body) != 1: + return None, None + guarded = _appended_value(statement.body[0], accumulator) + if guarded is None: + return None, None + return guarded, statement.test + + +def _appended_value(statement: ast.stmt, accumulator: str) -> ast.expr | None: + if not isinstance(statement, ast.Expr) or not isinstance(statement.value, ast.Call): + return None + call = statement.value + if len(call.args) != 1 or call.keywords: + return None + if not isinstance(call.func, ast.Attribute) or call.func.attr != "append": + return None + if not isinstance(call.func.value, ast.Name) or call.func.value.id != accumulator: + return None + return call.args[0] + + +def _empty_list_target(statement: ast.stmt) -> str | None: + if not isinstance(statement, ast.Assign) or len(statement.targets) != 1: + return None + target = statement.targets[0] + if not isinstance(target, ast.Name): + return None + if not isinstance(statement.value, ast.List) or statement.value.elts: + return None + return target.id + + +def _returned_bool(statement: ast.stmt) -> bool | None: + if not isinstance(statement, ast.Return): + return None + value = statement.value + if not isinstance(value, ast.Constant) or type(value.value) is not bool: + return None + return cast(bool, value.value) + + +def _returned_name(statement: ast.stmt) -> str | None: + if not isinstance(statement, ast.Return) or not isinstance(statement.value, ast.Name): + return None + return statement.value.id + + +def _safe_expression(expression: ast.expr) -> bool: + for node in ast.walk(expression): + if isinstance(node, _SCOPE_SENSITIVE_NODES): + return False + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name): + continue + if node.func.id in _SCOPE_OBSERVERS: + return False + return True + + +def _loads_name(expression: ast.expr, name: str) -> bool: + return any( + isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) and node.id == name + for node in ast.walk(expression) + ) + + +def _binds_name(tree: ast.AST, name: str) -> bool: + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id == name: + if isinstance(node.ctx, (ast.Store, ast.Del)): + return True + elif isinstance(node, ast.arg) and node.arg == name: + return True + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + if node.name == name: + return True + elif isinstance(node, ast.alias): + bound = node.asname or node.name.split(".", maxsplit=1)[0] + if bound == name: + return True + elif isinstance(node, ast.ExceptHandler) and node.name == name: + return True + elif isinstance(node, (ast.Global, ast.Nonlocal)) and name in node.names: + return True + elif isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name == name: + return True + elif isinstance(node, ast.MatchMapping) and node.rest == name: + return True + return False + + +def _statement_lists(tree: ast.AST) -> list[list[ast.stmt]]: + statement_lists: list[list[ast.stmt]] = [] + for node in ast.walk(tree): + for _field, value in ast.iter_fields(node): + if not isinstance(value, list) or not value: + continue + if all(isinstance(item, ast.stmt) for item in value): + statement_lists.append(cast(list[ast.stmt], value)) + return statement_lists + + +def _parent_nodes(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} + + +def _removed_bindings_are_observable( + tree: ast.Module, + node: ast.AST, + parents: dict[ast.AST, ast.AST], + *, + names: frozenset[str], + line_start: int, + line_end: int, +) -> bool: + """Reject scope-changing rewrites when removed locals can be observed.""" + scope = _enclosing_lexical_scope(tree, node, parents) + for candidate in ast.walk(scope): + if isinstance(candidate, (ast.Global, ast.Nonlocal)) and names.intersection( + candidate.names + ): + return True + candidate_line = getattr(candidate, "lineno", 0) + if line_start <= candidate_line <= line_end: + continue + if ( + isinstance(candidate, ast.Name) + and candidate.id in names + and isinstance(candidate.ctx, (ast.Load, ast.Del)) + ): + return True + if ( + isinstance(candidate, ast.Call) + and isinstance(candidate.func, ast.Name) + and candidate.func.id in {"dir", "eval", "exec", "locals", "vars"} + ): + return True + return False + + +def _enclosing_lexical_scope( + tree: ast.Module, + node: ast.AST, + parents: dict[ast.AST, ast.AST], +) -> ast.AST: + current = node + while current in parents: + current = parents[current] + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.Module)): + return current + return tree + + +def _candidate_from_patch(source: str, patch: _Patch) -> MutationCandidate | None: + replacement = _render_replacement( + source, + patch.line_start, + patch.line_end, + patch.code, + ) + after_source = _replace_lines( + source, + patch.line_start, + patch.line_end, + replacement, + ) + if after_source == source or _cpc_payloads(after_source) != _cpc_payloads(source): + return None + try: + ast.parse(after_source) + except SyntaxError: + return None + + source_hash = hashlib.sha256(source.encode("utf-8")).hexdigest() + identity = json.dumps( + { + "line_end": patch.line_end, + "line_start": patch.line_start, + "replacement": replacement, + "source_hash": source_hash, + "strategy": patch.strategy, + "version": 1, + }, + sort_keys=True, + separators=(",", ":"), + ) + candidate_hash = hashlib.sha256(identity.encode("utf-8")).hexdigest() + return MutationCandidate( + strategy=patch.strategy, + candidate_hash=candidate_hash, + source_hash=source_hash, + line_start=patch.line_start, + line_end=patch.line_end, + replacement=replacement, + after_source=after_source, + novelty_tags=patch.novelty_tags, + estimated_line_reduction=patch.line_end - patch.line_start, + ) + + +def _render_replacement(source: str, start: int, end: int, code: str) -> str: + lines = source.splitlines(keepends=True) + first_line = lines[start - 1] + indentation = first_line[: len(first_line) - len(first_line.lstrip(" \t"))] + final_line = lines[end - 1] + if final_line.endswith("\r\n"): + ending = "\r\n" + elif final_line.endswith(("\n", "\r")): + ending = final_line[-1] + else: + ending = "" + rendered = ending.join(f"{indentation}{line}" for line in code.splitlines()) + return rendered + ending + + +def _replacement_fits(source: str, line: int, code: str, *, limit: int = 100) -> bool: + first_line = source.splitlines()[line - 1] + indentation = first_line[: len(first_line) - len(first_line.lstrip(" \t"))] + return all(len(indentation + item) <= limit for item in code.splitlines()) + + +def _replace_lines( + source: str, + start: int, + end: int, + replacement: str, +) -> str: + lines = source.splitlines(keepends=True) + return "".join([*lines[: start - 1], replacement, *lines[end:]]) + + +def _cpc_payloads(source: str) -> tuple[str, ...]: + lines = source.splitlines(keepends=True) + return tuple("".join(lines[start - 1 : end]) for start, end in find_cpc_blocks(source)) + + +def _range_has_protected_tokens(source: str, start: int, end: int) -> bool: + try: + tokens = tokenize.generate_tokens(io.StringIO(source).readline) + for token in tokens: + if not start <= token.start[0] <= end: + continue + if token.type == tokenize.COMMENT: + return True + if token.type == tokenize.OP and token.string == ";": + return True + except (IndentationError, tokenize.TokenError): + return True + return False + + +def _node_lines(node: ast.AST) -> tuple[int, int]: + line_start = getattr(node, "lineno", 0) + line_end = getattr(node, "end_lineno", line_start) or line_start + return int(line_start), int(line_end) + + +def _seen_candidate_hashes(ctx: EngineContext) -> frozenset[str]: + feedback = getattr(ctx, "feedback", None) + found: set[str] = set() + _collect_feedback_hashes(feedback, found, direct=True) + return frozenset(found) + + +def _collect_feedback_hashes(value: Any, found: set[str], *, direct: bool) -> None: + if value is None: + return + if isinstance(value, str): + if direct and value: + found.add(value) + return + if isinstance(value, Mapping): + for key, nested in value.items(): + label = str(key) + if label in { + "candidate_hash", + "candidate_hashes", + "seen_candidate_hashes", + "seen_hashes", + }: + _collect_feedback_hashes(nested, found, direct=True) + elif label in { + "attempts", + "candidates", + "feedback", + "history", + "mutation", + "records", + }: + _collect_feedback_hashes(nested, found, direct=False) + return + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + for item in value: + _collect_feedback_hashes(item, found, direct=direct) + return + candidate_hash = getattr(value, "candidate_hash", None) + if isinstance(candidate_hash, str) and candidate_hash: + found.add(candidate_hash) + for attribute in ("candidate_hashes", "seen_candidate_hashes", "seen_hashes"): + nested = getattr(value, attribute, None) + if nested is not None: + _collect_feedback_hashes(nested, found, direct=True) diff --git a/code_covenant/portfolio/engines.py b/code_covenant/portfolio/engines.py index 80337be..d5b43f7 100644 --- a/code_covenant/portfolio/engines.py +++ b/code_covenant/portfolio/engines.py @@ -65,6 +65,10 @@ class EngineContext: program: str workdir: Path timeout_s: float = 120.0 + attempt_index: int = 0 + feedback: tuple[dict[str, Any], ...] = () + require_confinement: bool = False + allow_network: bool = False @dataclass @@ -105,22 +109,22 @@ class ShellEngine: def propose(self, ctx: EngineContext) -> EngineResult: start = time.perf_counter() - ctx.workdir.mkdir(parents=True, exist_ok=True) - before_path = ctx.workdir / "before.py" - cpc_path = ctx.workdir / "cpc.json" - program_path = ctx.workdir / "program.md" - after_path = ctx.workdir / "after.py" - before_path.write_text(ctx.before_source, encoding="utf-8") - cpc_path.write_text(json.dumps(ctx.cpc, indent=2), encoding="utf-8") - program_path.write_text(ctx.program, encoding="utf-8") try: + ctx.workdir.mkdir(parents=True, exist_ok=True) + before_path = _scratch_path(ctx.workdir, "before.py") + cpc_path = _scratch_path(ctx.workdir, "cpc.json") + program_path = _scratch_path(ctx.workdir, "program.md") + after_path = _scratch_path(ctx.workdir, "after.py") + before_path.write_text(ctx.before_source, encoding="utf-8") + cpc_path.write_text(json.dumps(ctx.cpc, indent=2), encoding="utf-8") + program_path.write_text(ctx.program, encoding="utf-8") rendered = self.command_template.format( - target=str(ctx.target_path), - before=str(before_path), - after=str(after_path), - cpc=str(cpc_path), - program=str(program_path), - workdir=str(ctx.workdir), + target=shlex.quote(str(ctx.target_path)), + before=shlex.quote(str(before_path)), + after=shlex.quote(str(after_path)), + cpc=shlex.quote(str(cpc_path)), + program=shlex.quote(str(program_path)), + workdir=shlex.quote(str(ctx.workdir)), ) except KeyError as exc: return EngineResult( @@ -130,9 +134,15 @@ def propose(self, ctx: EngineContext) -> EngineResult: error=f"unknown placeholder in command template: {exc}", prompt_id=_prompt_id(self.command_template), ) - return self._invoke_and_read( - rendered, after_path, ctx, start - ) + except OSError as exc: + return EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - start, + error=f"engine scratch setup failed: {exc}", + prompt_id=_prompt_id(self.command_template), + ) + return self._invoke_and_read(rendered, after_path, ctx, start) def _invoke_and_read( self, @@ -153,16 +163,34 @@ def _invoke_and_read( prompt_id=prompt_id, ) try: - completed = subprocess.run( # noqa: S603 - argv, - cwd=str(ctx.workdir), - capture_output=True, - text=True, - timeout=ctx.timeout_s, - env=self.env, - check=False, - ) - except (FileNotFoundError, subprocess.TimeoutExpired) as exc: + if ctx.require_confinement: + from code_covenant.piranha.execution import ( + ConfinementPolicy, + run_confined, + ) + + completed = run_confined( + argv, + cwd=ctx.workdir, + timeout_s=ctx.timeout_s, + policy=ConfinementPolicy( + read_roots=(ctx.workdir,), + write_root=ctx.workdir, + allow_network=ctx.allow_network, + extra_env=self.env or {}, + ), + ) + else: + completed = subprocess.run( # noqa: S603 + argv, + cwd=str(ctx.workdir), + capture_output=True, + text=True, + timeout=ctx.timeout_s, + env=self.env, + check=False, + ) + except (FileNotFoundError, OSError, RuntimeError, subprocess.TimeoutExpired) as exc: return EngineResult( ok=False, after_source=None, @@ -179,9 +207,7 @@ def _invoke_and_read( prompt_id=prompt_id, raw_output=completed.stdout, ) - return self._read_output( - after_path, completed.stdout, prompt_id, start - ) + return self._read_output(after_path, completed.stdout, prompt_id, start) def _read_output( self, @@ -215,7 +241,38 @@ def _read_output( prompt_id=prompt_id, raw_output=stdout, ) - after_source = after_path.read_text(encoding="utf-8") + if after_path.is_symlink() or not after_path.is_file() or after_path.stat().st_nlink != 1: + return EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - start, + error="engine after-source must be a regular, singly-linked file", + prompt_id=prompt_id, + raw_output=stdout, + ) + try: + after_path.resolve().relative_to(after_path.parent.resolve()) + if after_path.stat().st_size > 2 * 1024 * 1024: + raise OSError("engine after-source exceeds 2 MiB") + after_source = after_path.read_text(encoding="utf-8") + except (OSError, UnicodeError, ValueError) as exc: + return EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - start, + error=f"could not safely read engine after-source: {exc}", + prompt_id=prompt_id, + raw_output=stdout, + ) + if not after_source: + return EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - start, + error="engine produced an empty after-source file", + prompt_id=prompt_id, + raw_output=stdout, + ) return EngineResult( ok=True, after_source=after_source, @@ -288,9 +345,7 @@ def make_engine( if not command_template: raise ValueError("claude-code engine requires a command_template") mode = output_mode or "stdout" - return ClaudeCodeEngine( - command_template=command_template, output_mode=mode - ) + return ClaudeCodeEngine(command_template=command_template, output_mode=mode) if kind == "model": # The model-driven engine. From the factory/CLI it is backed by a real # LLM CLI (via ClaudeCodeEngine) — no fake default template. For the @@ -312,9 +367,27 @@ def make_engine( proposer=llm_cli_proposer(command_template, output_mode=mode), prompt_label="model-cli", ) + if kind == "utility": + if command_template: + raise ValueError("utility engine does not accept a command_template") + from code_covenant.piranha.utility_engine import UtilityMutationEngine + + return UtilityMutationEngine() raise ValueError(f"unknown engine kind: {kind!r}") def _prompt_id(template: str) -> str: digest = hashlib.sha256(template.encode("utf-8")).hexdigest() return digest[:12] + + +def _scratch_path(workdir: Path, name: str) -> Path: + path = workdir / name + if path.exists() and path.is_symlink(): + raise OSError(f"scratch path is a symlink: {path}") + if path.exists() and not path.is_file(): + raise OSError(f"scratch path is not a regular file: {path}") + if path.exists() and path.stat().st_nlink != 1: + raise OSError(f"scratch path has unexpected hard links: {path}") + path.parent.resolve().relative_to(workdir.resolve()) + return path diff --git a/code_covenant/portfolio/model_engine.py b/code_covenant/portfolio/model_engine.py index a2b3548..0d3fc36 100644 --- a/code_covenant/portfolio/model_engine.py +++ b/code_covenant/portfolio/model_engine.py @@ -18,7 +18,8 @@ # ║ ║ # ║ constraints: ║ # ║ - Must never raise; failures go into EngineResult.error ║ -# ║ - Must write ONLY under ctx.workdir (blast-radius parity) ║ +# ║ - Built-in scratch writes stay under ctx.workdir ║ +# ║ - Embedded proposers are trusted control-plane code ║ # ║ - Must respect the configured timeout ║ # ║ ║ # ║ invariants: ║ @@ -31,7 +32,7 @@ # ║ ║ # ║ forbidden_changes: ║ # ║ - Do not let the engine touch the gate or champion table ║ -# ║ - Do not write outside ctx.workdir ║ +# ║ - Do not describe embedded callables as process-confined ║ # ║ - Do not ship a fake default command_template ║ # ║ ║ # ║ optimization_targets: ║ @@ -51,7 +52,7 @@ import json import time -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Callable from code_covenant.portfolio.engines import ( @@ -71,10 +72,10 @@ class ModelEngine: """A model-driven Engine: it *thinks* (via `proposer`) about a mutation of `ctx.before_source` and returns it as a candidate. It sits strictly BELOW the - lab's sandbox/safety guards (run_lab → run_portfolio → engine.propose) and - inherits S1–S4 for free by (a) writing scratch only under ``ctx.workdir`` and - (b) never raising. It proposes; the portfolio gate decides merge/reject/review - and the champion table records the winner — the engine never touches either. + lab's path guards (run_lab → run_portfolio → engine.propose). Subprocess + proposers must also honor EngineContext.require_confinement; path containment + alone is not process isolation. It proposes; the portfolio gate decides + merge/reject/review and the champion table records the winner. """ proposer: Proposer @@ -92,19 +93,21 @@ def propose(self, ctx: EngineContext) -> EngineResult: # under ctx.workdir — before.py, cpc.json, program.md (and after.py # below). No path outside the workdir is ever touched. (ctx.workdir / "before.py").write_text(ctx.before_source, encoding="utf-8") - (ctx.workdir / "cpc.json").write_text( - json.dumps(ctx.cpc, indent=2), encoding="utf-8") + (ctx.workdir / "cpc.json").write_text(json.dumps(ctx.cpc, indent=2), encoding="utf-8") (ctx.workdir / "program.md").write_text(ctx.program, encoding="utf-8") after = self.proposer(ctx) except Exception as exc: # noqa: BLE001 — never raise past here (S3 depends on it) return EngineResult( - ok=False, after_source=None, + ok=False, + after_source=None, duration_s=time.perf_counter() - start, - error=f"model engine failed: {exc!r}", prompt_id=pid, + error=f"model engine failed: {exc!r}", + prompt_id=pid, ) if not isinstance(after, str) or not after: return EngineResult( - ok=False, after_source=None, + ok=False, + after_source=None, duration_s=time.perf_counter() - start, error="model engine returned empty or non-string output", prompt_id=pid, @@ -114,21 +117,23 @@ def propose(self, ctx: EngineContext) -> EngineResult: except OSError: pass # scratch write is best-effort; the return value is authoritative return EngineResult( - ok=True, after_source=after, - duration_s=time.perf_counter() - start, prompt_id=pid, + ok=True, + after_source=after, + duration_s=time.perf_counter() - start, + prompt_id=pid, ) # --- in-process deterministic proposers (offline; for the verifier + embedding) -- + def strip_lines_proposer(marker: str) -> Proposer: """A conservative, deterministic proposer: drop every source line containing ``marker`` (e.g. a redundant-comment tag). Behavior-preserving when the marked lines are inert — the perfect *plantable* improvement for the verifier.""" def _propose(ctx: EngineContext) -> str: - kept = [ln for ln in ctx.before_source.splitlines(keepends=True) - if marker not in ln] + kept = [ln for ln in ctx.before_source.splitlines(keepends=True) if marker not in ln] return "".join(kept) return _propose @@ -166,6 +171,7 @@ def __call__(self, ctx: EngineContext) -> str: # --- production proposer: a real LLM CLI via the existing adapters --------------- + def llm_cli_proposer( command_template: str, *, @@ -184,15 +190,18 @@ def llm_cli_proposer( def _propose(ctx: EngineContext) -> str: from code_covenant.portfolio.llm_engines import ClaudeCodeEngine - inner = ClaudeCodeEngine( - command_template=command_template, output_mode=output_mode) + inner = ClaudeCodeEngine(command_template=command_template, output_mode=output_mode) sub_ctx = EngineContext( target_path=ctx.target_path, before_source=ctx.before_source, cpc=ctx.cpc, program=ctx.program, - workdir=ctx.workdir / subdir, # stays under ctx.workdir + workdir=ctx.workdir / subdir, # stays under ctx.workdir timeout_s=ctx.timeout_s, + attempt_index=ctx.attempt_index, + feedback=ctx.feedback, + require_confinement=ctx.require_confinement, + allow_network=ctx.allow_network, ) result = inner.propose(sub_ctx) if not result.ok or not result.after_source: diff --git a/code_covenant/portfolio/runner.py b/code_covenant/portfolio/runner.py index 3f639bf..387ff04 100644 --- a/code_covenant/portfolio/runner.py +++ b/code_covenant/portfolio/runner.py @@ -47,14 +47,17 @@ from __future__ import annotations +import hashlib +import json import secrets import time from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from code_covenant.gate.pipeline import ProposalConfig, evaluate_proposal +from code_covenant.gate.ledger import append_ledger_entry, read_ledger from code_covenant.portfolio.brief import write_brief_artifacts from code_covenant.portfolio.engines import Engine, EngineContext, EngineResult from code_covenant.portfolio.inventory import TargetRecord, load_targets @@ -63,6 +66,9 @@ from code_covenant.portfolio.scope import ScopeFilter, apply_scope from code_covenant.portfolio.taxonomy import classify_outcome +if TYPE_CHECKING: + from code_covenant.piranha.policy import UtilityPolicy + @dataclass class PortfolioConfig: @@ -86,6 +92,9 @@ class PortfolioConfig: benchmark_history_path: Path | None = None require_metric_improvement: bool = False minimize_metrics: frozenset[str] = field(default_factory=frozenset) + utility_policy: "UtilityPolicy | None" = None + require_process_confinement: bool = False + allow_engine_network: bool = False @dataclass @@ -107,6 +116,7 @@ class AttemptRecord: duration_s: float proposal_id: str | None = None parent_proposal_id: str | None = None + mutation: dict[str, Any] = field(default_factory=dict) @dataclass @@ -153,7 +163,7 @@ def run_portfolio(config: PortfolioConfig) -> PortfolioResult: reason=attempt.reason, failure_dimensions=attempt.failure_dimensions, ) - if getattr(attempt, "proposal_id", None): + if getattr(attempt, "proposal_id", None) and attempt.decision == "merge": target.last_proposal_id = attempt.proposal_id result.ended_at = _iso_now() result.duration_s = time.monotonic() - started_monotonic @@ -172,14 +182,15 @@ def _should_continue( return True -def _run_single_attempt( - config: PortfolioConfig, target: TargetRecord, index: int -) -> AttemptRecord: +def _run_single_attempt(config: PortfolioConfig, target: TargetRecord, index: int) -> AttemptRecord: timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") workdir = config.output_dir / "engine_workdirs" / f"{timestamp}_{index:04d}" before_source = target.file_path.read_text(encoding="utf-8") + source_hash = hashlib.sha256(before_source.encode("utf-8")).hexdigest() program = program_for_cpc(target.cpc) start = time.perf_counter() + parent_proposal_id = _last_merged_proposal(config.ledger_path, target.module) + require_process_confinement = _process_confinement_required(config) ctx = EngineContext( target_path=target.file_path, before_source=before_source, @@ -187,9 +198,35 @@ def _run_single_attempt( program=program, workdir=workdir, timeout_s=config.engine_timeout_s, + attempt_index=index, + feedback=_feedback_for_target( + config.ledger_path, + target.module, + source_hash=source_hash, + ), + require_confinement=require_process_confinement, + allow_network=config.allow_engine_network, ) - engine_result: EngineResult = config.engine.propose(ctx) + try: + engine_result: EngineResult = config.engine.propose(ctx) + except Exception as exc: # noqa: BLE001 - orchestration must preserve the ledger/report + engine_result = EngineResult( + ok=False, + after_source=None, + duration_s=time.perf_counter() - start, + error=f"engine escaped its result contract: {exc!r}", + ) + mutation = _json_safe_metadata(engine_result.extra) if not engine_result.ok or not engine_result.after_source: + proposal_id = _record_engine_failure( + config=config, + target=target, + index=index, + timestamp=timestamp, + parent_proposal_id=parent_proposal_id, + engine_result=engine_result, + mutation=mutation, + ) return AttemptRecord( attempt_index=index, timestamp=timestamp, @@ -198,35 +235,69 @@ def _run_single_attempt( prompt_id=engine_result.prompt_id, engine_ok=False, engine_error=engine_result.error, - decision=None, + decision="reject", reason=engine_result.error, risk=None, constraint_grades=[], failure_dimensions=["engine_error"], duration_s=time.perf_counter() - start, + proposal_id=proposal_id, + parent_proposal_id=parent_proposal_id, + mutation=mutation, ) - proposal = evaluate_proposal( - ProposalConfig( - target_path=target.file_path, - after_source=engine_result.after_source, - proposals_dir=config.proposals_dir, - ledger_path=config.ledger_path, - test_command=config.test_command, - benchmark_command=config.benchmark_command, - cwd=config.project_cwd, - merge_mode=config.merge_mode, - rationale=f"portfolio run attempt #{index} via {config.engine.name}", - timeout_s=config.gate_timeout_s, - run_id=config.run_id, - iteration_id=index, - parent_proposal_id=_last_proposal_for_target(target), - engine_name=config.engine.name, - prompt_template=engine_result.prompt_id, - benchmark_history_path=config.benchmark_history_path, - require_metric_improvement=config.require_metric_improvement, - minimize_metrics=config.minimize_metrics, - ) + proposal_config = ProposalConfig( + target_path=target.file_path, + after_source=engine_result.after_source, + proposals_dir=config.proposals_dir, + ledger_path=config.ledger_path, + test_command=config.test_command, + benchmark_command=config.benchmark_command, + cwd=config.project_cwd, + merge_mode=config.merge_mode, + rationale=f"portfolio run attempt #{index} via {config.engine.name}", + timeout_s=config.gate_timeout_s, + run_id=config.run_id, + iteration_id=index, + parent_proposal_id=parent_proposal_id, + engine_name=config.engine.name, + prompt_template=engine_result.prompt_id, + benchmark_history_path=config.benchmark_history_path, + require_metric_improvement=config.require_metric_improvement, + minimize_metrics=config.minimize_metrics, + utility_policy=config.utility_policy, + require_process_confinement=require_process_confinement, + mutation_metadata=mutation, ) + try: + proposal = evaluate_proposal(proposal_config) + except Exception as exc: # noqa: BLE001 - preserve a terminal attempt record + reason = f"proposal gate escaped its result contract: {exc!r}" + proposal_id = _record_gate_failure( + config=config, + target=target, + index=index, + parent_proposal_id=parent_proposal_id, + reason=reason, + mutation=mutation, + ) + return AttemptRecord( + attempt_index=index, + timestamp=timestamp, + target_module=target.module, + engine=config.engine.name, + prompt_id=engine_result.prompt_id, + engine_ok=True, + engine_error="", + decision="reject", + reason=reason, + risk=None, + constraint_grades=[], + failure_dimensions=["gate_error"], + duration_s=time.perf_counter() - start, + proposal_id=proposal_id, + parent_proposal_id=parent_proposal_id, + mutation=mutation, + ) dims = classify_outcome( engine_ok=True, engine_error="", @@ -250,7 +321,8 @@ def _run_single_attempt( failure_dimensions=dims, duration_s=time.perf_counter() - start, proposal_id=proposal.proposal_id, - parent_proposal_id=target.last_proposal_id, + parent_proposal_id=parent_proposal_id, + mutation=mutation, ) @@ -283,6 +355,9 @@ def _attempt_to_dict(attempt: AttemptRecord) -> dict[str, Any]: "constraint_grades": attempt.constraint_grades, "failure_dimensions": attempt.failure_dimensions, "duration_s": attempt.duration_s, + "proposal_id": attempt.proposal_id, + "parent_proposal_id": attempt.parent_proposal_id, + "mutation": dict(attempt.mutation), } @@ -290,10 +365,156 @@ def _iso_now() -> str: return datetime.now(UTC).isoformat() -def _last_proposal_for_target(target: TargetRecord) -> str | None: - return target.last_proposal_id - - def _new_run_id(prefix: str) -> str: stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") return f"{prefix}-{stamp}-{secrets.token_hex(2)}" + + +def _process_confinement_required(config: PortfolioConfig) -> bool: + return config.require_process_confinement or bool( + config.utility_policy and config.utility_policy.require_process_confinement + ) + + +def _feedback_for_target( + ledger_path: Path, + module: str, + *, + source_hash: str | None = None, + limit: int = 20, +) -> tuple[dict[str, Any], ...]: + try: + entries = read_ledger(ledger_path) + except (OSError, json.JSONDecodeError): + return () + feedback: list[dict[str, Any]] = [] + seen_candidate_hashes: set[str] = set() + for entry in entries: + if entry.get("module") != module: + continue + mutation = entry.get("mutation") + if not isinstance(mutation, dict): + mutation = {} + candidate_hash = mutation.get("candidate_hash") + candidate_source_hash = mutation.get("source_hash") + if ( + isinstance(candidate_hash, str) + and candidate_hash + and ( + source_hash is None + or candidate_source_hash is None + or candidate_source_hash == source_hash + ) + ): + seen_candidate_hashes.add(candidate_hash) + feedback.append( + { + "proposal_id": entry.get("proposal_id"), + "decision": entry.get("decision"), + "reason": entry.get("reason"), + "metric_before": entry.get("metric_before"), + "metric_after": entry.get("metric_after"), + "mutation": mutation, + } + ) + rich_feedback = feedback[-limit:] if limit > 0 else [] + if not seen_candidate_hashes: + return tuple(rich_feedback) + seen_summary = { + "feedback_kind": "seen-candidate-hashes", + "seen_candidate_hashes": sorted(seen_candidate_hashes), + } + return (seen_summary, *rich_feedback) + + +def _last_merged_proposal(ledger_path: Path, module: str) -> str | None: + try: + entries = read_ledger(ledger_path) + except (OSError, json.JSONDecodeError): + return None + for entry in reversed(entries): + if entry.get("module") == module and entry.get("merged"): + proposal_id = entry.get("proposal_id") + return str(proposal_id) if proposal_id else None + return None + + +def _json_safe_metadata(value: dict[str, Any]) -> dict[str, Any]: + if not isinstance(value, dict): + return {} + try: + return json.loads(json.dumps(value, sort_keys=True)) + except (TypeError, ValueError): + return {} + + +def _record_engine_failure( + *, + config: PortfolioConfig, + target: TargetRecord, + index: int, + timestamp: str, + parent_proposal_id: str | None, + engine_result: EngineResult, + mutation: dict[str, Any], +) -> str: + proposal_id = f"{timestamp}_engine_{secrets.token_hex(2)}" + append_ledger_entry( + config.ledger_path, + { + "timestamp": _iso_now(), + "run_id": config.run_id, + "iteration_id": index, + "proposal_id": proposal_id, + "parent_proposal_id": parent_proposal_id, + "module": target.module, + "target": str(target.file_path), + "file": str(target.file_path), + "engine": config.engine.name, + "prompt_template": engine_result.prompt_id, + "decision": "reject", + "reason": engine_result.error or "engine produced no candidate", + "failure_reason": engine_result.error or "engine produced no candidate", + "result": "rejected", + "merged": False, + "metric_before": None, + "metric_after": None, + "mutation": mutation, + }, + ) + return proposal_id + + +def _record_gate_failure( + *, + config: PortfolioConfig, + target: TargetRecord, + index: int, + parent_proposal_id: str | None, + reason: str, + mutation: dict[str, Any], +) -> str: + proposal_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_gate_{secrets.token_hex(2)}" + append_ledger_entry( + config.ledger_path, + { + "timestamp": _iso_now(), + "run_id": config.run_id, + "iteration_id": index, + "proposal_id": proposal_id, + "parent_proposal_id": parent_proposal_id, + "module": target.module, + "target": str(target.file_path), + "file": str(target.file_path), + "engine": config.engine.name, + "decision": "reject", + "reason": reason, + "failure_reason": reason, + "result": "rejected", + "merged": False, + "metric_before": None, + "metric_after": None, + "mutation": mutation, + }, + ) + return proposal_id diff --git a/code_covenant/portfolio/scope.py b/code_covenant/portfolio/scope.py index 17e1495..3bd1f75 100644 --- a/code_covenant/portfolio/scope.py +++ b/code_covenant/portfolio/scope.py @@ -62,9 +62,7 @@ class ScopeFilter: min_risk: str | None = None -def apply_scope( - targets: list[TargetRecord], scope: ScopeFilter | None -) -> list[TargetRecord]: +def apply_scope(targets: list[TargetRecord], scope: ScopeFilter | None) -> list[TargetRecord]: """Return the subset of targets that pass every active scope rule.""" if scope is None: return list(targets) @@ -95,11 +93,11 @@ def _module_matches(target: TargetRecord, patterns: list[str]) -> bool: return any(fnmatch.fnmatchcase(target.module, pattern) for pattern in patterns) -def _risk_within( - target: TargetRecord, min_risk: str | None, max_risk: str | None -) -> bool: - risk = _cpc_field(target, "risk_level") or "low" - rank = _RISK_ORDER.get(risk, 0) +def _risk_within(target: TargetRecord, min_risk: str | None, max_risk: str | None) -> bool: + risk = _cpc_field(target, "risk_level") + # Unknown or missing risk is not evidence of low risk. Fail closed at the + # top of the ordering so bounded mutation scopes exclude it. + rank = _RISK_ORDER.get(risk, _RISK_ORDER["critical"]) if min_risk and rank < _RISK_ORDER.get(min_risk, 0): return False if max_risk and rank > _RISK_ORDER.get(max_risk, 3): diff --git a/code_covenant/tools/piranha.py b/code_covenant/tools/piranha.py index 05f3fa2..dc95938 100644 --- a/code_covenant/tools/piranha.py +++ b/code_covenant/tools/piranha.py @@ -19,7 +19,7 @@ # ║ constraints: ║ # ║ - Must refuse to run without the confirmation flag ║ # ║ - Must reject sandbox roots that overlap the source ║ -# ║ - Non-'shell' engines require an explicit --engine-command;║ +# ║ - External engines require an explicit --engine-command; ║ # ║ no engine bypasses the gate or the confirmation flag ║ # ║ ║ # ║ invariants: ║ @@ -52,6 +52,11 @@ from pathlib import Path from code_covenant.piranha.lab import LabConfig, run_lab +from code_covenant.piranha.policy import ( + BUILTIN_UTILITY_METRICS, + RISK_ORDER, + UtilityPolicy, +) from code_covenant.piranha.sandbox import SandboxError from code_covenant.portfolio.engines import make_engine from code_covenant.portfolio.scope import ScopeFilter @@ -86,42 +91,57 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--engine", - choices=["shell", "model", "codex", "claude-code"], - default="shell", - help="Proposal engine. 'model' is the model-driven engine (backed by an " - "LLM CLI given via --engine-command). No engine bypasses the gate or the " - "confirmation flag.", + choices=["utility", "shell", "model", "codex", "claude-code"], + default="utility", + help="Proposal engine. 'utility' is the default trusted AST mutator; " + "every other choice requires an explicitly supplied confined CLI.", ) parser.add_argument( "--engine-command", default="", - help="Command template for the shell/model/codex/claude-code engines " - "(required for all non-'shell' engines too). No fake default is shipped.", + help="Required command template for shell/model/codex/claude-code. " + "The utility engine accepts none; no fake default is shipped.", ) parser.add_argument("--rounds", type=int, default=3) parser.add_argument("--duration", type=float, default=60.0) parser.add_argument("--max-attempts-per-round", type=int, default=None) - parser.add_argument("--test-command", default=None) - parser.add_argument("--benchmark-command", default=None) + parser.add_argument( + "--test-command", + default=None, + help="Required immutable behavior oracle. Must be an explicit Python " + "command: python FILE, python -m MODULE, or python -m pytest.", + ) + parser.add_argument( + "--benchmark-command", + default=None, + help="Optional hard-gate benchmark, run in a separate confined cell.", + ) parser.add_argument("--engine-timeout", type=float, default=60.0) parser.add_argument("--gate-timeout", type=float, default=60.0) parser.add_argument("--module", action="append", default=[]) parser.add_argument("--cohort", action="append", default=[]) - parser.add_argument("--max-risk", default=None) - parser.add_argument("--min-risk", default=None) + parser.add_argument("--max-risk", choices=tuple(RISK_ORDER), default=None) + parser.add_argument("--min-risk", choices=tuple(RISK_ORDER), default=None) parser.add_argument( "--require-improvement", action="store_true", - help="Demote merge -> review unless metric_after strictly beats the most " - "recent merged baseline for this module (Autoresearch ratchet).", + help="Also require benchmark metrics to strictly beat the most recent " + "merged benchmark baseline for this module (Autoresearch ratchet).", ) parser.add_argument( "--minimize", action="append", default=[], metavar="METRIC", - help="Mark a metric as 'smaller is better' for the improvement check. " - "Repeatable.", + help="Mark a metric as 'smaller is better' for the improvement check. Repeatable.", + ) + parser.add_argument( + "--utility-metric", + action="append", + choices=sorted(BUILTIN_UTILITY_METRICS), + default=[], + help="Trusted built-in metric required for promotion. Repeatable. " + "Defaults to source_bytes, branch_nodes, and bytecode_instructions.", ) return parser @@ -141,6 +161,13 @@ def main(argv: list[str] | None = None) -> int: if not source.exists() or not source.is_dir(): print(f"source directory not found: {source}", file=sys.stderr) return 2 + if not args.test_command: + print( + "refusing to run without --test-command: Piranha promotion requires " + "an immutable behavior oracle", + file=sys.stderr, + ) + return 2 try: engine = make_engine(args.engine, command_template=args.engine_command) except ValueError as exc: @@ -153,6 +180,13 @@ def main(argv: list[str] | None = None) -> int: max_risk=args.max_risk, min_risk=args.min_risk, ) + required_metrics = frozenset( + args.utility_metric or ("source_bytes", "branch_nodes", "bytecode_instructions") + ) + utility_policy = UtilityPolicy( + required_metrics=required_metrics, + minimize_metrics=frozenset(BUILTIN_UTILITY_METRICS), + ) config = LabConfig( source_root=source, sandbox_root=sandbox, @@ -169,6 +203,8 @@ def main(argv: list[str] | None = None) -> int: allow_repo_root=args.allow_repo_root, require_metric_improvement=args.require_improvement, minimize_metrics=frozenset(args.minimize), + utility_policy=utility_policy, + require_process_confinement=True, ) try: result = run_lab(config) @@ -184,7 +220,9 @@ def _print_summary(result) -> None: print(f"rounds: {result.rounds_completed}") print(f"attempts: {result.total_attempts}") print(f"merges: {result.total_merges}") - print(f"champions: {len([s for s in result.champion_table.entries.values() if s.champion_proposal_id])}") + print( + f"champions: {len([s for s in result.champion_table.entries.values() if s.champion_proposal_id])}" + ) print(f"best chain: {len(result.lineage.best_lineage)} step(s)") print(f"worst chain: {len(result.lineage.worst_lineage)} reject(s)") print(f"report: {result.report_path}") diff --git a/code_covenant/tools/piranha_model_verify.py b/code_covenant/tools/piranha_model_verify.py index 247e749..830b226 100644 --- a/code_covenant/tools/piranha_model_verify.py +++ b/code_covenant/tools/piranha_model_verify.py @@ -48,13 +48,12 @@ from __future__ import annotations import hashlib +import json import shutil import sys import tempfile from pathlib import Path -import pytest - from code_covenant.piranha.lab import LabConfig, run_lab from code_covenant.piranha.sandbox import SandboxError, validate_contained from code_covenant.portfolio.model_engine import ( @@ -127,15 +126,16 @@ def _seed_source(tmp: Path, name: str = "source") -> Path: def _benchmark_for(sandbox_root: Path) -> str: """A deterministic, offline benchmark: count the cost markers in the APPLIED - target and emit a METRIC line. Uses only /bin/sh + grep + printf — no - sys.executable (the exact dependence that made the shell e2e flaky), no - network. Smaller cost is better (minimize).""" - target = (sandbox_root.resolve() / "src" / "alpha.py") + target and emit a METRIC line. Uses only /bin/bash builtins, so confinement + need not grant child-process authority. Smaller cost is better (minimize).""" + target = sandbox_root.resolve() / "src" / "alpha.py" script = ( - f'n=$(grep -c {_MARK} "{target}"); ' - 'printf "METRIC: cost %s\\n" "${n:-0}"' + "n=0; " + f'while IFS= read -r line; do case "$line" in *{_MARK}*) ' + 'n=$((n + 1));; esac; done < "' + f'{target}"; printf "METRIC: cost %s\\n" "$n"' ) - return f"/bin/sh -c '{script}'" + return f"/bin/bash -c '{script}'" def _tree_hash(root: Path) -> str: @@ -176,10 +176,16 @@ def _scenario_planted_wins(tmp: Path) -> None: engine = ModelEngine(strip_lines_proposer(_MARK), prompt_label="strip") config = LabConfig( - source_root=source, sandbox_root=sandbox, engine=engine, - confirm_sandbox_destructive=True, rounds=1, max_attempts_per_round=1, - duration_s=30.0, benchmark_command=_benchmark_for(sandbox), - require_metric_improvement=True, minimize_metrics=frozenset({"cost"}), + source_root=source, + sandbox_root=sandbox, + engine=engine, + confirm_sandbox_destructive=True, + rounds=1, + max_attempts_per_round=1, + duration_s=30.0, + benchmark_command=_benchmark_for(sandbox), + require_metric_improvement=True, + minimize_metrics=frozenset({"cost"}), ) result = run_lab(config) layout_root = result.layout.root @@ -187,19 +193,30 @@ def _scenario_planted_wins(tmp: Path) -> None: # (1) planted improvement wins the champion — matched to the planted variant, # not merely "some champion exists". merged = [e for e in result.ledger_entries if e.get("merged")] - _check(len(merged) == 1, f"expected exactly one merge, got {len(merged)}") + gate_details = [ + json.loads((Path(str(entry["folder"])) / "metrics.json").read_text(encoding="utf-8")) + for entry in result.ledger_entries + ] + _check( + len(merged) == 1, + f"expected exactly one merge, got {len(merged)}; " + f"ledger={result.ledger_entries}; gates={gate_details}", + ) champ_id = merged[0].get("proposal_id") entry = result.champion_table.entries.get("demo.alpha") _check(entry is not None, "no champion entry for demo.alpha") - _check(entry.champion_proposal_id == champ_id, - f"champion {entry.champion_proposal_id!r} != merged {champ_id!r}") - _check(merged[0].get("metric_after") == {"cost": 0.0}, - f"champion metric is not the planted (cost=0): {merged[0].get('metric_after')}") + _check( + entry.champion_proposal_id == champ_id, + f"champion {entry.champion_proposal_id!r} != merged {champ_id!r}", + ) + _check( + merged[0].get("metric_after") == {"cost": 0.0}, + f"champion metric is not the planted (cost=0): {merged[0].get('metric_after')}", + ) # (3) source untouched — byte-for-byte full-tree hash. _check(_tree_hash(source) == before_tree, "source tree changed during run") - _check((source / "alpha.py").read_text().count(_MARK) == 2, - "seed markers were mutated") + _check((source / "alpha.py").read_text().count(_MARK) == 2, "seed markers were mutated") # (3) every lab output path is contained inside the sandbox root. for label, path in [ @@ -218,16 +235,14 @@ def _scenario_planted_wins(tmp: Path) -> None: # (3) report always written with the required sections. body = result.report_path.read_text() - for section in ("# Piranha Lab Report", "Champion state", "Best lineage", - "Worst lineage"): + for section in ("# Piranha Lab Report", "Champion state", "Best lineage", "Worst lineage"): _check(section in body, f"report missing section {section!r}") # (4) blast radius not widened: every NEW file created during the run lives # inside the sandbox root — the model engine touched nothing the shell # engine could not. new_files = _all_files(tmp) - before_files - escaped = [f for f in new_files - if not f.startswith(str(layout_root.resolve()) + "/")] + escaped = [f for f in new_files if not f.startswith(str(layout_root.resolve()) + "/")] _check(not escaped, f"files written outside the sandbox root: {escaped[:5]}") @@ -237,33 +252,44 @@ def _scenario_null_demoted(tmp: Path) -> None: source = _seed_source(tmp, "srcB") sandbox = tmp / "sbxB" engine = ModelEngine( - SequenceProposer([ - strip_lines_proposer(_MARK), # attempt 1: real improvement - append_comment_proposer("noop variant"), # attempt 2: null (ties) - ]), + SequenceProposer( + [ + strip_lines_proposer(_MARK), # attempt 1: real improvement + append_comment_proposer("noop variant"), # attempt 2: null (ties) + ] + ), prompt_label="seq", ) config = LabConfig( - source_root=source, sandbox_root=sandbox, engine=engine, - confirm_sandbox_destructive=True, rounds=1, max_attempts_per_round=2, - duration_s=30.0, benchmark_command=_benchmark_for(sandbox), - require_metric_improvement=True, minimize_metrics=frozenset({"cost"}), + source_root=source, + sandbox_root=sandbox, + engine=engine, + confirm_sandbox_destructive=True, + rounds=1, + max_attempts_per_round=2, + duration_s=30.0, + benchmark_command=_benchmark_for(sandbox), + require_metric_improvement=True, + minimize_metrics=frozenset({"cost"}), ) result = run_lab(config) merged = [e for e in result.ledger_entries if e.get("merged")] - reviewed = [e for e in result.ledger_entries - if _decision_of(e) == "review"] + reviewed = [e for e in result.ledger_entries if _decision_of(e) == "review"] _check(len(merged) == 1, f"expected exactly one merge (the improver), got {len(merged)}") _check(len(reviewed) >= 1, "null variant was not demoted to review") entry = result.champion_table.entries.get("demo.alpha") _check(entry is not None, "no champion entry for demo.alpha") - _check(entry.champion_proposal_id == merged[0].get("proposal_id"), - "the null variant took the champion slot (must not)") + _check( + entry.champion_proposal_id == merged[0].get("proposal_id"), + "the null variant took the champion slot (must not)", + ) # the null variant is NOT the champion null_ids = {e.get("proposal_id") for e in reviewed} - _check(entry.champion_proposal_id not in null_ids, - "champion id matches a demoted (review) proposal") + _check( + entry.champion_proposal_id not in null_ids, + "champion id matches a demoted (review) proposal", + ) def _decision_of(entry: dict) -> str: @@ -281,16 +307,22 @@ def _scenario_safety_flags(tmp: Path) -> None: # S1 — refuses without confirmation, with the model engine installed. try: - run_lab(LabConfig(source_root=source, sandbox_root=tmp / "sbxC1", - engine=engine, rounds=1)) + run_lab(LabConfig(source_root=source, sandbox_root=tmp / "sbxC1", engine=engine, rounds=1)) raise _Fail("run_lab did not refuse without confirm_sandbox_destructive") except SandboxError: pass # S2 — rejects overlapping source/sandbox roots, with the model engine. try: - run_lab(LabConfig(source_root=source, sandbox_root=source / "inside", - engine=engine, confirm_sandbox_destructive=True, rounds=1)) + run_lab( + LabConfig( + source_root=source, + sandbox_root=source / "inside", + engine=engine, + confirm_sandbox_destructive=True, + rounds=1, + ) + ) raise _Fail("run_lab did not reject an overlapping sandbox root") except SandboxError: pass @@ -298,9 +330,16 @@ def _scenario_safety_flags(tmp: Path) -> None: # S3 — report is written even with ZERO targets (empty source dir). empty = tmp / "emptysrc" empty.mkdir() - zresult = run_lab(LabConfig( - source_root=empty, sandbox_root=tmp / "sbxC2", engine=engine, - confirm_sandbox_destructive=True, rounds=1, duration_s=5.0)) + zresult = run_lab( + LabConfig( + source_root=empty, + sandbox_root=tmp / "sbxC2", + engine=engine, + confirm_sandbox_destructive=True, + rounds=1, + duration_s=5.0, + ) + ) _check(zresult.report_path.exists(), "zero-target report not written") zbody = zresult.report_path.read_text() for section in ("# Piranha Lab Report", "Best lineage", "Worst lineage"): @@ -332,6 +371,7 @@ def main(argv: list[str] | None = None) -> int: # --- pytest surface (runs alongside the existing suite) ------------------------ + def test_piranha_model_engine_verify(): passed, reason = verify() assert passed, reason diff --git a/code_covenant/tools/piranha_utility_verify.py b/code_covenant/tools/piranha_utility_verify.py new file mode 100644 index 0000000..5698423 --- /dev/null +++ b/code_covenant/tools/piranha_utility_verify.py @@ -0,0 +1,366 @@ +""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: code_covenant.tools.piranha_utility_verify ║ +# ║ purpose: Run a deterministic check-of-the-check for ║ +# ║ safe, beneficial Piranha mutation campaigns. ║ +# ║ inputs: ║ +# ║ - Optional verification root ║ +# ║ outputs: ║ +# ║ - Greppable PASS/FAIL line and evidence dictionary ║ +# ║ constraints: ║ +# ║ - Offline, deterministic, and source-preserving ║ +# ║ - Exercises the real process-confinement path ║ +# ║ invariants: ║ +# ║ - Useful mutation promotes; deceptive mutation rejects ║ +# ║ side_effects: ║ +# ║ - Creates throwaway source and Piranha sandboxes ║ +# ║ forbidden_changes: ║ +# ║ - Do not replace the production confinement runner ║ +# ║ optimization_targets: ║ +# ║ - [security] adversarial end-to-end proof ║ +# ║ risk_level: high ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: tools ║ +# ║ super_cohort: code_covenant_core ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shlex +import sys +import tempfile +from pathlib import Path +from typing import Any + +from code_covenant.piranha.execution import ( + ConfinementError, + ConfinementPolicy, + run_confined, +) +from code_covenant.piranha.lab import LabConfig, run_lab +from code_covenant.piranha.policy import UtilityPolicy +from code_covenant.piranha.utility_engine import BRANCH_TO_BOOL, UtilityMutationEngine +from code_covenant.portfolio.engines import EngineContext, ScriptedEngine + + +def verify(root: Path, *, check_os_confinement: bool = True) -> dict[str, Any]: + """Run the beneficial and deceptive campaigns and return evidence.""" + root.mkdir(parents=True, exist_ok=True) + source = root / "source" + source.mkdir() + (source / "target.py").write_text(_target_source(), encoding="utf-8") + (source / "test_target.py").write_text(_test_source(), encoding="utf-8") + original_digest = _tree_digest(source) + test_command = f"{shlex.quote(sys.executable)} test_target.py" + policy = UtilityPolicy() + + useful = run_lab( + LabConfig( + source_root=source, + sandbox_root=root / "useful-sandbox", + engine=UtilityMutationEngine(), + confirm_sandbox_destructive=True, + rounds=1, + duration_s=10.0, + max_attempts_per_round=1, + test_command=test_command, + utility_policy=policy, + require_process_confinement=True, + ) + ) + _check( + useful.total_merges == 1, + f"useful candidate did not merge: {useful.ledger_entries[-1:]}", + ) + useful_entry = useful.ledger_entries[-1] + _check(useful_entry.get("decision") == "merge", "useful candidate was not promoted") + _check( + useful_entry.get("mutation", {}).get("strategy") == BRANCH_TO_BOOL, + "unexpected useful strategy", + ) + _check( + useful_entry.get("utility_evaluation", {}).get("improved") is True, + "useful candidate lacks positive utility evidence", + ) + _check(_tree_digest(source) == original_digest, "host source changed after useful run") + + sentinel = root / "outside-write.txt" + + def _deceptive(ctx: EngineContext) -> str: + needle = "def is_ready(value: object) -> bool:\n" + payload = needle + f" open({str(sentinel)!r}, 'w').write('escaped')\n" + return ctx.before_source.replace(needle, payload, 1) + + deceptive = run_lab( + LabConfig( + source_root=source, + sandbox_root=root / "deceptive-sandbox", + engine=ScriptedEngine(_deceptive, name="deceptive-verifier"), + confirm_sandbox_destructive=True, + rounds=1, + duration_s=10.0, + max_attempts_per_round=1, + test_command=test_command, + utility_policy=policy, + require_process_confinement=True, + ) + ) + _check(deceptive.total_merges == 0, "capability-expanding candidate merged") + deceptive_entry = deceptive.ledger_entries[-1] + _check(deceptive_entry.get("decision") == "reject", "deceptive candidate not rejected") + _check( + "new code capabilities" in str(deceptive_entry.get("reason", "")), + "deceptive rejection lacks capability evidence", + ) + _check(not sentinel.exists(), "deceptive candidate wrote outside its run") + _check(_tree_digest(source) == original_digest, "host source changed after deceptive run") + + confinement_evidence = _verify_confinement(root) if check_os_confinement else {"skipped": True} + + return { + "schema_version": "piranha-utility/evidence-v1", + "status": "PASS", + "useful_report": str(useful.report_path), + "useful_proposal": useful_entry.get("proposal_id"), + "useful_metrics_before": useful_entry.get("metric_before"), + "useful_metrics_after": useful_entry.get("metric_after"), + "deceptive_report": str(deceptive.report_path), + "deceptive_proposal": deceptive_entry.get("proposal_id"), + "confinement": confinement_evidence, + "source_digest": original_digest, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m code_covenant.tools.piranha_utility_verify", + description="Run Piranha's beneficial-mutation and confinement proofs.", + ) + parser.add_argument( + "--root", + type=Path, + help="Persist the verification cell and evidence JSON at a new directory.", + ) + args = parser.parse_args(argv) + try: + if args.root is None: + with tempfile.TemporaryDirectory(prefix="piranha_utility_verify_") as raw: + evidence = verify(Path(raw)) + else: + root = args.root.expanduser().absolute() + if root.is_symlink() or root.exists(): + raise ValueError(f"persistent verification root must be new: {root}") + evidence = verify(root) + evidence_path = root / "piranha_utility_evidence.json" + evidence_path.write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + evidence["evidence_path"] = str(evidence_path) + print(json.dumps(evidence, sort_keys=True)) + except Exception as exc: # noqa: BLE001 - verifier emits one terminal status + print(f"PIRANHA-UTILITY-VERIFY: FAIL: {exc}") + return 1 + print("PIRANHA-UTILITY-VERIFY: PASS") + return 0 + + +def _target_source() -> str: + return '''""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: verify.target ║ +# ║ purpose: Return the truth value of an object. ║ +# ║ inputs: ║ +# ║ - value: object ║ +# ║ outputs: ║ +# ║ - result: bool ║ +# ║ constraints: ║ +# ║ - Preserve Python truthiness semantics. ║ +# ║ invariants: ║ +# ║ - Public signature is stable. ║ +# ║ side_effects: ║ +# ║ - None ║ +# ║ forbidden_changes: ║ +# ║ - Do not add filesystem, network, or process effects. ║ +# ║ optimization_targets: ║ +# ║ - [clarity] reduce incidental control flow ║ +# ║ risk_level: low ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: verifier ║ +# ║ super_cohort: piranha ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +def is_ready(value: object) -> bool: + if value: + return True + else: + return False +''' + + +def _test_source() -> str: + return """from target import is_ready + +assert is_ready(1) is True +assert is_ready([]) is False +assert is_ready("ready") is True +""" + + +def _tree_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + digest.update(str(path.relative_to(root)).encode("utf-8")) + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def _verify_confinement(root: Path) -> dict[str, bool]: + cell = root / "confinement-cell" + cell.mkdir() + runtime = cell / "runtime" + runtime.mkdir() + runtime = runtime.resolve() + secret = root / "outside-secret.txt" + outside_write = root / "outside-cell-write.txt" + declared_read = cell / "declared-read.txt" + secret.write_text("do-not-read", encoding="utf-8") + declared_read.write_text("read-only", encoding="utf-8") + probe = cell / "probe.py" + probe.write_text( + "import json, os, pathlib, socket, subprocess, sys\n" + f"secret = pathlib.Path({str(secret)!r})\n" + f"outside = pathlib.Path({str(outside_write)!r})\n" + f"declared = pathlib.Path({str(declared_read.resolve())!r})\n" + f"allowed = pathlib.Path({str(runtime / 'allowed.txt')!r})\n" + "result = {}\n" + "try:\n" + " secret.read_text()\n" + " result['secret_read_blocked'] = False\n" + "except OSError:\n" + " result['secret_read_blocked'] = True\n" + "try:\n" + " outside.write_text('escaped')\n" + " result['outside_write_blocked'] = False\n" + "except OSError:\n" + " result['outside_write_blocked'] = True\n" + "try:\n" + " link = allowed.parent / 'outside-link'\n" + " link.symlink_to(outside)\n" + " link.write_text('escaped')\n" + " result['symlink_write_blocked'] = False\n" + "except OSError:\n" + " result['symlink_write_blocked'] = True\n" + "try:\n" + " hardlink = allowed.parent / 'declared-hardlink'\n" + " os.link(declared, hardlink)\n" + " hardlink.write_text('mutated')\n" + " result['hardlink_write_blocked'] = declared.read_text() == 'read-only'\n" + "except OSError:\n" + " result['hardlink_write_blocked'] = declared.read_text() == 'read-only'\n" + "try:\n" + " sock = socket.socket()\n" + " sock.connect(('127.0.0.1', 9))\n" + " result['network_blocked'] = False\n" + "except OSError:\n" + " result['network_blocked'] = True\n" + "try:\n" + " child = subprocess.run([sys.executable, '-c', 'pass'], check=False)\n" + " result['child_process_blocked'] = child.returncode != 0\n" + "except OSError:\n" + " result['child_process_blocked'] = True\n" + "allowed.parent.mkdir(parents=True, exist_ok=True)\n" + "allowed.write_text('allowed')\n" + "result['declared_write_succeeded'] = allowed.read_text() == 'allowed'\n" + "print(json.dumps(result, sort_keys=True))\n", + encoding="utf-8", + ) + completed = run_confined( + [sys.executable, str(probe.resolve())], + cwd=cell, + timeout_s=10.0, + policy=ConfinementPolicy( + read_roots=(cell,), + write_root=runtime, + require_completion_receipt=True, + ), + ) + _check(completed.returncode == 0, f"confinement probe failed: {completed.stderr}") + evidence = json.loads(completed.stdout.strip()) + expected_true = { + "secret_read_blocked", + "outside_write_blocked", + "symlink_write_blocked", + "hardlink_write_blocked", + "network_blocked", + "child_process_blocked", + "declared_write_succeeded", + } + _check( + all(evidence.get(key) is True for key in expected_true), + f"confinement probe exposed authority: {evidence}", + ) + _check(not outside_write.exists(), "confinement probe wrote outside its cell") + _check( + declared_read.read_text(encoding="utf-8") == "read-only", + "confinement probe modified a declared read-only file through a link", + ) + evidence["completion_artifacts_cleaned"] = not any( + cell.glob(".piranha-control-*") + ) + _check( + evidence["completion_artifacts_cleaned"] is True, + "completion control artifacts were not cleaned", + ) + evidence["false_success_blocked"] = _verify_false_success_receipt(root) + _check(evidence["false_success_blocked"] is True, "false success lacked a receipt") + return { + key: bool(evidence[key]) + for key in sorted( + expected_true | {"completion_artifacts_cleaned", "false_success_blocked"} + ) + } + + +def _verify_false_success_receipt(root: Path) -> bool: + cell = root / "completion-cell" + cell.mkdir() + runtime = cell / "runtime" + probe = cell / "premature_exit.py" + probe.write_text("import os\nos._exit(0)\n", encoding="utf-8") + try: + run_confined( + [sys.executable, str(probe.resolve())], + cwd=cell, + timeout_s=10.0, + policy=ConfinementPolicy( + read_roots=(cell,), + write_root=runtime, + require_completion_receipt=True, + ), + ) + except ConfinementError as exc: + return "completion receipt" in str(exc) + return False + + +def _check(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/PIRANHA_UTILITY_CAMPAIGNS.md b/docs/PIRANHA_UTILITY_CAMPAIGNS.md new file mode 100644 index 0000000..a521fb8 --- /dev/null +++ b/docs/PIRANHA_UTILITY_CAMPAIGNS.md @@ -0,0 +1,179 @@ +# Piranha utility campaigns + +Piranha now separates discovery from promotion. An engine may suggest a +surprising full-file replacement, but it has no authority over tests, metrics, +the ledger, the champion table, or the merge decision. + +## Safe default + +```bash +python -m code_covenant.cli piranha path/to/contractized/project \ + --sandbox-root /tmp/my-piranha-run \ + --i-understand-piranha-is-destructive \ + --engine utility \ + --test-command "python -m pytest -q -p no:cacheprovider" \ + --rounds 3 \ + --max-attempts-per-round 10 +``` + +The source is copied. The original checkout is never the evaluator workspace. +The sandbox root must either not exist or carry Piranha's exact ownership +marker from a prior run. + +The built-in engine currently explores four conservative strategy families: + +- boolean branch collapse; +- lazy guard-return collapse into a bounded conditional expression; +- short-circuit loops converted to `any()`; +- direct or filtered list accumulation converted to comprehensions. + +Each candidate records a stable hash, strategy, novelty tags, touched line +range, source hash, and estimated line reduction. Rejected hashes are fed back +to the engine. The ledger retains the complete seen-hash set across the +campaign while keeping the latest 20 detailed rejection records in the prompt, +so long runs do not cycle through old failures. + +## Promotion law + +A candidate cannot become champion unless all of these are green: + +1. bounded, parseable Python; +2. every CPC block byte-for-byte unchanged; +3. public functions, classes, methods, and signatures unchanged; +4. no new filesystem, network, process, native-code, dynamic-code, + environment, or unsafe-deserialization capability, and no change inside an + authority-bearing function or expression; +5. mandatory behavior tests pass inside the confined evaluator; +6. every required trusted metric is present and finite; +7. strict Pareto improvement over the source that produced the candidate. + +The default trusted metrics are source bytes, branch nodes, and bytecode +instructions, all minimized. They are calculated by the control plane without +executing the candidate, so candidate code cannot omit or forge them. An +optional benchmark command runs in a separate confinement cell and is still a +hard gate: any nonzero exit rejects. If `--require-improvement` is also set, +its benchmark ratchet applies in addition to the built-in utility policy; one +does not replace the other. + +Capability-expanding and non-improving proposals remain evidence, not +champions. Security failures reject before candidate execution; safe but tied +utility results route to review. + +The risk ceiling is also part of promotion law. Utility campaigns default to +`medium`; a broader command-line scope cannot relax that ceiling. A caller may +choose a stricter ceiling, but not a looser one. + +## Metric selection + +The default metric set is deliberately broad. A specialized campaign may +predeclare a smaller trusted set with repeated `--utility-metric` flags when a +CPC names a narrower objective, but the set must be chosen before candidate +generation. Do not remove a regressing metric after seeing a proposal merely +to promote it; that is metric shopping, not improvement. + +For example, this self-hosted default-metric campaign targets a low-risk helper +and uses a stdlib-only behavior oracle: + +```bash +python -m code_covenant.tools.piranha . \ + --sandbox-root /tmp/piranha-logging \ + --allow-repo-root \ + --i-understand-piranha-is-destructive \ + --module code_covenant.logging_setup \ + --test-command "python -m tests.piranha_logging_oracle" \ + --rounds 1 \ + --max-attempts-per-round 1 +``` + +The retained mutation reduces source size while holding branch count and +bytecode instruction count flat. The host checkout remains byte-identical. + +## Behavior-oracle commands + +Receipt-backed behavior tests must start with an explicit Python executable. +Supported forms are `python file.py`, `python -m module`, `python -m pytest`, +and `python -c ...`. The runner regains control after the test and writes the +completion receipt itself. A custom oracle must therefore finish by returning +normally; `raise SystemExit(0)`, `sys.exit(0)`, and abrupt interpreter exit are +treated as incomplete, even though their operating-system status is zero. + +`python -m pytest` is handled specially so a successful pytest session returns +to the receipt runner. Shell wrappers are refused for behavior tests. Benchmark +commands may be non-Python, but do not receive this same completion proof; they +run in their own confined cell and remain protected by the pre-execution +authority-integrity gate. Keep both commands in the copied project and keep +oracles free of credentials. If the selected Python is from a virtual +environment, that explicit environment becomes a read-only runtime dependency; +other home-directory paths remain unavailable. + +## Model-backed discovery + +The gate does not trust a model more than the built-in mutator. A local, +offline model CLI can be used as a discovery engine if it prints the complete +candidate source to standard output: + +```bash +python -m code_covenant.tools.piranha path/to/contractized/project \ + --sandbox-root /tmp/piranha-model-run \ + --i-understand-piranha-is-destructive \ + --engine model \ + --engine-command "local-model-cli --prompt-file {program} --source-file {before}" \ + --test-command "python -m pytest -q -p no:cacheprovider" \ + --rounds 2 +``` + +Available command-template fields are `{before}`, `{after}`, `{cpc}`, +`{program}`, `{workdir}`, and `{target}`. The engine runs with credential +variables scrubbed, network disabled, no child-process authority, and a +sandbox-local working directory. Consequently, a hosted model CLI that needs +network access or ambient authentication will fail closed; use a local model or +place the whole control plane in a separately governed environment. Never put +secrets in the command template. + +## Confinement boundary + +Generation and evaluation are separate authority cells. The macOS evaluator: + +- scrubs ambient credentials from the environment; +- denies reads from user homes, mounted volumes, and neighboring temp runs + except explicitly declared roots; +- denies writes outside its one runtime directory; +- denies network, child processes, hard links, clones, and macOS broker paths + such as LaunchServices, Apple Events, authorization, Mach/XPC, and IOKit; +- requires a trusted-return receipt for behavior tests, so early successful + interpreter exit does not impersonate a completed suite, then removes the + one-run control artifacts; +- supervises the complete process group with wall/CPU/file/output limits. + +On macOS, resident-memory enforcement is supervisory because the platform does +not provide a usable hard address-space limit for CPython. Piranha samples RSS +frequently and terminates over-limit jobs, but hostile native code can allocate +between samples. Use a memory-capped VM/container for native extensions or +truly adversarial generators; this host backend is intended for Python mutation +campaigns, not as a general malware sandbox. + +There is no silent unconfined fallback. On unsupported hosts, run Piranha in a +dedicated VM/container or add a reviewed backend before enabling subprocess +engines. Remote-model networking is not exposed by the Piranha CLI. + +Custom `Engine` objects and `ScriptedEngine` callables embedded directly in a +Python process are trusted control-plane code; they are not magically confined +after they have already entered the host interpreter. Keep untrusted generators +behind the CLI-backed subprocess adapters, or run the entire control plane in a +dedicated VM/container. + +## Proof command + +```bash +python -m code_covenant.tools.piranha_utility_verify + +# Or retain the complete verification cell and evidence JSON: +python -m code_covenant.tools.piranha_utility_verify --root /new/evidence/path +``` + +The verifier must print `PIRANHA-UTILITY-VERIFY: PASS`. It demonstrates a useful +mutation winning, a filesystem-capability mutation being rejected before +execution, source immutability, and live denial of secret reads, outside writes, +networking, child-process creation, symlink and hard-link write-through, and +false-success interpreter exits while a declared runtime write succeeds. It +also proves that completion-control artifacts are cleaned after supervision. diff --git a/docs/TUTORIAL.md b/docs/TUTORIAL.md index 060076d..fd37b04 100644 --- a/docs/TUTORIAL.md +++ b/docs/TUTORIAL.md @@ -405,7 +405,28 @@ Writes: Piranha materializes a fresh copy of your source under a sandbox root, then runs `merge_mode="auto"` portfolio rounds inside it. The original source is byte-identical before and after. The CLI **refuses** to run -without the explicit safety flag. +without the explicit safety flag or an immutable behavior oracle. Put the +oracle in the source tree so the confined evaluator can read it; files without +a CPC are not mutation targets. This small tutorial oracle imports the sandbox +copy of `click`, rather than the installed package: + +```bash +cat > "$ANNOT/piranha_oracle.py" <<'PYEOF' +import importlib.util +import pathlib +import sys + +root = pathlib.Path(__file__).parent +spec = importlib.util.spec_from_file_location( + "click", root / "__init__.py", submodule_search_locations=[str(root)] +) +assert spec is not None and spec.loader is not None +click = importlib.util.module_from_spec(spec) +sys.modules["click"] = click +spec.loader.exec_module(click) +assert click.unstyle(click.style("ok", fg="red")) == "ok" +PYEOF +``` ```bash code-covenant piranha "$ANNOT" \ @@ -413,11 +434,16 @@ code-covenant piranha "$ANNOT" \ --i-understand-piranha-is-destructive \ --engine shell \ --engine-command "$(which python3) /tmp/cc-tutorial/optimizer.py --before {before} --after {after}" \ + --test-command "$(which python3) piranha_oracle.py" \ --rounds 2 \ --duration 30 \ --max-attempts-per-round 3 ``` +Use your complete project test suite for a real campaign. Receipt-backed tests +must be an explicit Python command (`python file.py`, `python -m module`, or +`python -m pytest`), not a shell wrapper. + Outputs (all inside `--sandbox-root`): - `piranha_lab_report.md` — champion state, best lineage, worst lineage, diff --git a/tests/piranha_logging_oracle.py b/tests/piranha_logging_oracle.py new file mode 100644 index 0000000..ca21f72 --- /dev/null +++ b/tests/piranha_logging_oracle.py @@ -0,0 +1,19 @@ +"""Immutable stdlib-only behavior oracle for Piranha logging campaigns.""" + +from __future__ import annotations + +import logging + +from code_covenant.logging_setup import _resolve_level + + +def main() -> None: + assert _resolve_level(verbose=False, quiet=False) == logging.INFO + assert _resolve_level(verbose=True, quiet=False) == logging.DEBUG + assert _resolve_level(verbose=False, quiet=True) == logging.WARNING + assert _resolve_level(verbose=True, quiet=True) == logging.WARNING + print("PIRANHA-LOGGING-ORACLE: PASS") + + +if __name__ == "__main__": + main() diff --git a/tests/piranha_scope_oracle.py b/tests/piranha_scope_oracle.py new file mode 100644 index 0000000..2a90541 --- /dev/null +++ b/tests/piranha_scope_oracle.py @@ -0,0 +1,40 @@ +"""Immutable stdlib-only behavior oracle for self-hosted Piranha scope campaigns.""" + +from __future__ import annotations + +from pathlib import Path + +from code_covenant.portfolio.inventory import TargetRecord +from code_covenant.portfolio.scope import ScopeFilter, apply_scope + + +def _target(module: str, risk: str) -> TargetRecord: + return TargetRecord( + module=module, + file_path=Path(f"{module}.py"), + cpc={ + "module": module, + "cohort": "portfolio", + "super_cohort": "code_covenant_core", + "authority": "draft", + "risk_level": risk, + }, + ) + + +def main() -> None: + low = _target("demo.low", "low") + medium = _target("demo.medium", "medium") + unknown = _target("demo.unknown", "unknown") + targets = [low, medium, unknown] + + assert apply_scope(targets, None) == targets + assert apply_scope(targets, ScopeFilter(max_risk="low")) == [low] + assert apply_scope(targets, ScopeFilter(min_risk="medium")) == [medium, unknown] + assert apply_scope(targets, ScopeFilter(modules=["demo.m*"])) == [medium] + assert apply_scope(targets, ScopeFilter(cohorts=["different"])) == [] + print("PIRANHA-SCOPE-ORACLE: PASS") + + +if __name__ == "__main__": + main() diff --git a/tests/test_gate_improvement.py b/tests/test_gate_improvement.py index b41ded5..3b96fd6 100644 --- a/tests/test_gate_improvement.py +++ b/tests/test_gate_improvement.py @@ -3,10 +3,9 @@ from __future__ import annotations import json +import shlex from pathlib import Path -import pytest - from code_covenant.gate.improvement import ( evaluate_improvement, find_baseline_metrics, @@ -38,6 +37,13 @@ def test_blocks_when_candidate_has_no_metrics(): assert "no metric_after" in verdict.reason +def test_first_baseline_still_requires_candidate_metrics(): + verdict = evaluate_improvement(candidate_metrics={}, baseline_metrics=None) + + assert not verdict.improved + assert "no metric_after" in verdict.reason + + def test_blocks_when_no_shared_metrics(): verdict = evaluate_improvement( candidate_metrics={"throughput": 500.0}, @@ -111,6 +117,37 @@ def test_strict_pareto_blocks_when_one_metric_regresses_even_if_other_improves() assert not verdict.improved +def test_required_metrics_must_be_complete_even_for_first_baseline(): + verdict = evaluate_improvement( + candidate_metrics={"source_bytes": 10.0}, + baseline_metrics=None, + required_metrics=frozenset({"source_bytes", "branch_nodes"}), + ) + assert not verdict.improved + assert "missing required metrics" in verdict.reason + + +def test_required_metrics_reject_non_finite_values(): + verdict = evaluate_improvement( + candidate_metrics={"source_bytes": float("nan")}, + baseline_metrics=None, + required_metrics=frozenset({"source_bytes"}), + ) + assert not verdict.improved + assert "non-finite" in verdict.reason + + +def test_required_metrics_reject_incomplete_historical_baseline(): + verdict = evaluate_improvement( + candidate_metrics={"source_bytes": 9.0, "branch_nodes": 0.0}, + baseline_metrics={"source_bytes": 10.0}, + required_metrics=frozenset({"source_bytes", "branch_nodes"}), + minimize=frozenset({"source_bytes", "branch_nodes"}), + ) + assert not verdict.improved + assert "baseline missing required metrics" in verdict.reason + + # --------------------------------------------------------------------------- # find_baseline_metrics # --------------------------------------------------------------------------- @@ -122,42 +159,125 @@ def test_find_baseline_metrics_returns_none_when_ledger_missing(tmp_path: Path): def test_find_baseline_metrics_returns_none_when_no_merges_for_module(tmp_path: Path): ledger = tmp_path / "ledger.jsonl" - append_ledger_entry(ledger, { - "module": "demo.other", "merged": True, - "metric_after": {"x": 1.0}, "timestamp": "t1", - }) - append_ledger_entry(ledger, { - "module": "demo.mod", "merged": False, - "metric_after": {"x": 2.0}, "timestamp": "t2", - }) + append_ledger_entry( + ledger, + { + "module": "demo.other", + "merged": True, + "metric_after": {"x": 1.0}, + "timestamp": "t1", + }, + ) + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": False, + "metric_after": {"x": 2.0}, + "timestamp": "t2", + }, + ) assert find_baseline_metrics(ledger, "demo.mod") is None def test_find_baseline_metrics_picks_most_recent_merged_for_module(tmp_path: Path): ledger = tmp_path / "ledger.jsonl" - append_ledger_entry(ledger, { - "module": "demo.mod", "merged": True, - "metric_after": {"loop_us": 100.0}, "timestamp": "2026-01-01T00:00:00", - }) - append_ledger_entry(ledger, { - "module": "demo.mod", "merged": False, - "metric_after": {"loop_us": 50.0}, "timestamp": "2026-02-01T00:00:00", - }) - append_ledger_entry(ledger, { - "module": "demo.mod", "merged": True, - "metric_after": {"loop_us": 75.0}, "timestamp": "2026-03-01T00:00:00", - }) + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"loop_us": 100.0}, + "timestamp": "2026-01-01T00:00:00", + }, + ) + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": False, + "metric_after": {"loop_us": 50.0}, + "timestamp": "2026-02-01T00:00:00", + }, + ) + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"loop_us": 75.0}, + "timestamp": "2026-03-01T00:00:00", + }, + ) baseline = find_baseline_metrics(ledger, "demo.mod") assert baseline == {"loop_us": 75.0} +def test_find_baseline_metrics_supports_dedicated_field_with_legacy_fallback( + tmp_path: Path, +): + ledger = tmp_path / "ledger.jsonl" + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"cost": 2.0}, + "timestamp": "t1", + }, + ) + assert find_baseline_metrics( + ledger, + "demo.mod", + metric_field="benchmark_metric_after", + fallback_field="metric_after", + ) == {"cost": 2.0} + + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"source_bytes": 10.0}, + "benchmark_metric_after": {"cost": 1.0}, + "timestamp": "t2", + }, + ) + assert find_baseline_metrics( + ledger, + "demo.mod", + metric_field="benchmark_metric_after", + fallback_field="metric_after", + ) == {"cost": 1.0} + + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"source_bytes": 9.0}, + "timestamp": "t3", + }, + ) + assert find_baseline_metrics( + ledger, + "demo.mod", + metric_field="benchmark_metric_after", + fallback_field="metric_after", + ) == {"cost": 1.0} + + def test_find_baseline_metrics_filters_non_numeric(tmp_path: Path): ledger = tmp_path / "ledger.jsonl" - append_ledger_entry(ledger, { - "module": "demo.mod", "merged": True, - "metric_after": {"loop_us": 100.0, "tag": "foo"}, - "timestamp": "t1", - }) + append_ledger_entry( + ledger, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"loop_us": 100.0, "tag": "foo"}, + "timestamp": "t1", + }, + ) baseline = find_baseline_metrics(ledger, "demo.mod") assert baseline == {"loop_us": 100.0} @@ -166,8 +286,8 @@ def test_find_baseline_metrics_tolerates_malformed_lines(tmp_path: Path): ledger = tmp_path / "ledger.jsonl" ledger.parent.mkdir(parents=True, exist_ok=True) ledger.write_text( - '{ broken json\n' - '\n' + "{ broken json\n" + "\n" '{"module": "demo.mod", "merged": true, "metric_after": {"loop_us": 50.0}, "timestamp": "t1"}\n' ) assert find_baseline_metrics(ledger, "demo.mod") == {"loop_us": 50.0} @@ -240,11 +360,15 @@ def test_pipeline_demotes_to_review_when_metric_regresses(tmp_path: Path): target = _make_target(tmp_path) ledger_path = tmp_path / "ledger.jsonl" # Seed a prior merged entry. - append_ledger_entry(ledger_path, { - "module": "demo.mod", "merged": True, - "metric_after": {"loop_us": 50.0}, - "timestamp": "2026-01-01T00:00:00", - }) + append_ledger_entry( + ledger_path, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"loop_us": 50.0}, + "timestamp": "2026-01-01T00:00:00", + }, + ) after = target.read_text().replace("v * 2", "v + v") bench = "python3 -c \"print('METRIC: loop_us 120.0')\"" config = ProposalConfig( @@ -267,11 +391,15 @@ def test_pipeline_merges_when_metric_improves(tmp_path: Path): """Prior baseline at loop_us=100; candidate at loop_us=50 -> merge.""" target = _make_target(tmp_path) ledger_path = tmp_path / "ledger.jsonl" - append_ledger_entry(ledger_path, { - "module": "demo.mod", "merged": True, - "metric_after": {"loop_us": 100.0}, - "timestamp": "2026-01-01T00:00:00", - }) + append_ledger_entry( + ledger_path, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"loop_us": 100.0}, + "timestamp": "2026-01-01T00:00:00", + }, + ) after = target.read_text().replace("v * 2", "v + v") bench = "python3 -c \"print('METRIC: loop_us 50.0')\"" config = ProposalConfig( @@ -292,11 +420,15 @@ def test_pipeline_demotes_when_no_metric_after(tmp_path: Path): """No benchmark configured but improvement required -> review.""" target = _make_target(tmp_path) ledger_path = tmp_path / "ledger.jsonl" - append_ledger_entry(ledger_path, { - "module": "demo.mod", "merged": True, - "metric_after": {"loop_us": 100.0}, - "timestamp": "2026-01-01T00:00:00", - }) + append_ledger_entry( + ledger_path, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"loop_us": 100.0}, + "timestamp": "2026-01-01T00:00:00", + }, + ) after = target.read_text().replace("v * 2", "v + v") config = ProposalConfig( target_path=target, @@ -316,11 +448,15 @@ def test_pipeline_unaffected_when_improvement_gate_off(tmp_path: Path): """Without --require-improvement, regression still merges (existing behavior).""" target = _make_target(tmp_path) ledger_path = tmp_path / "ledger.jsonl" - append_ledger_entry(ledger_path, { - "module": "demo.mod", "merged": True, - "metric_after": {"loop_us": 50.0}, - "timestamp": "2026-01-01T00:00:00", - }) + append_ledger_entry( + ledger_path, + { + "module": "demo.mod", + "merged": True, + "metric_after": {"loop_us": 50.0}, + "timestamp": "2026-01-01T00:00:00", + }, + ) after = target.read_text().replace("v * 2", "v + v") bench = "python3 -c \"print('METRIC: loop_us 120.0')\"" config = ProposalConfig( @@ -357,20 +493,30 @@ def test_iterate_cli_accepts_require_improvement_flag(tmp_path: Path, capsys): "src = pathlib.Path(a.before).read_text().replace('v * 2', 'v + v')\n" "pathlib.Path(a.after).write_text(src)\n" ) - bench = f"{sys.executable} -c \"print('METRIC: loop_us 99.0')\"" - exit_code = iterate_main([ - str(target), - "--engine", "shell", - "--engine-command", - f"{sys.executable} {optimizer} --before {{before}} --after {{after}}", - "--proposals-dir", str(tmp_path / "proposals"), - "--ledger", str(tmp_path / "ledger.jsonl"), - "--max-attempts", "1", - "--merge-mode", "auto", - "--benchmark-command", bench, - "--require-improvement", - "--minimize", "loop_us", - ]) + python = shlex.quote(sys.executable) + bench = f"{python} -c \"print('METRIC: loop_us 99.0')\"" + exit_code = iterate_main( + [ + str(target), + "--engine", + "shell", + "--engine-command", + f"{python} {optimizer} --before {{before}} --after {{after}}", + "--proposals-dir", + str(tmp_path / "proposals"), + "--ledger", + str(tmp_path / "ledger.jsonl"), + "--max-attempts", + "1", + "--merge-mode", + "auto", + "--benchmark-command", + bench, + "--require-improvement", + "--minimize", + "loop_us", + ] + ) captured = capsys.readouterr() assert exit_code == 0, captured.err # First merge with no baseline passes through. diff --git a/tests/test_gate_pipeline.py b/tests/test_gate_pipeline.py index 8a4cd2a..ea91265 100644 --- a/tests/test_gate_pipeline.py +++ b/tests/test_gate_pipeline.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import shlex import sys from pathlib import Path @@ -73,7 +74,9 @@ def _target_file( constraints: list[str] | None = None, ) -> Path: target = tmp_path / "target.py" - target.write_text(_cpc_block(module=module, risk_level=risk_level, constraints=constraints) + body) + target.write_text( + _cpc_block(module=module, risk_level=risk_level, constraints=constraints) + body + ) return target @@ -137,9 +140,7 @@ def test_high_risk_routes_to_review_even_when_clean(tmp_path: Path): def test_default_review_mode_does_not_merge_even_when_clean(tmp_path: Path): - target = _target_file( - tmp_path, "def double(v: int) -> int:\n return v * 2\n" - ) + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") after = target.read_text().replace("return v * 2", "return v + v") result = evaluate_proposal(_config(tmp_path, target, after, merge_mode="review")) assert result.decision == "review" @@ -147,24 +148,24 @@ def test_default_review_mode_does_not_merge_even_when_clean(tmp_path: Path): def test_proposal_folder_contains_all_required_artifacts(tmp_path: Path): - target = _target_file( - tmp_path, "def double(v: int) -> int:\n return v * 2\n" - ) + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") after = target.read_text().replace("return v * 2", "return v + v") result = evaluate_proposal(_config(tmp_path, target, after)) folder = result.folder for name in ( - "before.py", "after.py", "diff.patch", - "metrics.json", "constraint_grade.json", - "decision.json", "rationale.md", + "before.py", + "after.py", + "diff.patch", + "metrics.json", + "constraint_grade.json", + "decision.json", + "rationale.md", ): assert (folder / name).exists(), f"missing artifact: {name}" def test_ledger_records_every_run(tmp_path: Path): - target = _target_file( - tmp_path, "def double(v: int) -> int:\n return v * 2\n" - ) + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") after = target.read_text().replace("return v * 2", "return v + v") evaluate_proposal(_config(tmp_path, target, after)) evaluate_proposal(_config(tmp_path, target, after)) @@ -181,9 +182,7 @@ def test_rollback_restores_target(tmp_path: Path): result = evaluate_proposal(_config(tmp_path, target, after)) assert result.merged is True assert target.read_bytes() != original_bytes - rollback_proposal( - tmp_path / "proposals", result.module, result.proposal_id, target - ) + rollback_proposal(tmp_path / "proposals", result.module, result.proposal_id, target) assert target.read_bytes() == original_bytes @@ -197,35 +196,36 @@ def test_intra_run_checkpoint_is_cleaned_up_after_restore(tmp_path: Path): benchmark gate runs. After the run it is redundant (before.py is the canonical rollback artifact) and must be removed so proposal folders don't accumulate duplicated bytes per attempt.""" - target = _target_file( - tmp_path, "def double(v: int) -> int:\n return v * 2\n" - ) + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") after = target.read_text().replace("return v * 2", "return v + v") - bench_cmd = f"{sys.executable} -c \"print('METRIC: x 1')\"" - result = evaluate_proposal( - _config(tmp_path, target, after, benchmark_command=bench_cmd) - ) + bench_cmd = f"{shlex.quote(sys.executable)} -c \"print('METRIC: x 1')\"" + result = evaluate_proposal(_config(tmp_path, target, after, benchmark_command=bench_cmd)) assert (result.folder / "before.py").exists() assert not (result.folder / "checkpoint.py").exists() def test_test_gate_integration_rejects_failing_tests(tmp_path: Path): - target = _target_file( - tmp_path, "def double(v: int) -> int:\n return v * 2\n" - ) + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") after = target.read_text().replace("return v * 2", "return v + v") - failing_cmd = f"{sys.executable} -c \"import sys; sys.exit(1)\"" - result = evaluate_proposal( - _config(tmp_path, target, after, test_command=failing_cmd) - ) + failing_cmd = f'{shlex.quote(sys.executable)} -c "import sys; sys.exit(1)"' + result = evaluate_proposal(_config(tmp_path, target, after, test_command=failing_cmd)) assert result.decision == "reject" assert target.read_text() != after # rollback happened +def test_benchmark_gate_integration_rejects_nonzero_exit(tmp_path: Path): + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") + before = target.read_text() + after = before.replace("return v * 2", "return v + v") + failing_cmd = f'{shlex.quote(sys.executable)} -c "import sys; sys.exit(7)"' + result = evaluate_proposal(_config(tmp_path, target, after, benchmark_command=failing_cmd)) + assert result.decision == "reject" + assert "benchmark gate failed" in result.reason + assert target.read_text() == before + + def test_metrics_json_contains_expected_keys(tmp_path: Path): - target = _target_file( - tmp_path, "def double(v: int) -> int:\n return v * 2\n" - ) + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") after = target.read_text().replace("return v * 2", "return v + v") result = evaluate_proposal(_config(tmp_path, target, after)) metrics = json.loads((result.folder / "metrics.json").read_text()) @@ -235,21 +235,23 @@ def test_metrics_json_contains_expected_keys(tmp_path: Path): def test_cli_evaluate_reports_decision(tmp_path: Path, capsys): - target = _target_file( - tmp_path, "def double(v: int) -> int:\n return v * 2\n" - ) + target = _target_file(tmp_path, "def double(v: int) -> int:\n return v * 2\n") after_path = tmp_path / "after.py" - after_path.write_text( - target.read_text().replace("return v * 2", "return v + v") + after_path.write_text(target.read_text().replace("return v * 2", "return v + v")) + exit_code = cli_main( + [ + "evaluate", + str(target), + "--after", + str(after_path), + "--proposals-dir", + str(tmp_path / "proposals"), + "--ledger", + str(tmp_path / "ledger.jsonl"), + "--merge-mode", + "auto", + ] ) - exit_code = cli_main([ - "evaluate", - str(target), - "--after", str(after_path), - "--proposals-dir", str(tmp_path / "proposals"), - "--ledger", str(tmp_path / "ledger.jsonl"), - "--merge-mode", "auto", - ]) captured = capsys.readouterr() assert exit_code == 0 assert "decision:" in captured.out @@ -261,24 +263,34 @@ def test_cli_rollback_restores_from_proposal(tmp_path: Path, capsys): target = _target_file(tmp_path, body) original_bytes = target.read_bytes() after_path = tmp_path / "after.py" - after_path.write_text( - target.read_text().replace("return v * 2", "return v + v") + after_path.write_text(target.read_text().replace("return v * 2", "return v + v")) + cli_main( + [ + "evaluate", + str(target), + "--after", + str(after_path), + "--proposals-dir", + str(tmp_path / "proposals"), + "--ledger", + str(tmp_path / "ledger.jsonl"), + "--merge-mode", + "auto", + ] ) - cli_main([ - "evaluate", - str(target), - "--after", str(after_path), - "--proposals-dir", str(tmp_path / "proposals"), - "--ledger", str(tmp_path / "ledger.jsonl"), - "--merge-mode", "auto", - ]) assert target.read_bytes() != original_bytes ledger_entries = read_ledger(tmp_path / "ledger.jsonl") proposal_id = ledger_entries[-1]["proposal_id"] module = ledger_entries[-1]["module"] - exit_code = cli_main([ - "rollback", module, proposal_id, str(target), - "--proposals-dir", str(tmp_path / "proposals"), - ]) + exit_code = cli_main( + [ + "rollback", + module, + proposal_id, + str(target), + "--proposals-dir", + str(tmp_path / "proposals"), + ] + ) assert exit_code == 0 assert target.read_bytes() == original_bytes diff --git a/tests/test_piranha.py b/tests/test_piranha.py index 2fe2f68..d625408 100644 --- a/tests/test_piranha.py +++ b/tests/test_piranha.py @@ -4,17 +4,10 @@ import json import shlex +import subprocess import sys from pathlib import Path -# The interpreter path can contain spaces (e.g. a venv under -# "Application Support/"). ShellEngine shlex.split()s its command template, so an -# unquoted `sys.executable` gets torn into two argv entries and the engine -# subprocess never runs — the sole source of this suite's historical -# environment-dependent flakiness. Quoting it is a determinism fix, not a -# behaviour change: the assertions below are unchanged. -_PY = shlex.quote(sys.executable) - import pytest from code_covenant.piranha.champion import ChampionState, ChampionTable @@ -29,6 +22,14 @@ from code_covenant.portfolio.engines import ShellEngine from code_covenant.tools.piranha import main as cli_main +# The interpreter path can contain spaces (e.g. a venv under +# "Application Support/"). ShellEngine shlex.split()s its command template, so an +# unquoted `sys.executable` gets torn into two argv entries and the engine +# subprocess never runs — the sole source of this suite's historical +# environment-dependent flakiness. Quoting it is a determinism fix, not a +# behaviour change: the assertions below are unchanged. +_PY = shlex.quote(sys.executable) + def _cpc_block(module: str) -> str: return f'''""" @@ -113,9 +114,7 @@ def test_validate_contained_rejects_outside(tmp_path: Path): def test_materialize_sandbox_copies_source(tmp_path: Path): source = _seed_source(tmp_path) sandbox = tmp_path / "sbx" - layout = materialize_sandbox( - SandboxConfig(source_root=source, sandbox_root=sandbox) - ) + layout = materialize_sandbox(SandboxConfig(source_root=source, sandbox_root=sandbox)) assert layout.src.exists() assert (layout.src / "alpha.py").exists() assert layout.root == sandbox.resolve() @@ -126,18 +125,14 @@ def test_materialize_sandbox_copies_source(tmp_path: Path): def test_materialize_sandbox_refuses_overlapping_roots(tmp_path: Path): source = _seed_source(tmp_path) with pytest.raises(SandboxError): - materialize_sandbox( - SandboxConfig(source_root=source, sandbox_root=source / "inside") - ) + materialize_sandbox(SandboxConfig(source_root=source, sandbox_root=source / "inside")) def test_materialize_sandbox_refuses_repo_root_without_flag(tmp_path: Path): source = _seed_source(tmp_path) (source / ".git").mkdir() with pytest.raises(SandboxError): - materialize_sandbox( - SandboxConfig(source_root=source, sandbox_root=tmp_path / "sbx") - ) + materialize_sandbox(SandboxConfig(source_root=source, sandbox_root=tmp_path / "sbx")) def test_materialize_sandbox_allows_repo_root_when_flag_set(tmp_path: Path): @@ -174,7 +169,9 @@ def test_champion_state_tracks_rejection_streak(): def test_champion_table_roundtrip(tmp_path: Path): table = ChampionTable() table.record_attempt( - module="demo.a", proposal_id="p1", decision="merge", + module="demo.a", + proposal_id="p1", + decision="merge", proposal_folder=None, ) path = tmp_path / "state.json" @@ -191,11 +188,46 @@ def test_champion_table_summary_with_no_entries(): def test_analyze_lineages_finds_best_and_worst(): ledger = [ - {"proposal_id": "p1", "parent_proposal_id": None, "module": "a", "decision": "merge", "merged": True, "reason": ""}, - {"proposal_id": "p2", "parent_proposal_id": "p1", "module": "a", "decision": "merge", "merged": True, "reason": ""}, - {"proposal_id": "p3", "parent_proposal_id": "p2", "module": "a", "decision": "reject", "merged": False, "reason": ""}, - {"proposal_id": "p4", "parent_proposal_id": "p3", "module": "a", "decision": "reject", "merged": False, "reason": ""}, - {"proposal_id": "p5", "parent_proposal_id": "p4", "module": "a", "decision": "reject", "merged": False, "reason": ""}, + { + "proposal_id": "p1", + "parent_proposal_id": None, + "module": "a", + "decision": "merge", + "merged": True, + "reason": "", + }, + { + "proposal_id": "p2", + "parent_proposal_id": "p1", + "module": "a", + "decision": "merge", + "merged": True, + "reason": "", + }, + { + "proposal_id": "p3", + "parent_proposal_id": "p2", + "module": "a", + "decision": "reject", + "merged": False, + "reason": "", + }, + { + "proposal_id": "p4", + "parent_proposal_id": "p3", + "module": "a", + "decision": "reject", + "merged": False, + "reason": "", + }, + { + "proposal_id": "p5", + "parent_proposal_id": "p4", + "module": "a", + "decision": "reject", + "merged": False, + "reason": "", + }, ] report = analyze_lineages(ledger) assert report.best_lineage == ["p1", "p2"] @@ -205,7 +237,13 @@ def test_analyze_lineages_finds_best_and_worst(): def test_analyze_lineages_tolerates_missing_parent(): ledger = [ - {"proposal_id": "p1", "parent_proposal_id": "ghost", "module": "a", "decision": "merge", "merged": True}, + { + "proposal_id": "p1", + "parent_proposal_id": "ghost", + "module": "a", + "decision": "merge", + "merged": True, + }, ] report = analyze_lineages(ledger) assert report.total_nodes == 1 @@ -229,9 +267,7 @@ def test_run_lab_end_to_end(tmp_path: Path): source = _seed_source(tmp_path) optimizer = _write_optimizer(tmp_path) engine = ShellEngine( - command_template=( - f"{_PY} {optimizer} --before {{before}} --after {{after}}" - ) + command_template=(f"{_PY} {optimizer} --before {{before}} --after {{after}}") ) config = LabConfig( source_root=source, @@ -255,9 +291,7 @@ def test_run_lab_writes_lab_report_content(tmp_path: Path): source = _seed_source(tmp_path) optimizer = _write_optimizer(tmp_path) engine = ShellEngine( - command_template=( - f"{_PY} {optimizer} --before {{before}} --after {{after}}" - ) + command_template=(f"{_PY} {optimizer} --before {{before}} --after {{after}}") ) config = LabConfig( source_root=source, @@ -280,9 +314,7 @@ def test_run_lab_champion_state_reflects_merges(tmp_path: Path): source = _seed_source(tmp_path) optimizer = _write_optimizer(tmp_path) engine = ShellEngine( - command_template=( - f"{_PY} {optimizer} --before {{before}} --after {{after}}" - ) + command_template=(f"{_PY} {optimizer} --before {{before}} --after {{after}}") ) config = LabConfig( source_root=source, @@ -301,54 +333,112 @@ def test_run_lab_champion_state_reflects_merges(tmp_path: Path): def test_cli_piranha_refuses_without_confirmation_flag(tmp_path: Path, capsys): source = _seed_source(tmp_path) - exit_code = cli_main([ - str(source), - "--sandbox-root", str(tmp_path / "sbx"), - "--engine-command", "echo hi", - ]) + exit_code = cli_main( + [ + str(source), + "--sandbox-root", + str(tmp_path / "sbx"), + "--engine-command", + "echo hi", + ] + ) captured = capsys.readouterr() assert exit_code == 3 assert "--i-understand" in captured.err -def test_cli_piranha_runs_with_confirmation(tmp_path: Path, capsys): - source = _seed_source(tmp_path) - optimizer = _write_optimizer(tmp_path) - exit_code = cli_main([ - str(source), - "--sandbox-root", str(tmp_path / "sbx"), - "--i-understand-piranha-is-destructive", - "--engine-command", - f"{_PY} {optimizer} --before {{before}} --after {{after}}", - "--rounds", "1", - "--duration", "5", - "--max-attempts-per-round", "1", - ]) +def test_cli_piranha_runs_with_confirmation(tmp_path: Path, capsys, monkeypatch): + source = tmp_path / "source" + source.mkdir() + (source / "alpha.py").write_text( + _cpc_block("demo.alpha") + + "def is_ready(value: object) -> bool:\n" + + " if value:\n" + + " return True\n" + + " else:\n" + + " return False\n" + ) + (source / "test_alpha.py").write_text( + "from alpha import is_ready\nassert is_ready(1) is True\nassert is_ready([]) is False\n" + ) + + def _fake_confined(argv, *, cwd, timeout_s, policy): + del policy + return subprocess.run( # noqa: S603 + argv, + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + + monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _fake_confined) + exit_code = cli_main( + [ + str(source), + "--sandbox-root", + str(tmp_path / "sbx"), + "--i-understand-piranha-is-destructive", + "--test-command", + f"{_PY} test_alpha.py", + "--rounds", + "1", + "--duration", + "5", + "--max-attempts-per-round", + "1", + ] + ) captured = capsys.readouterr() assert exit_code == 0 assert "sandbox:" in captured.out assert "report:" in captured.out + assert "merges: 1" in captured.out + + +def test_cli_piranha_refuses_without_behavior_oracle(tmp_path: Path, capsys): + source = _seed_source(tmp_path) + exit_code = cli_main( + [ + str(source), + "--sandbox-root", + str(tmp_path / "sbx"), + "--i-understand-piranha-is-destructive", + ] + ) + captured = capsys.readouterr() + assert exit_code == 2 + assert "--test-command" in captured.err def test_cli_piranha_rejects_missing_source(tmp_path: Path, capsys): - exit_code = cli_main([ - str(tmp_path / "nowhere"), - "--sandbox-root", str(tmp_path / "sbx"), - "--i-understand-piranha-is-destructive", - "--engine-command", "echo hi", - ]) + exit_code = cli_main( + [ + str(tmp_path / "nowhere"), + "--sandbox-root", + str(tmp_path / "sbx"), + "--i-understand-piranha-is-destructive", + "--engine-command", + "echo hi", + ] + ) assert exit_code == 2 def test_cli_piranha_surfaces_sandbox_error(tmp_path: Path, capsys): source = _seed_source(tmp_path) # Point sandbox inside the source => SandboxError - exit_code = cli_main([ - str(source), - "--sandbox-root", str(source / "inside"), - "--i-understand-piranha-is-destructive", - "--engine-command", "echo hi", - ]) + exit_code = cli_main( + [ + str(source), + "--sandbox-root", + str(source / "inside"), + "--i-understand-piranha-is-destructive", + "--test-command", + "python3 -c pass", + ] + ) captured = capsys.readouterr() assert exit_code == 2 assert "sandbox error" in captured.err diff --git a/tests/test_piranha_security.py b/tests/test_piranha_security.py new file mode 100644 index 0000000..06fa165 --- /dev/null +++ b/tests/test_piranha_security.py @@ -0,0 +1,320 @@ +"""Focused fail-closed path tests for Piranha's mutation boundaries.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from code_covenant.gate.folder import ( + build_proposal_folder, + write_proposal_artifacts, +) +from code_covenant.gate.pipeline import rollback_proposal +from code_covenant.piranha.sandbox import ( + SandboxConfig, + SandboxError, + materialize_sandbox, +) +from code_covenant.piranha.execution import ( + ConfinementError, + _cleanup_completion_receipt, + _prepare_completion_receipt, + _read_roots, + _sanitized_env, + _tree_usage, + _verify_completion_receipt, + require_confinement_backend, +) + + +def _source(tmp_path: Path) -> Path: + source = tmp_path / "source" + source.mkdir() + (source / "module.py").write_text("VALUE = 1\n", encoding="utf-8") + return source + + +def test_existing_unowned_sandbox_is_refused_without_deleting_it(tmp_path: Path): + source = _source(tmp_path) + sandbox = tmp_path / "sandbox" + sandbox.mkdir() + sentinel = sandbox / "keep.txt" + sentinel.write_text("not Piranha's", encoding="utf-8") + + with pytest.raises(SandboxError, match="unowned"): + materialize_sandbox(SandboxConfig(source, sandbox)) + + assert sentinel.read_text(encoding="utf-8") == "not Piranha's" + + +def test_owned_sandbox_can_be_safely_refreshed(tmp_path: Path): + source = _source(tmp_path) + sandbox = tmp_path / "sandbox" + first = materialize_sandbox(SandboxConfig(source, sandbox)) + stale = first.root / "stale.txt" + stale.write_text("remove me", encoding="utf-8") + + second = materialize_sandbox(SandboxConfig(source, sandbox)) + + assert not stale.exists() + assert (second.src / "module.py").is_file() + + +def test_owned_sandbox_refresh_handles_read_only_control_artifacts(tmp_path: Path): + source = _source(tmp_path) + sandbox = tmp_path / "sandbox" + first = materialize_sandbox(SandboxConfig(source, sandbox)) + control = first.root / ".piranha-control-stale" + control.mkdir() + (control / "runner.py").write_text("stale", encoding="utf-8") + control.chmod(0o500) + + second = materialize_sandbox(SandboxConfig(source, sandbox)) + + assert not control.exists() + assert (second.src / "module.py").is_file() + + +def test_invalid_ownership_marker_is_refused_without_deleting_root(tmp_path: Path): + source = _source(tmp_path) + sandbox = tmp_path / "sandbox" + layout = materialize_sandbox(SandboxConfig(source, sandbox)) + marker = layout.root / ".piranha-owned-v1" + marker.write_text("not-the-owner-token\n", encoding="utf-8") + sentinel = layout.root / "keep.txt" + sentinel.write_text("keep", encoding="utf-8") + + with pytest.raises(SandboxError, match="invalid ownership marker"): + materialize_sandbox(SandboxConfig(source, sandbox)) + + assert sentinel.read_text(encoding="utf-8") == "keep" + + +def test_sandbox_root_symlink_is_refused(tmp_path: Path): + source = _source(tmp_path) + target = tmp_path / "target" + target.mkdir() + sandbox_link = tmp_path / "sandbox-link" + sandbox_link.symlink_to(target, target_is_directory=True) + + with pytest.raises(SandboxError, match="must not be a symlink"): + materialize_sandbox(SandboxConfig(source, sandbox_link)) + + +def test_dangerous_sandbox_root_is_refused_before_deletion(tmp_path: Path): + source = _source(tmp_path) + with pytest.raises(SandboxError, match="dangerous sandbox_root"): + materialize_sandbox(SandboxConfig(source, Path.home())) + + +def test_source_root_symlink_is_refused(tmp_path: Path): + source = _source(tmp_path) + source_link = tmp_path / "source-link" + source_link.symlink_to(source, target_is_directory=True) + + with pytest.raises(SandboxError, match="source_root must not be a symlink"): + materialize_sandbox(SandboxConfig(source_link, tmp_path / "sandbox")) + + +def test_source_tree_symlink_is_refused_instead_of_followed(tmp_path: Path): + source = _source(tmp_path) + outside = tmp_path / "secret.txt" + outside.write_text("secret", encoding="utf-8") + (source / "linked.py").symlink_to(outside) + + with pytest.raises(SandboxError, match="source contains symlink"): + materialize_sandbox(SandboxConfig(source, tmp_path / "sandbox")) + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFO unsupported") +def test_source_tree_special_file_is_refused(tmp_path: Path): + source = _source(tmp_path) + os.mkfifo(source / "commands.fifo") + + with pytest.raises(SandboxError, match="source contains special file"): + materialize_sandbox(SandboxConfig(source, tmp_path / "sandbox")) + + +def test_git_file_marks_repo_root_and_is_ignored_when_allowed(tmp_path: Path): + source = _source(tmp_path) + (source / ".git").write_text("gitdir: ../actual.git\n", encoding="utf-8") + + with pytest.raises(SandboxError, match="git repo root"): + materialize_sandbox(SandboxConfig(source, tmp_path / "sandbox-refused")) + + layout = materialize_sandbox( + SandboxConfig(source, tmp_path / "sandbox-allowed", allow_repo_root=True) + ) + assert not (layout.src / ".git").exists() + + +@pytest.mark.parametrize( + ("module", "proposal_id"), + [ + ("../escape", "p1"), + ("demo/module", "p1"), + ("demo..module", "p1"), + ("demo.module", "../escape"), + ("demo.module", "/absolute"), + ("demo.module", ".."), + ], +) +def test_proposal_folder_rejects_unsafe_names(tmp_path: Path, module: str, proposal_id: str): + with pytest.raises(ValueError, match="invalid"): + build_proposal_folder(tmp_path / "proposals", module, proposal_id) + + +def test_proposal_folder_rejects_symlinked_module_directory(tmp_path: Path): + proposals = tmp_path / "proposals" + proposals.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (proposals / "demo.module").symlink_to(outside, target_is_directory=True) + + with pytest.raises(ValueError, match="symlink|escapes"): + build_proposal_folder(proposals, "demo.module", "p1") + + assert not (outside / "p1").exists() + + +def test_proposal_folder_rejects_symlinked_proposals_root(tmp_path: Path): + outside = tmp_path / "outside" + outside.mkdir() + proposals = tmp_path / "proposals" + proposals.symlink_to(outside, target_is_directory=True) + + with pytest.raises(ValueError, match="must not be a symlink"): + build_proposal_folder(proposals, "demo.module", "p1") + + +def test_artifact_writer_will_not_follow_preplanted_symlink(tmp_path: Path): + folder = build_proposal_folder(tmp_path / "proposals", "demo.module", "p1") + outside = tmp_path / "outside.py" + outside.write_text("do not replace\n", encoding="utf-8") + (folder / "before.py").symlink_to(outside) + + with pytest.raises(FileExistsError): + write_proposal_artifacts( + folder, + before_source="before\n", + after_source="after\n", + diff_text="diff\n", + metrics={}, + constraint_grade=[], + decision={}, + rationale="reason\n", + ) + + assert outside.read_text(encoding="utf-8") == "do not replace\n" + + +def test_rollback_rejects_module_traversal(tmp_path: Path): + target = tmp_path / "target.py" + target.write_text("original", encoding="utf-8") + with pytest.raises(ValueError, match="invalid CPC module"): + rollback_proposal( + tmp_path / "proposals", + "../../outside", + "proposal-1", + target, + ) + + +def test_missing_confinement_backend_fails_closed(monkeypatch): + monkeypatch.setattr("code_covenant.piranha.execution.sys.platform", "linux") + + with pytest.raises(ConfinementError, match="no supported"): + require_confinement_backend() + + +def test_completion_runner_receipts_only_after_regaining_control(tmp_path: Path): + normal_root = tmp_path / "normal" + normal_root.mkdir() + normal = _prepare_completion_receipt(normal_root) + assert normal.control_dir.parent == normal_root.parent + assert normal.control_dir != normal_root + completed = subprocess.run( # noqa: S603 + [sys.executable, normal.runner, "-c", "assert 2 + 2 == 4"], + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0 + _verify_completion_receipt(normal) + + abrupt_root = tmp_path / "abrupt" + abrupt_root.mkdir() + abrupt = _prepare_completion_receipt(abrupt_root) + completed = subprocess.run( # noqa: S603 + [sys.executable, abrupt.runner, "-c", "raise SystemExit(0)"], + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0 + with pytest.raises(ConfinementError, match="completion receipt"): + _verify_completion_receipt(abrupt) + + _cleanup_completion_receipt(normal) + _cleanup_completion_receipt(abrupt) + assert not normal.control_dir.exists() + assert not abrupt.control_dir.exists() + + +def test_confinement_environment_reserves_interpreter_authority(tmp_path: Path): + with pytest.raises(ConfinementError, match="reserved environment key"): + _sanitized_env(tmp_path, {"PYTHONPATH": "/untrusted"}) + with pytest.raises(ConfinementError, match="reserved environment key"): + _sanitized_env(tmp_path, {"DYLD_INSERT_LIBRARIES": "/untrusted"}) + + +def test_selected_virtual_environment_is_a_declared_runtime_dependency(tmp_path: Path): + cwd = tmp_path / "project" + cwd.mkdir() + venv = tmp_path / "chosen-venv" + executable = venv / "bin" / "python" + executable.parent.mkdir(parents=True) + executable.write_text("", encoding="utf-8") + (venv / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8") + + roots = _read_roots( + (), + cwd, + Path(sys.executable).resolve(), + requested_executable=executable, + ) + + assert venv.resolve() in roots + + +def test_unmarked_executable_parent_is_not_granted_as_a_read_root(tmp_path: Path): + cwd = tmp_path / "project" + cwd.mkdir() + executable = tmp_path / "arbitrary" / "bin" / "python" + executable.parent.mkdir(parents=True) + executable.write_text("", encoding="utf-8") + + roots = _read_roots( + (), + cwd, + Path(sys.executable).resolve(), + requested_executable=executable, + ) + + assert executable.parent.parent.resolve() not in roots + + +def test_writable_tree_accounting_does_not_follow_symlinks(tmp_path: Path): + root = tmp_path / "write-root" + root.mkdir() + (root / "small.txt").write_bytes(b"abc") + outside = tmp_path / "outside.bin" + outside.write_bytes(b"x" * 1000) + (root / "outside-link").symlink_to(outside) + (root / "empty-directory").mkdir() + + assert _tree_usage(root) == (3, 3) diff --git a/tests/test_piranha_utility_engine.py b/tests/test_piranha_utility_engine.py new file mode 100644 index 0000000..dee2022 --- /dev/null +++ b/tests/test_piranha_utility_engine.py @@ -0,0 +1,357 @@ +"""Tests for Piranha's deterministic utility mutation engine.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from code_covenant.piranha.utility_engine import ( + BRANCH_TO_BOOL, + GUARD_RETURN_TO_CONDITIONAL, + LIST_ACCUMULATOR_TO_COMPREHENSION, + LOOP_TO_ANY, + MutationCandidate, + UtilityMutationEngine, +) +from code_covenant.portfolio.engines import EngineContext + +_HEADER = '''"""Governed fixture. +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +''' + + +def _context(tmp_path: Path, source: str) -> EngineContext: + return EngineContext( + target_path=tmp_path / "target.py", + before_source=source, + cpc={"module": "demo.target"}, + program="preserve behavior", + workdir=tmp_path / "work", + ) + + +def _function(source: str, name: str) -> Any: + namespace: dict[str, Any] = {} + exec(compile(source, "", "exec"), namespace) # noqa: S102 + return namespace[name] + + +def test_branch_to_bool_is_typed_stable_and_preserves_header(tmp_path: Path): + source = ( + _HEADER + + """def is_ready(value: object) -> bool: + if value: + return True + else: + return False +""" + ) + engine = UtilityMutationEngine() + ctx = _context(tmp_path, source) + + candidates = engine.candidates(ctx) + result = engine.propose(ctx) + repeated = engine.propose(ctx) + + assert len(candidates) == 1 + assert isinstance(candidates[0], MutationCandidate) + assert result.ok, result.error + assert result.after_source is not None + assert result.after_source.startswith(_HEADER) + assert "return bool(value)" in result.after_source + assert result.extra["strategy"] == BRANCH_TO_BOOL + assert result.extra["candidate_hash"] == repeated.extra["candidate_hash"] + assert len(result.extra["candidate_hash"]) == 64 + assert "cpc-byte-preserving" in result.extra["novelty_tags"] + mutated = _function(result.after_source, "is_ready") + assert mutated([1]) is True + assert mutated([]) is False + + +def test_adjacent_boolean_fallback_becomes_one_return(tmp_path: Path): + source = ( + _HEADER + + """def is_blocked(value: object) -> bool: + if value: + return False + return True +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert result.ok, result.error + assert result.after_source is not None + assert "return not (value)" in result.after_source + assert "guard-clause-collapse" in result.extra["novelty_tags"] + original = _function(source, "is_blocked") + mutated = _function(result.after_source, "is_blocked") + for value in (None, [], [1], 0, 1): + assert mutated(value) == original(value) + + +def test_adjacent_boolean_fallback_refuses_intervening_comment(tmp_path: Path): + source = ( + _HEADER + + """def is_ready(value: object) -> bool: + if value: + return True + # This rationale must remain attached to the fallback. + return False +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert not result.ok + assert result.after_source is None + + +def test_guard_return_becomes_lazy_conditional(tmp_path: Path): + source = ( + _HEADER + + """def level(verbose: bool) -> int: + if verbose: + return 10 + return 20 +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert result.ok, result.error + assert result.after_source is not None + assert result.extra["strategy"] == GUARD_RETURN_TO_CONDITIONAL + assert "return 10 if verbose else 20" in result.after_source + assert "lazy-arm-preserving" in result.extra["novelty_tags"] + original = _function(source, "level") + mutated = _function(result.after_source, "level") + assert mutated(True) == original(True) + assert mutated(False) == original(False) + + +def test_guard_return_refuses_overlong_replacement(tmp_path: Path): + long_name = "a_very_long_fallback_name_that_would_make_the_replacement_exceed_the_line_limit" + source = ( + _HEADER + + f"""def choose(value: bool, {long_name}: object) -> object: + if value: + return {long_name} + return {long_name} +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert not result.ok + assert result.after_source is None + + +def test_guard_return_keeps_unselected_arm_lazy(tmp_path: Path): + source = ( + _HEADER + + """calls: list[str] = [] + +def mark(label: str) -> str: + calls.append(label) + return label + +def choose(primary: bool) -> str: + if primary: + return mark("primary") + return mark("fallback") +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert result.ok, result.error + assert result.after_source is not None + mutated = _function(result.after_source, "choose") + assert mutated(True) == "primary" + assert mutated.__globals__["calls"] == ["primary"] + + +def test_loop_to_any_preserves_short_circuit_behavior(tmp_path: Path): + source = ( + _HEADER + + """def contains_even(values: list[int]) -> bool: + for value in values: + if value % 2 == 0: + return True + return False +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert result.ok, result.error + assert result.after_source is not None + assert result.extra["strategy"] == LOOP_TO_ANY + assert "return any(value % 2 == 0 for value in values)" in result.after_source + original = _function(source, "contains_even") + mutated = _function(result.after_source, "contains_even") + for values in ([], [1, 3], [1, 4, 7]): + assert mutated(values) == original(values) + + +def test_loop_to_any_refuses_target_observed_by_finally(tmp_path: Path): + source = ( + _HEADER + + """observed: list[int] = [] + +def contains(values: list[int]) -> bool: + try: + for value in values: + if value: + return True + return False + finally: + observed.append(value) +""" + ) + + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert not result.ok + assert result.after_source is None + + +def test_list_accumulator_becomes_filtered_comprehension(tmp_path: Path): + source = ( + _HEADER + + """def even_squares(values: list[int]) -> list[int]: + result = [] + for value in values: + if value % 2 == 0: + result.append(value * value) + return result +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert result.ok, result.error + assert result.after_source is not None + assert result.extra["strategy"] == LIST_ACCUMULATOR_TO_COMPREHENSION + assert "return [value * value for value in values if value % 2 == 0]" in (result.after_source) + assert "filter-fusion" in result.extra["novelty_tags"] + original = _function(source, "even_squares") + mutated = _function(result.after_source, "even_squares") + assert mutated([-2, -1, 0, 3, 4]) == original([-2, -1, 0, 3, 4]) + + +def test_plain_list_accumulator_becomes_comprehension(tmp_path: Path): + source = ( + _HEADER + + """def squares(values: list[int]) -> list[int]: + result = [] + for value in values: + result.append(value * value) + return result +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert result.ok, result.error + assert result.after_source is not None + assert result.extra["strategy"] == LIST_ACCUMULATOR_TO_COMPREHENSION + assert "return [value * value for value in values]" in result.after_source + + +def test_list_comprehension_refuses_captured_accumulator(tmp_path: Path): + source = ( + _HEADER + + """def collect(values: list[int]) -> list[object]: + def capture() -> list[object]: + return result + + result = [] + for value in values: + result.append(capture()) + return result +""" + ) + + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert not result.ok + assert result.after_source is None + + +def test_feedback_skips_seen_hash_and_selects_next_candidate(tmp_path: Path): + source = ( + _HEADER + + """def is_ready(value: object) -> bool: + if value: + return True + else: + return False + +def contains_even(values: list[int]) -> bool: + for value in values: + if value % 2 == 0: + return True + return False +""" + ) + engine = UtilityMutationEngine() + ctx = _context(tmp_path, source) + first = engine.propose(ctx) + assert first.ok + setattr( + ctx, + "feedback", + ( + { + "decision": "reject", + "mutation": {"candidate_hash": first.extra["candidate_hash"]}, + }, + ), + ) + + second = engine.propose(ctx) + + assert second.ok, second.error + assert first.extra["strategy"] == BRANCH_TO_BOOL + assert second.extra["strategy"] == LOOP_TO_ANY + assert second.extra["candidate_hash"] != first.extra["candidate_hash"] + + +def test_comment_in_patch_range_is_preserved_by_refusing_candidate(tmp_path: Path): + source = ( + _HEADER + + """def squares(values: list[int]) -> list[int]: + result = [] + for value in values: + # This rationale must survive mutation discovery. + result.append(value * value) + return result +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert not result.ok + assert result.after_source is None + assert "no unseen" in result.error + + +def test_loop_to_any_refuses_shadowed_builtin(tmp_path: Path): + source = ( + _HEADER + + """def contains(values: list[int], any: object) -> bool: + for value in values: + if value: + return True + return False +""" + ) + result = UtilityMutationEngine().propose(_context(tmp_path, source)) + + assert not result.ok + assert result.after_source is None + + +def test_invalid_source_is_a_failed_result_not_an_exception(tmp_path: Path): + result = UtilityMutationEngine().propose(_context(tmp_path, "def broken(:\n pass\n")) + + assert not result.ok + assert result.after_source is None + assert "not valid Python" in result.error diff --git a/tests/test_piranha_utility_gate.py b/tests/test_piranha_utility_gate.py new file mode 100644 index 0000000..37ba1d5 --- /dev/null +++ b/tests/test_piranha_utility_gate.py @@ -0,0 +1,631 @@ +"""Security and promotion tests for Piranha utility campaigns.""" + +from __future__ import annotations + +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + +from code_covenant.gate.ledger import append_ledger_entry +from code_covenant.gate.pipeline import ProposalConfig, evaluate_proposal +from code_covenant.gate.utility import evaluate_static_utility +from code_covenant.piranha.lab import LabConfig, _effective_scope +from code_covenant.piranha.policy import UtilityPolicy +from code_covenant.piranha.utility_engine import UtilityMutationEngine +from code_covenant.portfolio.engines import EngineContext, ScriptedEngine +from code_covenant.portfolio.scope import ScopeFilter +from code_covenant.tools.piranha_utility_verify import verify + + +_HEADER = '''""" +# ╔══════════════════════════════════════════════════════════════╗ +# ║ CPC: Code Purpose Contract ║ +# ╠══════════════════════════════════════════════════════════════╣ +# ║ module: demo.target ║ +# ║ purpose: Return the truth value of an object. ║ +# ║ inputs: ║ +# ║ - value: object ║ +# ║ outputs: ║ +# ║ - result: bool ║ +# ║ constraints: ║ +# ║ - Preserve Python truthiness semantics. ║ +# ║ invariants: ║ +# ║ - Public signature is stable. ║ +# ║ side_effects: ║ +# ║ - None ║ +# ║ forbidden_changes: ║ +# ║ - Do not add filesystem, network, or process effects. ║ +# ║ optimization_targets: ║ +# ║ - [clarity] reduce incidental control flow ║ +# ║ risk_level: low ║ +# ║ authority: draft ║ +# ║ source_basis: human_authored ║ +# ║ language: python ║ +# ║ cohort: verifier ║ +# ║ super_cohort: piranha ║ +# ╚══════════════════════════════════════════════════════════════╝ +""" + +''' + + +def _before() -> str: + return ( + _HEADER + + """def is_ready(value: object) -> bool: + if value: + return True + else: + return False +""" + ) + + +def _after(tmp_path: Path) -> str: + result = UtilityMutationEngine().propose( + EngineContext( + target_path=tmp_path / "target.py", + before_source=_before(), + cpc={"module": "demo.target"}, + program="reduce control flow", + workdir=tmp_path / "work", + ) + ) + assert result.ok, result.error + assert result.after_source is not None + return result.after_source + + +def _before_with_os_import() -> str: + return _before().replace( + "def is_ready(value: object) -> bool:\n", + "import os\n\ndef is_ready(value: object) -> bool:\n", + 1, + ) + + +def test_static_utility_is_complete_strict_and_non_executing(tmp_path: Path): + evaluation = evaluate_static_utility(_before(), _after(tmp_path), UtilityPolicy()) + + assert evaluation.safe + assert evaluation.improvement.improved + assert evaluation.promotable + assert evaluation.after_metrics["branch_nodes"] < evaluation.before_metrics["branch_nodes"] + assert evaluation.after_metrics["source_bytes"] < evaluation.before_metrics["source_bytes"] + + +def test_capability_expansion_is_rejected_before_runtime(tmp_path: Path): + candidate = _before().replace( + "def is_ready(value: object) -> bool:\n", + "def is_ready(value: object) -> bool:\n open('/tmp/piranha-escape', 'w').write('x')\n", + ) + + evaluation = evaluate_static_utility(_before(), candidate, UtilityPolicy()) + + assert not evaluation.safe + assert "filesystem" in evaluation.after_capabilities + assert any("new code capabilities" in reason for reason in evaluation.safety_reasons) + + +def test_existing_sensitive_import_does_not_block_legitimate_utility_mutation( + tmp_path: Path, +): + before = _before_with_os_import() + result = UtilityMutationEngine().propose( + EngineContext( + target_path=tmp_path / "target.py", + before_source=before, + cpc={"module": "demo.target"}, + program="reduce control flow", + workdir=tmp_path / "work", + ) + ) + assert result.ok, result.error + assert result.after_source is not None + + evaluation = evaluate_static_utility(before, result.after_source, UtilityPolicy()) + + assert evaluation.safe + assert evaluation.improvement.improved + + +def test_existing_os_import_plus_new_exit_is_authority_change(): + before = _before_with_os_import() + candidate = before.replace( + "def is_ready(value: object) -> bool:\n", + "os._exit(0)\n\ndef is_ready(value: object) -> bool:\n", + 1, + ) + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert evaluation.before_capabilities == evaluation.after_capabilities + assert not evaluation.safe + assert any( + "authority-sensitive code changed" in reason and "os._exit" in reason + for reason in evaluation.safety_reasons + ) + + +@pytest.mark.parametrize( + ("setup", "statement", "expected_surface"), + [ + ("", "eval('1')", "call:eval"), + ("", "exec('pass')", "call:exec"), + ("", "compile('1', '', 'eval')", "call:compile"), + ("", "getattr(value, '__class__')", "call:getattr"), + ("", "setattr(value, 'candidate_flag', True)", "call:setattr"), + ("", "delattr(value, 'candidate_flag')", "call:delattr"), + ( + "import signal\n\n", + "signal.raise_signal(2)", + "call:signal.raise_signal", + ), + ], +) +def test_new_dynamic_or_process_authority_surface_is_rejected( + setup: str, + statement: str, + expected_surface: str, +): + before = _before().replace("def is_ready", setup + "def is_ready", 1) + candidate = before.replace( + "def is_ready(value: object) -> bool:\n", + f"def is_ready(value: object) -> bool:\n {statement}\n", + 1, + ) + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert not evaluation.safe + assert any(expected_surface in reason for reason in evaluation.safety_reasons) + + +def test_termination_surface_arguments_must_remain_exact(): + before = _before().replace( + "def is_ready(value: object) -> bool:\n", + "def stop(code: int) -> None:\n" + " raise SystemExit(code)\n\n" + "def is_ready(value: object) -> bool:\n", + 1, + ) + candidate = before.replace("raise SystemExit(code)", "raise SystemExit(0)", 1) + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert not evaluation.safe + assert any("raise:SystemExit" in reason for reason in evaluation.safety_reasons) + + +def test_new_call_into_transitively_terminating_function_is_rejected(): + before = _before().replace( + "def is_ready(value: object) -> bool:\n", + "import os\n\n" + "def stop() -> None:\n" + " os._exit(0)\n\n" + "def stop_indirectly() -> None:\n" + " stop()\n\n" + "def is_ready(value: object) -> bool:\n", + 1, + ) + candidate = before.replace( + "def is_ready(value: object) -> bool:\n", + "stop_indirectly()\n\ndef is_ready(value: object) -> bool:\n", + 1, + ) + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert not evaluation.safe + assert any("call:stop_indirectly" in reason for reason in evaluation.safety_reasons) + + +def test_aliased_terminating_function_call_is_rejected_even_when_metrics_improve(): + before = _before().replace( + "def is_ready(value: object) -> bool:\n", + "def stop() -> None:\n" + " raise SystemExit(0)\n\n" + "alias = stop\n\n" + "def is_ready(value: object) -> bool:\n", + 1, + ) + candidate = before.replace( + "def is_ready(value: object) -> bool:\n" + " if value:\n" + " return True\n" + " else:\n" + " return False\n", + "alias()\n\ndef is_ready(value: object) -> bool:\n return bool(value)\n", + 1, + ) + + evaluation = evaluate_static_utility( + before, + candidate, + UtilityPolicy(required_metrics=frozenset({"source_bytes", "branch_nodes"})), + ) + + assert evaluation.improvement.improved + assert not evaluation.safe + assert any("call:stop" in reason for reason in evaluation.safety_reasons) + + +def test_existing_io_import_does_not_hide_new_low_level_file_write(): + before = _before().replace("def is_ready", "import io\n\ndef is_ready", 1) + candidate = before.replace( + "def is_ready(value: object) -> bool:\n" + " if value:\n" + " return True\n" + " else:\n" + " return False\n", + "def is_ready(value: object) -> bool:\n" + " if value:\n" + " io.FileIO('/tmp/piranha-write', 'w')\n" + " return bool(value)\n", + 1, + ) + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert not evaluation.safe + assert any("io.FileIO" in reason for reason in evaluation.safety_reasons) + + +def test_dynamic_builtins_lookup_is_authority_sensitive(): + candidate = _before().replace( + "def is_ready(value: object) -> bool:\n", + "__builtins__['exit']()\n\ndef is_ready(value: object) -> bool:\n", + 1, + ) + + evaluation = evaluate_static_utility(_before(), candidate, UtilityPolicy()) + + assert not evaluation.safe + assert any("dynamic-attribute-subscript" in reason for reason in evaluation.safety_reasons) + + +def test_lambda_wrapped_termination_makes_the_enclosing_function_immutable(): + before = _before().replace( + "def is_ready(value: object) -> bool:\n", + "import os\n\n" + "def maybe_stop(enabled: bool) -> None:\n" + " stop = lambda: os._exit(0)\n" + " if enabled:\n" + " stop()\n\n" + "def is_ready(value: object) -> bool:\n", + 1, + ) + candidate = before.replace(" if enabled:\n", " if True:\n", 1) + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert not evaluation.safe + assert any( + "authority-function:maybe_stop" in reason for reason in evaluation.safety_reasons + ) + + +def test_moving_termination_out_of_its_function_is_rejected(): + before = _before().replace( + "def is_ready(value: object) -> bool:\n", + "import os\n\n" + "def dormant_stop() -> None:\n" + " os._exit(0)\n\n" + "def is_ready(value: object) -> bool:\n", + 1, + ) + candidate = before.replace( + "def dormant_stop() -> None:\n os._exit(0)\n", + "os._exit(0)\n\ndef dormant_stop() -> None:\n pass\n", + 1, + ) + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert not evaluation.safe + assert any("authority-function:dormant_stop" in reason for reason in evaluation.safety_reasons) + + +def test_public_signature_change_is_rejected(tmp_path: Path): + del tmp_path + candidate = _before().replace("value: object", "value: object, surprise: bool = False") + + evaluation = evaluate_static_utility(_before(), candidate, UtilityPolicy()) + + assert not evaluation.safe + assert "public API or signature changed" in evaluation.safety_reasons + + +def test_every_cpc_block_is_preserved_byte_for_byte(): + nested_cpc = """ +# ╔════════════════════╗ +# ║ CPC: Nested ║ +# ║ purpose: original ║ +# ╚════════════════════╝ +""" + before = _before() + nested_cpc + candidate = before.replace("purpose: original", "purpose: modified") + + evaluation = evaluate_static_utility(before, candidate, UtilityPolicy()) + + assert not evaluation.safe + assert "one or more CPC block bytes changed" in evaluation.safety_reasons + + +@pytest.mark.parametrize( + ("policy_requires_confinement", "config_requires_confinement"), + [(True, False), (False, True)], +) +def test_pipeline_process_confinement_is_policy_or_config( + tmp_path: Path, + monkeypatch, + policy_requires_confinement: bool, + config_requires_confinement: bool, +): + target = tmp_path / "target.py" + target.write_text(_before(), encoding="utf-8") + (tmp_path / "test_target.py").write_text( + "from target import is_ready\nassert is_ready(1) is True\nassert is_ready([]) is False\n", + encoding="utf-8", + ) + calls: list[list[str]] = [] + + def _fake_confined(argv, *, cwd, timeout_s, policy): + del policy + calls.append(list(argv)) + return subprocess.run( # noqa: S603 + argv, + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + + monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _fake_confined) + result = evaluate_proposal( + ProposalConfig( + target_path=target, + after_source=_after(tmp_path), + proposals_dir=tmp_path / "proposals", + ledger_path=tmp_path / "ledger.jsonl", + cwd=tmp_path, + test_command=f"{shlex.quote(sys.executable)} test_target.py", + merge_mode="auto", + utility_policy=UtilityPolicy(require_process_confinement=policy_requires_confinement), + require_process_confinement=config_requires_confinement, + ) + ) + + assert result.decision == "merge" + assert calls + assert result.ledger_entry["metric_before"] + assert result.ledger_entry["metric_after"] + assert result.ledger_entry["utility_evaluation"]["improved"] is True + + +def test_utility_and_benchmark_ratchets_both_apply_with_separate_cells( + tmp_path: Path, + monkeypatch, +): + target = tmp_path / "target.py" + target.write_text(_before(), encoding="utf-8") + (tmp_path / "test_target.py").write_text( + "from target import is_ready\nassert is_ready(1) is True\n", + encoding="utf-8", + ) + ledger = tmp_path / "ledger.jsonl" + append_ledger_entry( + ledger, + { + "module": "demo.target", + "merged": True, + "benchmark_metric_after": {"cost": 1.0}, + "timestamp": "2026-01-01T00:00:00+00:00", + }, + ) + cells: list[tuple[Path, bool]] = [] + + def _fake_confined(argv, *, cwd, timeout_s, policy): + cells.append((policy.write_root, policy.require_completion_receipt)) + return subprocess.run( # noqa: S603 + argv, + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + + monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _fake_confined) + result = evaluate_proposal( + ProposalConfig( + target_path=target, + after_source=_after(tmp_path), + proposals_dir=tmp_path / "proposals", + ledger_path=ledger, + cwd=tmp_path, + test_command=f"{shlex.quote(sys.executable)} test_target.py", + benchmark_command=(f"{shlex.quote(sys.executable)} -c \"print('METRIC: cost 2')\""), + merge_mode="auto", + utility_policy=UtilityPolicy(), + require_metric_improvement=True, + minimize_metrics=frozenset({"cost"}), + ) + ) + + assert result.decision == "review" + assert "benchmark" not in result.reason + assert "regressed" in result.reason + assert cells == [ + (result.folder / "runtime" / "tests", True), + (result.folder / "runtime" / "benchmark", False), + ] + assert result.ledger_entry["benchmark_metric_before"] == {"cost": 1.0} + assert result.ledger_entry["benchmark_metric_after"] == {"cost": 2.0} + assert result.ledger_entry["utility_evaluation"]["improved"] is True + + +def test_pipeline_never_executes_capability_expansion(tmp_path: Path, monkeypatch): + target = tmp_path / "target.py" + target.write_text(_before(), encoding="utf-8") + candidate = _before().replace( + "def is_ready(value: object) -> bool:\n", + "def is_ready(value: object) -> bool:\n open('/tmp/piranha-escape', 'w').write('x')\n", + ) + called = False + + def _must_not_run(*args, **kwargs): + del args, kwargs + nonlocal called + called = True + raise AssertionError("unsafe candidate reached runtime") + + monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _must_not_run) + result = evaluate_proposal( + ProposalConfig( + target_path=target, + after_source=candidate, + proposals_dir=tmp_path / "proposals", + ledger_path=tmp_path / "ledger.jsonl", + cwd=tmp_path, + test_command=f"{shlex.quote(sys.executable)} -c pass", + merge_mode="auto", + utility_policy=UtilityPolicy(), + require_process_confinement=True, + ) + ) + + assert result.decision == "reject" + assert not called + assert "new code capabilities" in result.reason + + +def test_pipeline_rejects_early_interpreter_exit_before_tests(tmp_path: Path, monkeypatch): + before = _before_with_os_import() + target = tmp_path / "target.py" + target.write_text(before, encoding="utf-8") + candidate = before.replace( + "def is_ready(value: object) -> bool:\n", + "os._exit(0)\n\ndef is_ready(value: object) -> bool:\n", + 1, + ) + called = False + + def _must_not_run(*args, **kwargs): + del args, kwargs + nonlocal called + called = True + raise AssertionError("termination candidate reached runtime") + + monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _must_not_run) + result = evaluate_proposal( + ProposalConfig( + target_path=target, + after_source=candidate, + proposals_dir=tmp_path / "proposals", + ledger_path=tmp_path / "ledger.jsonl", + cwd=tmp_path, + test_command=f"{shlex.quote(sys.executable)} -c pass", + merge_mode="auto", + utility_policy=UtilityPolicy(), + ) + ) + + assert result.decision == "reject" + assert not called + assert "authority-sensitive code changed" in result.reason + assert "os._exit" in result.reason + + +def test_pipeline_never_executes_above_policy_risk(tmp_path: Path, monkeypatch): + before = _before().replace("risk_level: low", "risk_level: medium") + target = tmp_path / "target.py" + target.write_text(before, encoding="utf-8") + candidate_result = UtilityMutationEngine().propose( + EngineContext( + target_path=target, + before_source=before, + cpc={"module": "demo.target"}, + program="reduce control flow", + workdir=tmp_path / "work", + ) + ) + assert candidate_result.ok + assert candidate_result.after_source is not None + called = False + + def _must_not_run(*args, **kwargs): + del args, kwargs + nonlocal called + called = True + raise AssertionError("above-policy candidate reached runtime") + + monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _must_not_run) + result = evaluate_proposal( + ProposalConfig( + target_path=target, + after_source=candidate_result.after_source, + proposals_dir=tmp_path / "proposals", + ledger_path=tmp_path / "ledger.jsonl", + cwd=tmp_path, + test_command=f"{shlex.quote(sys.executable)} -c pass", + merge_mode="auto", + utility_policy=UtilityPolicy(max_risk="low"), + require_process_confinement=True, + ) + ) + + assert result.decision == "reject" + assert not called + assert "utility risk policy" in result.reason + + +def test_utility_policy_rejects_unknown_risk_ceiling(): + with pytest.raises(ValueError, match="max_risk"): + UtilityPolicy(max_risk="mystery") + + +def test_utility_policy_cannot_reverse_builtin_metric_direction(): + with pytest.raises(ValueError, match="must be minimized"): + UtilityPolicy( + required_metrics=frozenset({"source_bytes"}), + minimize_metrics=frozenset(), + ) + + +def test_lab_scope_cannot_relax_utility_risk_ceiling(tmp_path: Path): + config = LabConfig( + source_root=tmp_path / "source", + sandbox_root=tmp_path / "sandbox", + engine=ScriptedEngine(lambda ctx: ctx.before_source), + scope=ScopeFilter(max_risk="critical"), + utility_policy=UtilityPolicy(max_risk="medium"), + ) + + assert _effective_scope(config).max_risk == "medium" + config.scope = ScopeFilter(max_risk="low") + assert _effective_scope(config).max_risk == "low" + + +def test_end_to_end_verifier_with_confined_runner_contract(tmp_path: Path, monkeypatch): + def _fake_confined(argv, *, cwd, timeout_s, policy): + del policy + return subprocess.run( # noqa: S603 + argv, + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + + monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _fake_confined) + + evidence = verify(tmp_path / "verify", check_os_confinement=False) + + assert evidence["useful_proposal"] + assert evidence["deceptive_proposal"] diff --git a/tests/test_portfolio_engines_and_program.py b/tests/test_portfolio_engines_and_program.py index 15a4dff..3352947 100644 --- a/tests/test_portfolio_engines_and_program.py +++ b/tests/test_portfolio_engines_and_program.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shlex import sys from pathlib import Path @@ -15,6 +16,8 @@ from code_covenant.portfolio.llm_engines import ClaudeCodeEngine, CodexEngine from code_covenant.portfolio.program import program_for_cpc +_PY = shlex.quote(sys.executable) + def _ctx(tmp_path: Path, target: Path, before: str, cpc: dict) -> EngineContext: return EngineContext( @@ -45,11 +48,7 @@ def test_shell_engine_reads_after_file_after_subprocess(tmp_path: Path): script = _write_optimizer(tmp_path) target = tmp_path / "target.py" target.write_text("def double(value: int) -> int:\n return value * 2\n") - engine = ShellEngine( - command_template=( - f'{sys.executable} {script} --before {{before}} --after {{after}}' - ) - ) + engine = ShellEngine(command_template=(f"{_PY} {script} --before {{before}} --after {{after}}")) result = engine.propose(_ctx(tmp_path, target, target.read_text(), {"module": "demo"})) assert result.ok, result.error assert result.after_source is not None @@ -59,7 +58,7 @@ def test_shell_engine_reads_after_file_after_subprocess(tmp_path: Path): def test_shell_engine_reports_missing_after_file(tmp_path: Path): # Exit 0 but never write the after file. - engine = ShellEngine(command_template=f'{sys.executable} -c "pass"') + engine = ShellEngine(command_template=f'{_PY} -c "pass"') target = tmp_path / "target.py" target.write_text("def foo(): pass\n") result = engine.propose(_ctx(tmp_path, target, target.read_text(), {})) @@ -68,9 +67,7 @@ def test_shell_engine_reports_missing_after_file(tmp_path: Path): def test_shell_engine_surfaces_nonzero_exit(tmp_path: Path): - engine = ShellEngine( - command_template=f'{sys.executable} -c "import sys; sys.exit(2)"' - ) + engine = ShellEngine(command_template=f'{_PY} -c "import sys; sys.exit(2)"') target = tmp_path / "target.py" target.write_text("def foo(): pass\n") result = engine.propose(_ctx(tmp_path, target, target.read_text(), {})) @@ -88,9 +85,7 @@ def test_shell_engine_rejects_unknown_placeholder(tmp_path: Path): def test_codex_engine_delegates_to_shell(tmp_path: Path): - engine = CodexEngine( - command_template=f'{sys.executable} -c "print(\\"codex-output\\")"' - ) + engine = CodexEngine(command_template=f'{_PY} -c "print(\\"codex-output\\")"') target = tmp_path / "target.py" target.write_text("def f(): pass\n") result = engine.propose(_ctx(tmp_path, target, target.read_text(), {})) @@ -99,9 +94,7 @@ def test_codex_engine_delegates_to_shell(tmp_path: Path): def test_claude_code_engine_delegates_to_shell(tmp_path: Path): - engine = ClaudeCodeEngine( - command_template=f'{sys.executable} -c "print(\\"claude-output\\")"' - ) + engine = ClaudeCodeEngine(command_template=f'{_PY} -c "print(\\"claude-output\\")"') target = tmp_path / "target.py" target.write_text("def f(): pass\n") result = engine.propose(_ctx(tmp_path, target, target.read_text(), {})) @@ -111,7 +104,7 @@ def test_claude_code_engine_delegates_to_shell(tmp_path: Path): def test_shell_engine_stdout_mode_returns_captured_output(tmp_path: Path): engine = ShellEngine( - command_template=f'{sys.executable} -c "print(\\"from-stdout\\")"', + command_template=f'{_PY} -c "print(\\"from-stdout\\")"', output_mode="stdout", ) target = tmp_path / "target.py" @@ -123,7 +116,7 @@ def test_shell_engine_stdout_mode_returns_captured_output(tmp_path: Path): def test_shell_engine_stdout_mode_fails_on_empty_output(tmp_path: Path): engine = ShellEngine( - command_template=f'{sys.executable} -c "pass"', + command_template=f'{_PY} -c "pass"', output_mode="stdout", ) target = tmp_path / "target.py" @@ -135,12 +128,8 @@ def test_shell_engine_stdout_mode_fails_on_empty_output(tmp_path: Path): def test_make_engine_returns_expected_types(): assert isinstance(make_engine("shell", command_template="echo hi"), ShellEngine) - assert isinstance( - make_engine("codex", command_template="codex exec"), CodexEngine - ) - assert isinstance( - make_engine("claude-code", command_template="claude -p"), ClaudeCodeEngine - ) + assert isinstance(make_engine("codex", command_template="codex exec"), CodexEngine) + assert isinstance(make_engine("claude-code", command_template="claude -p"), ClaudeCodeEngine) def test_make_engine_codex_requires_command_template(): diff --git a/tests/test_portfolio_runner.py b/tests/test_portfolio_runner.py index 9dc1aec..c11cf4c 100644 --- a/tests/test_portfolio_runner.py +++ b/tests/test_portfolio_runner.py @@ -5,11 +5,18 @@ import json import sys from pathlib import Path +from types import SimpleNamespace -from code_covenant.gate.ledger import read_ledger -from code_covenant.portfolio.engines import ShellEngine +from code_covenant.gate.ledger import append_ledger_entry, read_ledger +from code_covenant.piranha.policy import UtilityPolicy +from code_covenant.piranha.utility_engine import UtilityMutationEngine +from code_covenant.portfolio.engines import EngineContext, ScriptedEngine, ShellEngine from code_covenant.portfolio.inventory import load_targets -from code_covenant.portfolio.runner import PortfolioConfig, run_portfolio +from code_covenant.portfolio.runner import ( + PortfolioConfig, + _feedback_for_target, + run_portfolio, +) from code_covenant.portfolio.scope import ScopeFilter from code_covenant.tools.portfolio import main as cli_main @@ -85,9 +92,7 @@ def _config(tmp_path: Path, **overrides) -> PortfolioConfig: source = _seed_project(tmp_path) optimizer = _write_optimizer(tmp_path) engine = ShellEngine( - command_template=( - f"{sys.executable} {optimizer} --before {{before}} --after {{after}}" - ) + command_template=(f"{sys.executable} {optimizer} --before {{before}} --after {{after}}") ) defaults = dict( source_root=source, @@ -118,6 +123,101 @@ def test_runner_merges_clean_proposals_under_auto(tmp_path: Path): assert "merge" in decisions +def test_runner_utility_policy_confinement_cannot_be_disabled(tmp_path: Path, monkeypatch): + source = _seed_project(tmp_path) + contexts = [] + proposal_configs = [] + + def _propose(ctx): + contexts.append(ctx) + return ctx.before_source + + def _evaluate(proposal_config): + proposal_configs.append(proposal_config) + return SimpleNamespace( + proposal_id="captured-proposal", + decision="review", + reason="captured", + risk_grade="low", + metrics={}, + constraint_grades=[], + ) + + monkeypatch.setattr("code_covenant.portfolio.runner.evaluate_proposal", _evaluate) + config = PortfolioConfig( + source_root=source, + output_dir=tmp_path / "out", + proposals_dir=tmp_path / "out" / "proposals", + ledger_path=tmp_path / "out" / "ledger.jsonl", + engine=ScriptedEngine(_propose), + max_attempts=1, + utility_policy=UtilityPolicy(require_process_confinement=True), + require_process_confinement=False, + ) + + run_portfolio(config) + + assert contexts[0].require_confinement is True + assert proposal_configs[0].require_process_confinement is True + + +def test_feedback_keeps_all_seen_hashes_beyond_rich_limit(tmp_path: Path): + source = _cpc_block("demo.many") + "\n".join( + f"def check_{index}(value: object) -> bool:\n" + " if value:\n" + " return True\n" + " return False\n" + for index in range(22) + ) + target = tmp_path / "many.py" + target.write_text(source, encoding="utf-8") + context = EngineContext( + target_path=target, + before_source=source, + cpc={"module": "demo.many"}, + program="simplify all branches", + workdir=tmp_path / "work", + ) + engine = UtilityMutationEngine() + candidates = engine.candidates(context) + assert len(candidates) == 22 + + ledger_path = tmp_path / "ledger.jsonl" + for index, candidate in enumerate(candidates[:21]): + append_ledger_entry( + ledger_path, + { + "module": "demo.many", + "proposal_id": f"proposal-{index:02d}", + "decision": "reject", + "reason": "behavior oracle rejected candidate", + "mutation": { + "candidate_hash": candidate.candidate_hash, + "source_hash": candidate.source_hash, + }, + }, + ) + + feedback = _feedback_for_target( + ledger_path, + "demo.many", + source_hash=candidates[0].source_hash, + ) + rich_feedback = [item for item in feedback if item.get("proposal_id")] + seen_summary = next( + item for item in feedback if item.get("feedback_kind") == "seen-candidate-hashes" + ) + context.feedback = feedback + result = engine.propose(context) + + assert len(rich_feedback) == 20 + assert set(seen_summary["seen_candidate_hashes"]) == { + candidate.candidate_hash for candidate in candidates[:21] + } + assert result.ok, result.error + assert result.extra["candidate_hash"] == candidates[21].candidate_hash + + def test_runner_writes_three_artifacts(tmp_path: Path): config = _config(tmp_path, max_attempts=2) result = run_portfolio(config) @@ -141,9 +241,7 @@ def test_runner_stops_at_duration(tmp_path: Path): def test_runner_records_engine_failures_as_dimension(tmp_path: Path): _seed_project(tmp_path) - engine = ShellEngine( - command_template=f'{sys.executable} -c "import sys; sys.exit(1)"' - ) + engine = ShellEngine(command_template=f'{sys.executable} -c "import sys; sys.exit(1)"') config = PortfolioConfig( source_root=tmp_path / "source", output_dir=tmp_path / "out", @@ -205,27 +303,38 @@ def test_heatmap_json_shape_is_stable(tmp_path: Path): def test_cli_portfolio_main_runs_and_exits_zero(tmp_path: Path, capsys): source = _seed_project(tmp_path) optimizer = _write_optimizer(tmp_path) - exit_code = cli_main([ - str(source), - "--output", str(tmp_path / "out"), - "--engine", "shell", - "--engine-command", - f"{sys.executable} {optimizer} --before {{before}} --after {{after}}", - "--duration", "30", - "--max-attempts", "2", - "--merge-mode", "auto", - ]) + exit_code = cli_main( + [ + str(source), + "--output", + str(tmp_path / "out"), + "--engine", + "shell", + "--engine-command", + f"{sys.executable} {optimizer} --before {{before}} --after {{after}}", + "--duration", + "30", + "--max-attempts", + "2", + "--merge-mode", + "auto", + ] + ) captured = capsys.readouterr() assert exit_code == 0 assert "brief:" in captured.out def test_cli_rejects_unknown_source(tmp_path: Path, capsys): - exit_code = cli_main([ - str(tmp_path / "nowhere"), - "--engine", "shell", - "--engine-command", "echo hi", - ]) + exit_code = cli_main( + [ + str(tmp_path / "nowhere"), + "--engine", + "shell", + "--engine-command", + "echo hi", + ] + ) assert exit_code == 2 From ac43e2d85ca29841ae591f0b624275e7f714f8ba Mon Sep 17 00:00:00 2001 From: Cryptosourus Tex <268834908+cryptosourusTex@users.noreply.github.com> Date: Tue, 21 Jul 2026 00:47:25 -0700 Subject: [PATCH 4/5] Fix Piranha release type gate --- code_covenant/gate/utility.py | 8 ++++---- code_covenant/piranha/utility_engine.py | 2 +- code_covenant/portfolio/engines.py | 3 ++- code_covenant/tools/piranha_model_verify.py | 6 ++++-- tests/test_portfolio_engines_and_program.py | 2 ++ 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/code_covenant/gate/utility.py b/code_covenant/gate/utility.py index 573a827..4cd2aa1 100644 --- a/code_covenant/gate/utility.py +++ b/code_covenant/gate/utility.py @@ -524,15 +524,15 @@ def _import_aliases(tree: ast.AST) -> dict[str, str]: assignments.append((node.target.id, node.value)) for _ in range(len(assignments) + 1): changed = False - for target, value in assignments: + for bound_name, value in assignments: if isinstance(value, (ast.Name, ast.Attribute)): resolved = _qualified_name(value, aliases) elif isinstance(value, (ast.Call, ast.Lambda)): - resolved = f"__dynamic_callable__.{target}" + resolved = f"__dynamic_callable__.{bound_name}" else: continue - if resolved and aliases.get(target) != resolved: - aliases[target] = resolved + if resolved and aliases.get(bound_name) != resolved: + aliases[bound_name] = resolved changed = True if not changed: break diff --git a/code_covenant/piranha/utility_engine.py b/code_covenant/piranha/utility_engine.py index 226df9d..3658ec4 100644 --- a/code_covenant/piranha/utility_engine.py +++ b/code_covenant/piranha/utility_engine.py @@ -519,7 +519,7 @@ def _returned_bool(statement: ast.stmt) -> bool | None: value = statement.value if not isinstance(value, ast.Constant) or type(value.value) is not bool: return None - return cast(bool, value.value) + return value.value def _returned_name(statement: ast.stmt) -> str | None: diff --git a/code_covenant/portfolio/engines.py b/code_covenant/portfolio/engines.py index d5b43f7..f895508 100644 --- a/code_covenant/portfolio/engines.py +++ b/code_covenant/portfolio/engines.py @@ -87,7 +87,8 @@ class EngineResult: class Engine(Protocol): """Any callable that turns an EngineContext into an EngineResult.""" - name: str + @property + def name(self) -> str: ... def propose(self, ctx: EngineContext) -> EngineResult: ... diff --git a/code_covenant/tools/piranha_model_verify.py b/code_covenant/tools/piranha_model_verify.py index 830b226..fbda27f 100644 --- a/code_covenant/tools/piranha_model_verify.py +++ b/code_covenant/tools/piranha_model_verify.py @@ -204,7 +204,8 @@ def _scenario_planted_wins(tmp: Path) -> None: ) champ_id = merged[0].get("proposal_id") entry = result.champion_table.entries.get("demo.alpha") - _check(entry is not None, "no champion entry for demo.alpha") + if entry is None: + raise _Fail("no champion entry for demo.alpha") _check( entry.champion_proposal_id == champ_id, f"champion {entry.champion_proposal_id!r} != merged {champ_id!r}", @@ -279,7 +280,8 @@ def _scenario_null_demoted(tmp: Path) -> None: _check(len(merged) == 1, f"expected exactly one merge (the improver), got {len(merged)}") _check(len(reviewed) >= 1, "null variant was not demoted to review") entry = result.champion_table.entries.get("demo.alpha") - _check(entry is not None, "no champion entry for demo.alpha") + if entry is None: + raise _Fail("no champion entry for demo.alpha") _check( entry.champion_proposal_id == merged[0].get("proposal_id"), "the null variant took the champion slot (must not)", diff --git a/tests/test_portfolio_engines_and_program.py b/tests/test_portfolio_engines_and_program.py index 3352947..07e574d 100644 --- a/tests/test_portfolio_engines_and_program.py +++ b/tests/test_portfolio_engines_and_program.py @@ -8,6 +8,7 @@ import pytest +from code_covenant.piranha.utility_engine import UtilityMutationEngine from code_covenant.portfolio.engines import ( EngineContext, ShellEngine, @@ -130,6 +131,7 @@ def test_make_engine_returns_expected_types(): assert isinstance(make_engine("shell", command_template="echo hi"), ShellEngine) assert isinstance(make_engine("codex", command_template="codex exec"), CodexEngine) assert isinstance(make_engine("claude-code", command_template="claude -p"), ClaudeCodeEngine) + assert isinstance(make_engine("utility"), UtilityMutationEngine) def test_make_engine_codex_requires_command_template(): From aae177a543652464755ec534d1e9cc8b753e06a8 Mon Sep 17 00:00:00 2001 From: Cryptosourus Tex <268834908+cryptosourusTex@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:09:23 -0700 Subject: [PATCH 5/5] Make confinement tests portable --- tests/test_piranha.py | 7 +++++++ tests/test_piranha_utility_gate.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/tests/test_piranha.py b/tests/test_piranha.py index d625408..658bd66 100644 --- a/tests/test_piranha.py +++ b/tests/test_piranha.py @@ -277,6 +277,7 @@ def test_run_lab_end_to_end(tmp_path: Path): rounds=2, duration_s=5.0, max_attempts_per_round=2, + require_process_confinement=False, ) result = run_lab(config) assert result.total_attempts >= 1 @@ -301,6 +302,7 @@ def test_run_lab_writes_lab_report_content(tmp_path: Path): rounds=1, duration_s=5.0, max_attempts_per_round=1, + require_process_confinement=False, ) result = run_lab(config) body = result.report_path.read_text() @@ -324,6 +326,7 @@ def test_run_lab_champion_state_reflects_merges(tmp_path: Path): rounds=1, duration_s=5.0, max_attempts_per_round=1, + require_process_confinement=False, ) result = run_lab(config) assert "demo.alpha" in result.champion_table.entries @@ -374,6 +377,10 @@ def _fake_confined(argv, *, cwd, timeout_s, policy): ) monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _fake_confined) + monkeypatch.setattr( + "code_covenant.piranha.execution.require_confinement_backend", + lambda: None, + ) exit_code = cli_main( [ str(source), diff --git a/tests/test_piranha_utility_gate.py b/tests/test_piranha_utility_gate.py index 37ba1d5..543c9fb 100644 --- a/tests/test_piranha_utility_gate.py +++ b/tests/test_piranha_utility_gate.py @@ -624,6 +624,10 @@ def _fake_confined(argv, *, cwd, timeout_s, policy): ) monkeypatch.setattr("code_covenant.piranha.execution.run_confined", _fake_confined) + monkeypatch.setattr( + "code_covenant.piranha.execution.require_confinement_backend", + lambda: None, + ) evidence = verify(tmp_path / "verify", check_os_confinement=False)