diff --git a/CHANGELOG.md b/CHANGELOG.md index 21b9dd7..e17961c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ## Unreleased +### Fixed + +- Rail evaluations charge Jev input tokens only for the Jev backend. Local Laya evaluations keep + `jev_input_tokens` and `cost_usd` at zero, matching the tool and browser fronts. + ### Added - `docs/benchmarks.md`: the Google Flights driver comparison rerun on 2026-09-23 from Poland, every arm three times on diff --git a/s1a/rails.py b/s1a/rails.py index b8941da..2935f18 100644 --- a/s1a/rails.py +++ b/s1a/rails.py @@ -120,7 +120,8 @@ async def evaluate( tp = sum(a and label for a, label in zip(acted, labels)) fp = sum(a and not label for a, label in zip(acted, labels)) fn = sum(label and not a for a, label in zip(acted, labels)) - jev_input_tokens = sum(verdict.input_tokens for verdict in verdicts) + # Local models have no Jev API charges, as on the tool and browser fronts. + jev_input_tokens = sum(verdict.input_tokens for verdict in verdicts) if decision_model.name == "jev" else 0 summary = { "rail": spec.name, "records": len(records), diff --git a/tests/test_rails.py b/tests/test_rails.py index 2fa1fff..a5f10dc 100644 --- a/tests/test_rails.py +++ b/tests/test_rails.py @@ -18,7 +18,7 @@ from s1a import rails from s1a.agents.injection_guard import SPEC as GUARD -from s1a.decision_models import JevModel, ScriptedModel, ScriptedTransport, Usage +from s1a.decision_models import JevModel, LayaModel, ScriptedModel, ScriptedTransport, Usage from s1a.spec import RailSpec, Thresholds INJECTED = ( @@ -204,7 +204,12 @@ async def test_precision_and_recall_of_the_act_band_against_the_labels(self) -> labelled = Path(tmp) / "set.jsonl" labelled.write_text("".join(json.dumps(r) + "\n" for r in records), encoding="utf-8") summary = await rails.evaluate( - GUARD, labelled, decision_model=_noul([0.1, 0.9, 0.95, 0.2, 0.5]), results_dir=Path(tmp) / "results" + GUARD, + labelled, + decision_model=JevModel( + ScriptedTransport(noul=[0.1, 0.9, 0.95, 0.2, 0.5], usage={"input_tokens": 300}, latency_ms=7) + ), + results_dir=Path(tmp) / "results", ) job_dir = Path(summary["job_dir"]) verdicts = [json.loads(line) for line in (job_dir / "verdicts.jsonl").read_text().splitlines()] @@ -217,7 +222,29 @@ async def test_precision_and_recall_of_the_act_band_against_the_labels(self) -> self.assertEqual([v["band"] for v in verdicts], ["allow", "act", "act", "allow", "uncertain"]) self.assertEqual(written["rail"], "injection_guard") self.assertEqual(job_dir.parent, Path(tmp) / "results" / "injection_guard") - self.assertTrue(job_dir.name.endswith("__scripted")) # the model's name, jev or laya on a real run + self.assertTrue(job_dir.name.endswith("__jev")) + + async def test_laya_usage_is_not_charged_as_jev_in_the_returned_or_saved_summary(self) -> None: + agent = SimpleNamespace( + cfg={"max_len": 512}, + system_one=lambda state, questions: { + "answers": {"check": {"noul": 0.9}}, + "usage": {"input_tokens": 300}, + }, + ) + decision_model = LayaModel(agent, model="laya-test") + verdict = await rails.ask(GUARD, {"text": INJECTED}, decision_model) + self.assertEqual(verdict.input_tokens, 300) + with tempfile.TemporaryDirectory() as tmp: + labelled = Path(tmp) / "set.jsonl" + labelled.write_text(json.dumps({"state": {"text": INJECTED}, "label": True}) + "\n", encoding="utf-8") + summary = await rails.evaluate(GUARD, labelled, decision_model=decision_model, results_dir=Path(tmp)) + job_dir = Path(summary["job_dir"]) + written = json.loads((job_dir / "summary.json").read_text(encoding="utf-8")) + for result in (summary, written): + self.assertEqual((result["jev_input_tokens"], result["cost_usd"]), (0, 0.0)) + self.assertEqual((result["records"], result["accuracy"]), (1, 1.0)) + self.assertTrue(job_dir.name.endswith("__laya")) def test_a_record_without_a_boolean_label_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as tmp: