diff --git a/benchmaxxing/extract.py b/benchmaxxing/extract.py index cbf5a90..e7252fa 100644 --- a/benchmaxxing/extract.py +++ b/benchmaxxing/extract.py @@ -87,6 +87,51 @@ def is_abstention(text: str) -> bool: ) +def declared_mcq_choice( + text: str, + options: tuple[str, ...] | list[str], +) -> tuple[str, bool]: + """Return (declared_answer, declared_flag). + + Unlike parse_mcq_choice(), this only recognizes explicit answer + declarations and intentionally ignores fallback heuristics such as + trailing-letter scans or last-mentioned-option resolution. + """ + + if not text: + return "", False + + num_options = len(options) + if num_options == 0: + return "", False + + valid_letters = { + chr(ord("A") + i) + for i in range(min(num_options, 26)) + } + + declarations = [ + _first_group(m) + for m in _DECLARATION_REGEX.finditer(text) + if _first_group(m) in valid_letters + ] + + if declarations: + letter = declarations[-1] + return options[ord(letter) - ord("A")], True + + stripped = text.strip() + + if ( + len(stripped) == 1 + and stripped.upper() in valid_letters + ): + return options[ord(stripped.upper()) - ord("A")], True + + return "", False + + + def _first_group(m: re.Match) -> str: """Return the first non-None group from a regex match with alternations.""" for g in m.groups(): diff --git a/benchmaxxing/gateway.py b/benchmaxxing/gateway.py index 3c2ba4a..44b7b3a 100644 --- a/benchmaxxing/gateway.py +++ b/benchmaxxing/gateway.py @@ -141,7 +141,9 @@ def __init__( timeout: float | None = 60.0, ): self.model = model - self.default_decoding = dict(default_decoding or {}) + self.default_decoding = {"max_output_tokens": 16384} + if default_decoding: + self.default_decoding.update(default_decoding) self.timeout = timeout if client is not None: # Injected client (used by tests): no SDK import required. diff --git a/experiments/referee/referee_self_inconsistency.py b/experiments/referee/referee_self_inconsistency.py new file mode 100644 index 0000000..1406d84 --- /dev/null +++ b/experiments/referee/referee_self_inconsistency.py @@ -0,0 +1,150 @@ +"""Referee self-inconsistency floor (#417). + +Measures whether identical cache-bypassed temperature-0 private re-queries +produce different answers in the absence of committee influence. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from benchmaxxing.data import load_cases +from benchmaxxing.extract import parse_legacy_string, declared_mcq_choice +from experiments.referee.referee_threshold import ( + _Cache, + _key, + _mcq, + HOLDOUT, +) + + + +def build_row(case_id, answer_1, answer_2, declared_1, declared_2): + return { + "case_id": case_id, + "answer_1": answer_1, + "answer_2": answer_2, + "declared_1": declared_1, + "declared_2": declared_2, + "temp0_flip": answer_1 != answer_2, + } + + +def run_one(case, cache): + opts = list(case.options) + prompt, _ = _mcq(case) + + raw_1 = cache.complete( + HOLDOUT, prompt, temperature=0.0, draw=1 + ) + raw_2 = cache.complete( + HOLDOUT, prompt, temperature=0.0, draw=2 + ) + + answer_1 = parse_legacy_string(raw_1, opts) + answer_2 = parse_legacy_string(raw_2, opts) + + _, declared_1 = declared_mcq_choice(raw_1, opts) + _, declared_2 = declared_mcq_choice(raw_2, opts) + + return build_row( + case.case_id, + answer_1, + answer_2, + declared_1, + declared_2, + ) + + +def summarize(rows): + n = len(rows) + + declared_pairs = sum( + 1 + for r in rows + if r["declared_1"] and r["declared_2"] + ) + + undeclared_pairs = sum( + 1 + for r in rows + if not (r["declared_1"] and r["declared_2"]) + ) + + undeclared_draws = sum( + int(not r["declared_1"]) + int(not r["declared_2"]) + for r in rows + ) + + unstable = sum( + 1 + for r in rows + if r["declared_1"] + and r["declared_2"] + and r["temp0_flip"] + ) + + stable = declared_pairs - unstable + + return { + "n": n, + "temperature": 0, + "cache_bypassed": True, + "declared_pairs": declared_pairs, + "undeclared_pairs": undeclared_pairs, + "undeclared_draws": undeclared_draws, + "stable_cases": stable, + "unstable_cases": unstable, + "temp0_self_inconsistency_rate": ( + unstable / declared_pairs + if declared_pairs + else None + ), + } + + +def main(): + ap = argparse.ArgumentParser( + description="Referee self-inconsistency floor (#417)." + ) + ap.add_argument("--manifest", required=True) + ap.add_argument( + "--cache", + default="experiments/referee/results/referee_self_inconsistency_cache.jsonl", + ) + ap.add_argument( + "--out", + default="experiments/referee/results", + ) + ap.add_argument("--n", type=int, default=40) + + args = ap.parse_args() + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + cache = _Cache(args.cache, _key()) + + rows = [ + run_one(case, cache) + for case in load_cases(args.manifest)[:args.n] + ] + + summary = summarize(rows) + summary["new_api_calls_this_run"] = cache.calls + + (out / "referee_self_inconsistency.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows) + ) + + (out / "referee_self_inconsistency_summary.json").write_text( + json.dumps(summary, indent=2) + ) + + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_extract.py b/tests/test_extract.py index c80597e..9ff7fe5 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -169,3 +169,29 @@ def test_leading_article_a_is_not_the_answer(self): text = (r"A patient with these findings is best treated with option C. " r"The final answer is $\boxed{C}$") assert parse_mcq_choice(text, self.OPTS) == 2 # C, never the article-A (0) + + + +# ── Declaration detection (Phase 4B) ─────────────────────────────────── + +def test_declared_mcq_choice_explicit_declaration(): + from benchmaxxing.extract import declared_mcq_choice + + assert declared_mcq_choice("The answer is B.", OPTS_5) == ("B", True) + + +def test_declared_mcq_choice_bare_letter(): + from benchmaxxing.extract import declared_mcq_choice + + assert declared_mcq_choice("C", OPTS_5) == ("C", True) + + +def test_declared_mcq_choice_undeclared_reasoning(): + from benchmaxxing.extract import declared_mcq_choice + + text = ( + "A is unlikely. B is possible. " + "C could fit. D is also plausible." + ) + + assert declared_mcq_choice(text, OPTS_5) == ("", False) diff --git a/tests/test_gateway.py b/tests/test_gateway.py index aed08be..fe1f6f2 100644 --- a/tests/test_gateway.py +++ b/tests/test_gateway.py @@ -193,7 +193,7 @@ def test_gemini_backend_injected_client_multimodal_path(): be.complete("describe", image=b"imgbytes") _, contents, kwargs = client.models.received[0] assert contents == ["describe", b"imgbytes"] - assert "config" not in kwargs # no decoding overrides -> no config passed + assert kwargs["config"] == {"max_output_tokens": 16384} def test_gemini_backend_merges_default_decoding(): @@ -201,7 +201,7 @@ def test_gemini_backend_merges_default_decoding(): be = gateway.GeminiBackend(client=client, default_decoding={"temperature": 0.7, "top_p": 0.9}) be.complete("hi", decoding={"temperature": 0.1}) _, _, kwargs = client.models.received[0] - assert kwargs["config"] == {"temperature": 0.1, "top_p": 0.9} # per-call overrides default + assert kwargs["config"] == {"temperature": 0.1, "top_p": 0.9, "max_output_tokens": 16384} # per-call overrides default def test_timeout_ms_converts_seconds_and_rejects_non_positive(): @@ -264,3 +264,16 @@ def test_gemini_backend_leaves_http_options_unset_when_timeout_disabled(monkeypa captured = _stub_genai(monkeypatch) gateway.GeminiBackend(api_key="k", timeout=None) assert captured["http_options"] is None + + + +def test_gemini_backend_applies_default_max_output_tokens(): + client = _FakeClient() + + be = gateway.GeminiBackend(client=client) + + be.complete("hi") + + _, _, kwargs = client.models.received[0] + + assert kwargs["config"]["max_output_tokens"] == 16384 diff --git a/tests/test_referee_self_inconsistency.py b/tests/test_referee_self_inconsistency.py new file mode 100644 index 0000000..72ed384 --- /dev/null +++ b/tests/test_referee_self_inconsistency.py @@ -0,0 +1,141 @@ +from experiments.referee.referee_self_inconsistency import build_row, summarize + + +def test_summary_all_stable(): + rows = [ + {"temp0_flip": False, "declared_1": True, "declared_2": True}, + {"temp0_flip": False, "declared_1": True, "declared_2": True}, + ] + + result = summarize(rows) + + assert result["n"] == 2 + assert result["stable_cases"] == 2 + assert result["unstable_cases"] == 0 + assert result["temp0_self_inconsistency_rate"] == 0.0 + + +def test_summary_all_unstable(): + rows = [ + {"temp0_flip": True, "declared_1": True, "declared_2": True}, + {"temp0_flip": True, "declared_1": True, "declared_2": True}, + ] + + result = summarize(rows) + + assert result["n"] == 2 + assert result["stable_cases"] == 0 + assert result["unstable_cases"] == 2 + assert result["temp0_self_inconsistency_rate"] == 1.0 + + +def test_summary_mixed(): + rows = [ + {"temp0_flip": True, "declared_1": True, "declared_2": True}, + {"temp0_flip": False, "declared_1": True, "declared_2": True}, + {"temp0_flip": False, "declared_1": True, "declared_2": True}, + {"temp0_flip": True, "declared_1": True, "declared_2": True}, + ] + + result = summarize(rows) + + assert result["n"] == 4 + assert result["stable_cases"] == 2 + assert result["unstable_cases"] == 2 + assert result["temp0_self_inconsistency_rate"] == 0.5 + + +def test_summary_contains_metadata(): + result = summarize([]) + + assert result["n"] == 0 + assert result["temperature"] == 0 + assert result["cache_bypassed"] is True + +def test_summary_single_unstable_case(): + rows = [ + { + "case_id": "medqa-1", + "answer_1": "A", + "answer_2": "B", + "declared_1": True, + "declared_2": True, + "temp0_flip": True, + } + ] + + result = summarize(rows) + + assert result["n"] == 1 + assert result["unstable_cases"] == 1 + assert result["stable_cases"] == 0 + assert result["temp0_self_inconsistency_rate"] == 1.0 + + +def test_build_row_detects_flip(): + row = build_row("medqa-1", "A", "B", True, True) + + assert row["case_id"] == "medqa-1" + assert row["answer_1"] == "A" + assert row["answer_2"] == "B" + assert row["temp0_flip"] is True + + + + +def test_build_row_tracks_declaration_state(): + row = build_row( + "medqa-1", + "A", + "B", + True, + False, + ) + + assert row["declared_1"] is True + assert row["declared_2"] is False + + +def test_summary_counts_declared_pairs_only(): + rows = [ + { + "temp0_flip": True, + "declared_1": True, + "declared_2": True, + }, + { + "temp0_flip": False, + "declared_1": True, + "declared_2": True, + }, + { + "temp0_flip": True, + "declared_1": True, + "declared_2": False, + }, + ] + + result = summarize(rows) + + assert result["declared_pairs"] == 2 + assert result["undeclared_pairs"] == 1 + assert result["undeclared_draws"] == 1 + + assert result["temp0_self_inconsistency_rate"] == 0.5 + + +def test_summary_ignores_undeclared_pairs_in_rate(): + rows = [ + { + "temp0_flip": True, + "declared_1": False, + "declared_2": False, + } + ] + + result = summarize(rows) + + assert result["declared_pairs"] == 0 + assert result["undeclared_pairs"] == 1 + assert result["undeclared_draws"] == 2 + assert result["temp0_self_inconsistency_rate"] is None