From 8f5a859a087aaf1adc9f3833460d8b68f0a024de Mon Sep 17 00:00:00 2001
From: armaanvgrewal <armaanvgrewal@gmail.com>
Date: Sun, 30 Aug 2026 12:44:18 -0500
Subject: [PATCH 1/7] Start #417 referee self-inconsistency experiment scaffold

---
 .../referee/referee_self_inconsistency.py     | 29 ++++++++++++
 tests/test_referee_self_inconsistency.py      | 45 +++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 experiments/referee/referee_self_inconsistency.py
 create mode 100644 tests/test_referee_self_inconsistency.py

diff --git a/experiments/referee/referee_self_inconsistency.py b/experiments/referee/referee_self_inconsistency.py
new file mode 100644
index 0000000..27fbe45
--- /dev/null
+++ b/experiments/referee/referee_self_inconsistency.py
@@ -0,0 +1,29 @@
+"""Referee self-inconsistency floor (#417).
+
+Measures whether identical cache-bypassed temperature-0 private re-queries
+produce different answers in the absence of committee influence.
+"""
+
+from __future__ import annotations
+
+
+def summarize(rows):
+    n = len(rows)
+    unstable = sum(1 for r in rows if r["temp0_flip"])
+
+    return {
+        "n": n,
+        "stable_cases": n - unstable,
+        "unstable_cases": unstable,
+        "temp0_self_inconsistency_rate": (
+            unstable / n if n else None
+        ),
+    }
+
+
+def main():
+    raise NotImplementedError
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/test_referee_self_inconsistency.py b/tests/test_referee_self_inconsistency.py
new file mode 100644
index 0000000..5f194b8
--- /dev/null
+++ b/tests/test_referee_self_inconsistency.py
@@ -0,0 +1,45 @@
+from experiments.referee.referee_self_inconsistency import summarize
+
+
+def test_summary_all_stable():
+    rows = [
+        {"temp0_flip": False},
+        {"temp0_flip": False},
+    ]
+
+    result = summarize(rows)
+
+    assert result["n"] == 2
+    assert result["stable_cases"] == 2
+    assert result["unstable_cases"] == 0
+    assert result["temp0_self_inconsistency_rate"] == 0.0
+
+
+def test_summary_all_unstable():
+    rows = [
+        {"temp0_flip": True},
+        {"temp0_flip": True},
+    ]
+
+    result = summarize(rows)
+
+    assert result["n"] == 2
+    assert result["stable_cases"] == 0
+    assert result["unstable_cases"] == 2
+    assert result["temp0_self_inconsistency_rate"] == 1.0
+
+
+def test_summary_mixed():
+    rows = [
+        {"temp0_flip": True},
+        {"temp0_flip": False},
+        {"temp0_flip": False},
+        {"temp0_flip": True},
+    ]
+
+    result = summarize(rows)
+
+    assert result["n"] == 4
+    assert result["stable_cases"] == 2
+    assert result["unstable_cases"] == 2
+    assert result["temp0_self_inconsistency_rate"] == 0.5

From a7e2b00b87a61097bf367e8f873f4b9beaebf646 Mon Sep 17 00:00:00 2001
From: armaanvgrewal <armaanvgrewal@gmail.com>
Date: Sun, 30 Aug 2026 12:52:05 -0500
Subject: [PATCH 2/7] Add draw-aware cache key for #417

---
 .../referee/referee_self_inconsistency.py     | 26 +++++++++++++++++++
 tests/test_referee_self_inconsistency.py      | 17 ++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/experiments/referee/referee_self_inconsistency.py b/experiments/referee/referee_self_inconsistency.py
index 27fbe45..0e8eead 100644
--- a/experiments/referee/referee_self_inconsistency.py
+++ b/experiments/referee/referee_self_inconsistency.py
@@ -6,6 +6,30 @@
 
 from __future__ import annotations
 
+import hashlib
+import json
+from pathlib import Path
+
+
+
+class _DrawCache:
+    """Replayable cache whose key keeps independent temp-0 draws distinct."""
+
+    def __init__(self, path):
+        self.path = Path(path)
+        self.store = {}
+        if self.path.exists():
+            for line in self.path.read_text().splitlines():
+                if line.strip():
+                    record = json.loads(line)
+                    self.store[record["k"]] = record["resp"]
+
+    @staticmethod
+    def key(model, prompt, draw):
+        return hashlib.sha256(
+            f"{model}\x000.0\x00{draw}\x00{prompt}".encode()
+        ).hexdigest()
+
 
 def summarize(rows):
     n = len(rows)
@@ -13,6 +37,8 @@ def summarize(rows):
 
     return {
         "n": n,
+        "temperature": 0,
+        "cache_bypassed": True,
         "stable_cases": n - unstable,
         "unstable_cases": unstable,
         "temp0_self_inconsistency_rate": (
diff --git a/tests/test_referee_self_inconsistency.py b/tests/test_referee_self_inconsistency.py
index 5f194b8..2bb84ef 100644
--- a/tests/test_referee_self_inconsistency.py
+++ b/tests/test_referee_self_inconsistency.py
@@ -43,3 +43,20 @@ def test_summary_mixed():
     assert result["stable_cases"] == 2
     assert result["unstable_cases"] == 2
     assert result["temp0_self_inconsistency_rate"] == 0.5
+
+
+def test_summary_contains_metadata():
+    result = summarize([])
+
+    assert result["n"] == 0
+    assert result["temperature"] == 0
+    assert result["cache_bypassed"] is True
+
+from experiments.referee.referee_self_inconsistency import _DrawCache
+
+
+def test_draw_cache_distinguishes_draws():
+    k1 = _DrawCache.key("model", "prompt", 1)
+    k2 = _DrawCache.key("model", "prompt", 2)
+
+    assert k1 != k2

From 2062d9ca6eaacb8827fa2f2bdc6cb5ec1b37ff8c Mon Sep 17 00:00:00 2001
From: armaanvgrewal <armaanvgrewal@gmail.com>
Date: Sun, 30 Aug 2026 13:44:26 -0500
Subject: [PATCH 3/7] Implement referee self-inconsistency runner

---
 .../referee/referee_self_inconsistency.py     | 149 +++++++++++++++++-
 tests/test_referee_self_inconsistency.py      |  57 +++++++
 2 files changed, 205 insertions(+), 1 deletion(-)

diff --git a/experiments/referee/referee_self_inconsistency.py b/experiments/referee/referee_self_inconsistency.py
index 0e8eead..057e54f 100644
--- a/experiments/referee/referee_self_inconsistency.py
+++ b/experiments/referee/referee_self_inconsistency.py
@@ -6,10 +6,36 @@
 
 from __future__ import annotations
 
+import argparse
 import hashlib
 import json
+import os
 from pathlib import Path
 
+from benchmaxxing import gateway
+from benchmaxxing.data import load_cases
+from benchmaxxing.extract import parse_legacy_string
+
+
+HOLDOUT = "gemini-2.5-flash-lite"
+
+
+def _key():
+    return os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+
+
+def _letters(n):
+    return [chr(65 + i) for i in range(n)]
+
+
+def _mcq(case, prefix=""):
+    opts = list(case.options)
+    body = "\n".join(f"{L}. {o}" for L, o in zip(_letters(len(opts)), opts))
+    return (
+        f"{prefix}Question: {case.question}\n\nOptions:\n{body}\n\n"
+        "Answer with only the single letter of the best option."
+    ), opts
+
 
 
 class _DrawCache:
@@ -30,6 +56,86 @@ def key(model, prompt, draw):
             f"{model}\x000.0\x00{draw}\x00{prompt}".encode()
         ).hexdigest()
 
+    def get(self, model, prompt, draw):
+        return self.store.get(self.key(model, prompt, draw))
+
+    def put(self, model, prompt, draw, response):
+        k = self.key(model, prompt, draw)
+        self.store[k] = response
+
+        with self.path.open("a") as f:
+            f.write(
+                json.dumps(
+                    {
+                        "k": k,
+                        "model": model,
+                        "draw": draw,
+                        "resp": response,
+                    }
+                )
+                + "\n"
+            )
+
+
+def _query_uncached(model, prompt, api_key):
+    """Always hits the model; never serves from cache."""
+
+    backend = gateway.RetryBackend(
+        gateway.GeminiBackend(model=model, api_key=api_key),
+        tries=5,
+        backoff=3.0,
+    )
+
+    return backend.complete(
+        prompt,
+        decoding={"temperature": 0},
+    )
+
+
+
+def _complete_draw(cache, model, prompt, draw, api_key):
+    """Replay a recorded draw, or make and record a fresh cache-bypassed temp-0 call."""
+    cached = cache.get(model, prompt, draw)
+    if cached is not None:
+        return cached, False
+
+    if not api_key:
+        raise SystemExit(
+            "Draw missing from self-inconsistency cache and no GEMINI_API_KEY set."
+        )
+
+    response = _query_uncached(model, prompt, api_key)
+    cache.put(model, prompt, draw, response)
+    return response, True
+
+
+def build_row(case_id, answer_1, answer_2):
+    return {
+        "case_id": case_id,
+        "answer_1": answer_1,
+        "answer_2": answer_2,
+        "temp0_flip": answer_1 != answer_2,
+    }
+
+
+def run_one(case, cache, api_key):
+    opts = list(case.options)
+
+    prompt, _ = _mcq(case)
+
+    raw_1, call_1 = _complete_draw(
+        cache, HOLDOUT, prompt, draw=1, api_key=api_key
+    )
+    raw_2, call_2 = _complete_draw(
+        cache, HOLDOUT, prompt, draw=2, api_key=api_key
+    )
+
+    answer_1 = parse_legacy_string(raw_1, opts)
+    answer_2 = parse_legacy_string(raw_2, opts)
+
+    row = build_row(case.case_id, answer_1, answer_2)
+
+    return row, int(call_1) + int(call_2)
 
 def summarize(rows):
     n = len(rows)
@@ -48,7 +154,48 @@ def summarize(rows):
 
 
 def main():
-    raise NotImplementedError
+    ap = argparse.ArgumentParser(
+        description="Referee self-inconsistency floor (#417)."
+    )
+    ap.add_argument("--manifest", required=True)
+    ap.add_argument(
+        "--cache",
+        default="experiments/referee/results/referee_self_inconsistency_cache.jsonl",
+    )
+    ap.add_argument(
+        "--out",
+        default="experiments/referee/results",
+    )
+    ap.add_argument("--n", type=int, default=40)
+
+    args = ap.parse_args()
+
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    cache = _DrawCache(args.cache)
+    api_key = _key()
+
+    rows = []
+    new_api_calls = 0
+
+    for case in load_cases(args.manifest)[: args.n]:
+        row, calls = run_one(case, cache, api_key)
+        rows.append(row)
+        new_api_calls += calls
+
+    summary = summarize(rows)
+    summary["new_api_calls_this_run"] = new_api_calls
+
+    (out / "referee_self_inconsistency.jsonl").write_text(
+        "".join(json.dumps(r) + "\n" for r in rows)
+    )
+
+    (out / "referee_self_inconsistency_summary.json").write_text(
+        json.dumps(summary, indent=2)
+    )
+
+    print(json.dumps(summary, indent=2))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_referee_self_inconsistency.py b/tests/test_referee_self_inconsistency.py
index 2bb84ef..75cac15 100644
--- a/tests/test_referee_self_inconsistency.py
+++ b/tests/test_referee_self_inconsistency.py
@@ -60,3 +60,60 @@ def test_draw_cache_distinguishes_draws():
     k2 = _DrawCache.key("model", "prompt", 2)
 
     assert k1 != k2
+
+
+def test_summary_single_unstable_case():
+    rows = [
+        {
+            "case_id": "medqa-1",
+            "answer_1": "A",
+            "answer_2": "B",
+            "temp0_flip": True,
+        }
+    ]
+
+    result = summarize(rows)
+
+    assert result["n"] == 1
+    assert result["unstable_cases"] == 1
+    assert result["stable_cases"] == 0
+    assert result["temp0_self_inconsistency_rate"] == 1.0
+
+
+def test_draw_cache_same_draw_same_key():
+    k1 = _DrawCache.key("model", "prompt", 1)
+    k2 = _DrawCache.key("model", "prompt", 1)
+
+    assert k1 == k2
+
+def test_draw_cache_round_trip(tmp_path):
+    cache = _DrawCache(tmp_path / "cache.jsonl")
+
+    assert cache.get("model", "prompt", 1) is None
+
+    cache.put("model", "prompt", 1, "answer")
+
+    assert cache.get("model", "prompt", 1) == "answer"
+    assert cache.get("model", "prompt", 2) is None
+
+
+from experiments.referee.referee_self_inconsistency import build_row
+
+
+def test_build_row_detects_flip():
+    row = build_row("medqa-1", "A", "B")
+
+    assert row["case_id"] == "medqa-1"
+    assert row["answer_1"] == "A"
+    assert row["answer_2"] == "B"
+    assert row["temp0_flip"] is True
+
+def test_draw_cache_reloads_from_disk(tmp_path):
+    cache_file = tmp_path / "cache.jsonl"
+
+    cache1 = _DrawCache(cache_file)
+    cache1.put("model", "prompt", 1, "answer")
+
+    cache2 = _DrawCache(cache_file)
+
+    assert cache2.get("model", "prompt", 1) == "answer"

From 2c546b2b2303b03bc34a55c5cd50674d6a90bbb2 Mon Sep 17 00:00:00 2001
From: armaanvgrewal <armaanvgrewal@gmail.com>
Date: Sun, 30 Aug 2026 14:40:40 -0500
Subject: [PATCH 4/7] Align #417 with referee threshold cache architecture

---
 .../referee/referee_self_inconsistency.py     | 130 +++---------------
 tests/test_referee_self_inconsistency.py      |  41 +-----
 2 files changed, 19 insertions(+), 152 deletions(-)

diff --git a/experiments/referee/referee_self_inconsistency.py b/experiments/referee/referee_self_inconsistency.py
index 057e54f..568a71e 100644
--- a/experiments/referee/referee_self_inconsistency.py
+++ b/experiments/referee/referee_self_inconsistency.py
@@ -7,107 +7,19 @@
 from __future__ import annotations
 
 import argparse
-import hashlib
 import json
-import os
 from pathlib import Path
 
-from benchmaxxing import gateway
 from benchmaxxing.data import load_cases
 from benchmaxxing.extract import parse_legacy_string
+from experiments.referee.referee_threshold import (
+    _Cache,
+    _key,
+    _mcq,
+    HOLDOUT,
+)
 
 
-HOLDOUT = "gemini-2.5-flash-lite"
-
-
-def _key():
-    return os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
-
-
-def _letters(n):
-    return [chr(65 + i) for i in range(n)]
-
-
-def _mcq(case, prefix=""):
-    opts = list(case.options)
-    body = "\n".join(f"{L}. {o}" for L, o in zip(_letters(len(opts)), opts))
-    return (
-        f"{prefix}Question: {case.question}\n\nOptions:\n{body}\n\n"
-        "Answer with only the single letter of the best option."
-    ), opts
-
-
-
-class _DrawCache:
-    """Replayable cache whose key keeps independent temp-0 draws distinct."""
-
-    def __init__(self, path):
-        self.path = Path(path)
-        self.store = {}
-        if self.path.exists():
-            for line in self.path.read_text().splitlines():
-                if line.strip():
-                    record = json.loads(line)
-                    self.store[record["k"]] = record["resp"]
-
-    @staticmethod
-    def key(model, prompt, draw):
-        return hashlib.sha256(
-            f"{model}\x000.0\x00{draw}\x00{prompt}".encode()
-        ).hexdigest()
-
-    def get(self, model, prompt, draw):
-        return self.store.get(self.key(model, prompt, draw))
-
-    def put(self, model, prompt, draw, response):
-        k = self.key(model, prompt, draw)
-        self.store[k] = response
-
-        with self.path.open("a") as f:
-            f.write(
-                json.dumps(
-                    {
-                        "k": k,
-                        "model": model,
-                        "draw": draw,
-                        "resp": response,
-                    }
-                )
-                + "\n"
-            )
-
-
-def _query_uncached(model, prompt, api_key):
-    """Always hits the model; never serves from cache."""
-
-    backend = gateway.RetryBackend(
-        gateway.GeminiBackend(model=model, api_key=api_key),
-        tries=5,
-        backoff=3.0,
-    )
-
-    return backend.complete(
-        prompt,
-        decoding={"temperature": 0},
-    )
-
-
-
-def _complete_draw(cache, model, prompt, draw, api_key):
-    """Replay a recorded draw, or make and record a fresh cache-bypassed temp-0 call."""
-    cached = cache.get(model, prompt, draw)
-    if cached is not None:
-        return cached, False
-
-    if not api_key:
-        raise SystemExit(
-            "Draw missing from self-inconsistency cache and no GEMINI_API_KEY set."
-        )
-
-    response = _query_uncached(model, prompt, api_key)
-    cache.put(model, prompt, draw, response)
-    return response, True
-
 
 def build_row(case_id, answer_1, answer_2):
     return {
@@ -118,24 +30,22 @@ def build_row(case_id, answer_1, answer_2):
     }
 
 
-def run_one(case, cache, api_key):
+def run_one(case, cache):
     opts = list(case.options)
-
     prompt, _ = _mcq(case)
 
-    raw_1, call_1 = _complete_draw(
-        cache, HOLDOUT, prompt, draw=1, api_key=api_key
+    raw_1 = cache.complete(
+        HOLDOUT, prompt, temperature=0.0, draw=1
     )
-    raw_2, call_2 = _complete_draw(
-        cache, HOLDOUT, prompt, draw=2, api_key=api_key
+    raw_2 = cache.complete(
+        HOLDOUT, prompt, temperature=0.0, draw=2
     )
 
     answer_1 = parse_legacy_string(raw_1, opts)
     answer_2 = parse_legacy_string(raw_2, opts)
 
-    row = build_row(case.case_id, answer_1, answer_2)
+    return build_row(case.case_id, answer_1, answer_2)
 
-    return row, int(call_1) + int(call_2)
 
 def summarize(rows):
     n = len(rows)
@@ -173,19 +83,15 @@ def main():
     out = Path(args.out)
     out.mkdir(parents=True, exist_ok=True)
 
-    cache = _DrawCache(args.cache)
-    api_key = _key()
-
-    rows = []
-    new_api_calls = 0
+    cache = _Cache(args.cache, _key())
 
-    for case in load_cases(args.manifest)[: args.n]:
-        row, calls = run_one(case, cache, api_key)
-        rows.append(row)
-        new_api_calls += calls
+    rows = [
+        run_one(case, cache)
+        for case in load_cases(args.manifest)[:args.n]
+    ]
 
     summary = summarize(rows)
-    summary["new_api_calls_this_run"] = new_api_calls
+    summary["new_api_calls_this_run"] = cache.calls
 
     (out / "referee_self_inconsistency.jsonl").write_text(
         "".join(json.dumps(r) + "\n" for r in rows)
diff --git a/tests/test_referee_self_inconsistency.py b/tests/test_referee_self_inconsistency.py
index 75cac15..4b9b679 100644
--- a/tests/test_referee_self_inconsistency.py
+++ b/tests/test_referee_self_inconsistency.py
@@ -1,4 +1,4 @@
-from experiments.referee.referee_self_inconsistency import summarize
+from experiments.referee.referee_self_inconsistency import build_row, summarize
 
 
 def test_summary_all_stable():
@@ -52,16 +52,6 @@ def test_summary_contains_metadata():
     assert result["temperature"] == 0
     assert result["cache_bypassed"] is True
 
-from experiments.referee.referee_self_inconsistency import _DrawCache
-
-
-def test_draw_cache_distinguishes_draws():
-    k1 = _DrawCache.key("model", "prompt", 1)
-    k2 = _DrawCache.key("model", "prompt", 2)
-
-    assert k1 != k2
-
-
 def test_summary_single_unstable_case():
     rows = [
         {
@@ -80,26 +70,6 @@ def test_summary_single_unstable_case():
     assert result["temp0_self_inconsistency_rate"] == 1.0
 
 
-def test_draw_cache_same_draw_same_key():
-    k1 = _DrawCache.key("model", "prompt", 1)
-    k2 = _DrawCache.key("model", "prompt", 1)
-
-    assert k1 == k2
-
-def test_draw_cache_round_trip(tmp_path):
-    cache = _DrawCache(tmp_path / "cache.jsonl")
-
-    assert cache.get("model", "prompt", 1) is None
-
-    cache.put("model", "prompt", 1, "answer")
-
-    assert cache.get("model", "prompt", 1) == "answer"
-    assert cache.get("model", "prompt", 2) is None
-
-
-from experiments.referee.referee_self_inconsistency import build_row
-
-
 def test_build_row_detects_flip():
     row = build_row("medqa-1", "A", "B")
 
@@ -108,12 +78,3 @@ def test_build_row_detects_flip():
     assert row["answer_2"] == "B"
     assert row["temp0_flip"] is True
 
-def test_draw_cache_reloads_from_disk(tmp_path):
-    cache_file = tmp_path / "cache.jsonl"
-
-    cache1 = _DrawCache(cache_file)
-    cache1.put("model", "prompt", 1, "answer")
-
-    cache2 = _DrawCache(cache_file)
-
-    assert cache2.get("model", "prompt", 1) == "answer"

From f476bf088834f3cf99a74568732a284578233cfa Mon Sep 17 00:00:00 2001
From: armaanvgrewal <armaanvgrewal@gmail.com>
Date: Sun, 30 Aug 2026 18:25:19 -0500
Subject: [PATCH 5/7] Checkpoint: investigate Gemini temperature=0 timeout
 behavior


From 7ce2096bb735eda0d98c734e636bef175d2f48aa Mon Sep 17 00:00:00 2001
From: armaanvgrewal <armaanvgrewal@gmail.com>
Date: Tue, 1 Sep 2026 19:37:23 -0500
Subject: [PATCH 6/7] Track declared answers separately from parser fallbacks

---
 benchmaxxing/extract.py                       | 45 +++++++++++
 .../referee/referee_self_inconsistency.py     | 54 +++++++++++--
 tests/test_extract.py                         | 26 ++++++
 tests/test_referee_self_inconsistency.py      | 79 ++++++++++++++++---
 4 files changed, 189 insertions(+), 15 deletions(-)

diff --git a/benchmaxxing/extract.py b/benchmaxxing/extract.py
index cbf5a90..e7252fa 100644
--- a/benchmaxxing/extract.py
+++ b/benchmaxxing/extract.py
@@ -87,6 +87,51 @@ def is_abstention(text: str) -> bool:
 )
 
 
+def declared_mcq_choice(
+    text: str,
+    options: tuple[str, ...] | list[str],
+) -> tuple[str, bool]:
+    """Return (declared_answer, declared_flag).
+
+    Unlike parse_mcq_choice(), this only recognizes explicit answer
+    declarations and intentionally ignores fallback heuristics such as
+    trailing-letter scans or last-mentioned-option resolution.
+    """
+
+    if not text:
+        return "", False
+
+    num_options = len(options)
+    if num_options == 0:
+        return "", False
+
+    valid_letters = {
+        chr(ord("A") + i)
+        for i in range(min(num_options, 26))
+    }
+
+    declarations = [
+        _first_group(m)
+        for m in _DECLARATION_REGEX.finditer(text)
+        if _first_group(m) in valid_letters
+    ]
+
+    if declarations:
+        letter = declarations[-1]
+        return options[ord(letter) - ord("A")], True
+
+    stripped = text.strip()
+
+    if (
+        len(stripped) == 1
+        and stripped.upper() in valid_letters
+    ):
+        return options[ord(stripped.upper()) - ord("A")], True
+
+    return "", False
+
+
+
 def _first_group(m: re.Match) -> str:
     """Return the first non-None group from a regex match with alternations."""
     for g in m.groups():
diff --git a/experiments/referee/referee_self_inconsistency.py b/experiments/referee/referee_self_inconsistency.py
index 568a71e..1406d84 100644
--- a/experiments/referee/referee_self_inconsistency.py
+++ b/experiments/referee/referee_self_inconsistency.py
@@ -11,7 +11,7 @@
 from pathlib import Path
 
 from benchmaxxing.data import load_cases
-from benchmaxxing.extract import parse_legacy_string
+from benchmaxxing.extract import parse_legacy_string, declared_mcq_choice
 from experiments.referee.referee_threshold import (
     _Cache,
     _key,
@@ -21,11 +21,13 @@
 
 
 
-def build_row(case_id, answer_1, answer_2):
+def build_row(case_id, answer_1, answer_2, declared_1, declared_2):
     return {
         "case_id": case_id,
         "answer_1": answer_1,
         "answer_2": answer_2,
+        "declared_1": declared_1,
+        "declared_2": declared_2,
         "temp0_flip": answer_1 != answer_2,
     }
 
@@ -44,21 +46,61 @@ def run_one(case, cache):
     answer_1 = parse_legacy_string(raw_1, opts)
     answer_2 = parse_legacy_string(raw_2, opts)
 
-    return build_row(case.case_id, answer_1, answer_2)
+    _, declared_1 = declared_mcq_choice(raw_1, opts)
+    _, declared_2 = declared_mcq_choice(raw_2, opts)
+
+    return build_row(
+        case.case_id,
+        answer_1,
+        answer_2,
+        declared_1,
+        declared_2,
+    )
 
 
 def summarize(rows):
     n = len(rows)
-    unstable = sum(1 for r in rows if r["temp0_flip"])
+
+    declared_pairs = sum(
+        1
+        for r in rows
+        if r["declared_1"] and r["declared_2"]
+    )
+
+    undeclared_pairs = sum(
+        1
+        for r in rows
+        if not (r["declared_1"] and r["declared_2"])
+    )
+
+    undeclared_draws = sum(
+        int(not r["declared_1"]) + int(not r["declared_2"])
+        for r in rows
+    )
+
+    unstable = sum(
+        1
+        for r in rows
+        if r["declared_1"]
+        and r["declared_2"]
+        and r["temp0_flip"]
+    )
+
+    stable = declared_pairs - unstable
 
     return {
         "n": n,
         "temperature": 0,
         "cache_bypassed": True,
-        "stable_cases": n - unstable,
+        "declared_pairs": declared_pairs,
+        "undeclared_pairs": undeclared_pairs,
+        "undeclared_draws": undeclared_draws,
+        "stable_cases": stable,
         "unstable_cases": unstable,
         "temp0_self_inconsistency_rate": (
-            unstable / n if n else None
+            unstable / declared_pairs
+            if declared_pairs
+            else None
         ),
     }
 
diff --git a/tests/test_extract.py b/tests/test_extract.py
index c80597e..9ff7fe5 100644
--- a/tests/test_extract.py
+++ b/tests/test_extract.py
@@ -169,3 +169,29 @@ def test_leading_article_a_is_not_the_answer(self):
         text = (r"A patient with these findings is best treated with option C. "
                 r"The final answer is $\boxed{C}$")
         assert parse_mcq_choice(text, self.OPTS) == 2  # C, never the article-A (0)
+
+
+
+# ── Declaration detection (Phase 4B) ───────────────────────────────────
+
+def test_declared_mcq_choice_explicit_declaration():
+    from benchmaxxing.extract import declared_mcq_choice
+
+    assert declared_mcq_choice("The answer is B.", OPTS_5) == ("B", True)
+
+
+def test_declared_mcq_choice_bare_letter():
+    from benchmaxxing.extract import declared_mcq_choice
+
+    assert declared_mcq_choice("C", OPTS_5) == ("C", True)
+
+
+def test_declared_mcq_choice_undeclared_reasoning():
+    from benchmaxxing.extract import declared_mcq_choice
+
+    text = (
+        "A is unlikely. B is possible. "
+        "C could fit. D is also plausible."
+    )
+
+    assert declared_mcq_choice(text, OPTS_5) == ("", False)
diff --git a/tests/test_referee_self_inconsistency.py b/tests/test_referee_self_inconsistency.py
index 4b9b679..72ed384 100644
--- a/tests/test_referee_self_inconsistency.py
+++ b/tests/test_referee_self_inconsistency.py
@@ -3,8 +3,8 @@
 
 def test_summary_all_stable():
     rows = [
-        {"temp0_flip": False},
-        {"temp0_flip": False},
+        {"temp0_flip": False, "declared_1": True, "declared_2": True},
+        {"temp0_flip": False, "declared_1": True, "declared_2": True},
     ]
 
     result = summarize(rows)
@@ -17,8 +17,8 @@ def test_summary_all_stable():
 
 def test_summary_all_unstable():
     rows = [
-        {"temp0_flip": True},
-        {"temp0_flip": True},
+        {"temp0_flip": True, "declared_1": True, "declared_2": True},
+        {"temp0_flip": True, "declared_1": True, "declared_2": True},
     ]
 
     result = summarize(rows)
@@ -31,10 +31,10 @@ def test_summary_all_unstable():
 
 def test_summary_mixed():
     rows = [
-        {"temp0_flip": True},
-        {"temp0_flip": False},
-        {"temp0_flip": False},
-        {"temp0_flip": True},
+        {"temp0_flip": True, "declared_1": True, "declared_2": True},
+        {"temp0_flip": False, "declared_1": True, "declared_2": True},
+        {"temp0_flip": False, "declared_1": True, "declared_2": True},
+        {"temp0_flip": True, "declared_1": True, "declared_2": True},
     ]
 
     result = summarize(rows)
@@ -58,6 +58,8 @@ def test_summary_single_unstable_case():
             "case_id": "medqa-1",
             "answer_1": "A",
             "answer_2": "B",
+            "declared_1": True,
+            "declared_2": True,
             "temp0_flip": True,
         }
     ]
@@ -71,10 +73,69 @@ def test_summary_single_unstable_case():
 
 
 def test_build_row_detects_flip():
-    row = build_row("medqa-1", "A", "B")
+    row = build_row("medqa-1", "A", "B", True, True)
 
     assert row["case_id"] == "medqa-1"
     assert row["answer_1"] == "A"
     assert row["answer_2"] == "B"
     assert row["temp0_flip"] is True
 
+
+
+
+def test_build_row_tracks_declaration_state():
+    row = build_row(
+        "medqa-1",
+        "A",
+        "B",
+        True,
+        False,
+    )
+
+    assert row["declared_1"] is True
+    assert row["declared_2"] is False
+
+
+def test_summary_counts_declared_pairs_only():
+    rows = [
+        {
+            "temp0_flip": True,
+            "declared_1": True,
+            "declared_2": True,
+        },
+        {
+            "temp0_flip": False,
+            "declared_1": True,
+            "declared_2": True,
+        },
+        {
+            "temp0_flip": True,
+            "declared_1": True,
+            "declared_2": False,
+        },
+    ]
+
+    result = summarize(rows)
+
+    assert result["declared_pairs"] == 2
+    assert result["undeclared_pairs"] == 1
+    assert result["undeclared_draws"] == 1
+
+    assert result["temp0_self_inconsistency_rate"] == 0.5
+
+
+def test_summary_ignores_undeclared_pairs_in_rate():
+    rows = [
+        {
+            "temp0_flip": True,
+            "declared_1": False,
+            "declared_2": False,
+        }
+    ]
+
+    result = summarize(rows)
+
+    assert result["declared_pairs"] == 0
+    assert result["undeclared_pairs"] == 1
+    assert result["undeclared_draws"] == 2
+    assert result["temp0_self_inconsistency_rate"] is None

From 336f78d5c0174c67a29d41afc70747660cbf1f8a Mon Sep 17 00:00:00 2001
From: armaanvgrewal <armaanvgrewal@gmail.com>
Date: Tue, 1 Sep 2026 20:29:41 -0500
Subject: [PATCH 7/7] Limit Gemini output tokens by default

---
 benchmaxxing/gateway.py |  4 +++-
 tests/test_gateway.py   | 17 +++++++++++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/benchmaxxing/gateway.py b/benchmaxxing/gateway.py
index 3c2ba4a..44b7b3a 100644
--- a/benchmaxxing/gateway.py
+++ b/benchmaxxing/gateway.py
@@ -141,7 +141,9 @@ def __init__(
         timeout: float | None = 60.0,
     ):
         self.model = model
-        self.default_decoding = dict(default_decoding or {})
+        self.default_decoding = {"max_output_tokens": 16384}
+        if default_decoding:
+            self.default_decoding.update(default_decoding)
         self.timeout = timeout
         if client is not None:
             # Injected client (used by tests): no SDK import required.
diff --git a/tests/test_gateway.py b/tests/test_gateway.py
index aed08be..fe1f6f2 100644
--- a/tests/test_gateway.py
+++ b/tests/test_gateway.py
@@ -193,7 +193,7 @@ def test_gemini_backend_injected_client_multimodal_path():
     be.complete("describe", image=b"imgbytes")
     _, contents, kwargs = client.models.received[0]
     assert contents == ["describe", b"imgbytes"]
-    assert "config" not in kwargs  # no decoding overrides -> no config passed
+    assert kwargs["config"] == {"max_output_tokens": 16384}
 
 
 def test_gemini_backend_merges_default_decoding():
@@ -201,7 +201,7 @@ def test_gemini_backend_merges_default_decoding():
     be = gateway.GeminiBackend(client=client, default_decoding={"temperature": 0.7, "top_p": 0.9})
     be.complete("hi", decoding={"temperature": 0.1})
     _, _, kwargs = client.models.received[0]
-    assert kwargs["config"] == {"temperature": 0.1, "top_p": 0.9}  # per-call overrides default
+    assert kwargs["config"] == {"temperature": 0.1, "top_p": 0.9, "max_output_tokens": 16384}  # per-call overrides default
 
 
 def test_timeout_ms_converts_seconds_and_rejects_non_positive():
@@ -264,3 +264,16 @@ def test_gemini_backend_leaves_http_options_unset_when_timeout_disabled(monkeypa
     captured = _stub_genai(monkeypatch)
     gateway.GeminiBackend(api_key="k", timeout=None)
     assert captured["http_options"] is None
+
+
+
+def test_gemini_backend_applies_default_max_output_tokens():
+    client = _FakeClient()
+
+    be = gateway.GeminiBackend(client=client)
+
+    be.complete("hi")
+
+    _, _, kwargs = client.models.received[0]
+
+    assert kwargs["config"]["max_output_tokens"] == 16384