Prevent truncated OpenAI RAGAS structured outputs

configs/experiments/benchmark-local-v01.yaml

@@ -35,6 +35,7 @@ judge:
   model: gpt-5.4-mini-2026-03-17
   embedding_model: text-embedding-3-small
   reasoning_effort: medium
+  max_output_tokens: 8192
 audit:
   enabled: false
   provider: openai

configs/experiments/benchmark-v01.yaml

@@ -43,6 +43,9 @@ judge:
   model: gpt-5.4-mini-2026-03-17
   embedding_model: text-embedding-3-small
   reasoning_effort: medium
+  # Responses API budget includes hidden reasoning tokens and structured
+  # output. RAGAS's 1024-token default truncates long NLI judgments.
+  max_output_tokens: 8192
   # Development fallback when no OpenAI credential is available (ADR-0018) -
   # labeled "exploratory_same_provider_judge" in the run record, since
   # generation is also Gemini-based:

src/ragforge/evaluation/judge_ports.py

@@ -34,6 +34,7 @@ class ModelIdentity:
     model: str
     reasoning_effort: str | None
     output_schema_version: int
+    max_output_tokens: int | None = None
 
 
 @dataclass(frozen=True, slots=True)

src/ragforge/evaluation/ragas_judge.py

@@ -208,6 +208,7 @@ def evaluate(self, sample: JudgeSample) -> JudgeResult:
             provider=self._identity.provider,
             model=self._identity.model,
             reasoning_effort=self._identity.reasoning_effort,
+            max_output_tokens=self._identity.max_output_tokens,
             output_schema_version=self._identity.output_schema_version,
             abstention_prompt_version=ABSTENTION_PROMPT_VERSION,
             question=sample.question,
@@ -351,6 +352,7 @@ def build_openai_ragas_judge(
     api_key: str | None = None,
     cache: LLMCache | None = None,
     max_in_flight: int = _DEFAULT_MAX_IN_FLIGHT,
+    max_output_tokens: int = 8192,
 ) -> RagasJudge:
     """Construct a RagasJudge backed by real OpenAI models via ragas + instructor (ADR-0018).
 
@@ -367,6 +369,9 @@ def build_openai_ragas_judge(
         reasoning_effort: Threaded straight into every underlying
             chat.completions.create() call (ragas.llms.InstructorLLM passes
             unknown kwargs through) - ADR-0018's "medium" default.
+        max_output_tokens: Responses API budget shared by hidden reasoning
+            and the structured output. RAGAS's 1024-token default is too
+            small for long NLI statement lists.
         api_key: Overrides OPENAI_API_KEY from the environment.
         cache: Optional LLMCache (ADR-0004), forwarded to the RagasJudge.
         max_in_flight: Bounds concurrent evaluate() calls to this provider,
@@ -380,6 +385,8 @@ def build_openai_ragas_judge(
         raise GenerationError(
             "no OpenAI API key found: set OPENAI_API_KEY, or pass api_key explicitly"
         )
+    if max_output_tokens <= 0:
+        raise ValueError("max_output_tokens must be positive")
     try:
         instructor_client = instructor.from_provider(
             f"openai/{llm_model_name}",
@@ -396,12 +403,14 @@ def build_openai_ragas_judge(
         model=llm_model_name,
         provider="openai",
         reasoning_effort=reasoning_effort,
+        max_tokens=max_output_tokens,
     )
     abstention_llm = _OpenAIResponsesInstructorLLM(
         client=instructor_client,
         model=llm_model_name,
         provider="openai",
         reasoning_effort=reasoning_effort,
+        max_tokens=max_output_tokens,
         system_prompt=_ABSTENTION_SYSTEM_PROMPT,
     )
     return RagasJudge(
@@ -413,6 +422,7 @@ def build_openai_ragas_judge(
             model=llm_model_name,
             reasoning_effort=reasoning_effort,
             output_schema_version=_OUTPUT_SCHEMA_VERSION,
+            max_output_tokens=max_output_tokens,
         ),
         cache=cache,
         max_in_flight=max_in_flight,

src/ragforge/evaluation/run.py

@@ -454,6 +454,7 @@ def _run() -> None:
     judge_model = config["judge"]["model"]
     judge_embedding_model = config["judge"]["embedding_model"]
     judge_reasoning_effort = config["judge"].get("reasoning_effort", "medium")
+    judge_max_output_tokens = config["judge"].get("max_output_tokens", 8192)
     audit_config = config.get("audit", {})
     audit_enabled = audit_config.get("enabled", False)
     audit_provider = audit_config.get("provider", "openai")
@@ -613,6 +614,7 @@ def _run() -> None:
         judge_model,
         judge_embedding_model,
         judge_reasoning_effort,
+        judge_max_output_tokens,
         cache,
         gemini_max_in_flight,
     )

src/ragforge/evaluation/run_strategies.py

@@ -194,6 +194,7 @@ def _build_judge_factory(
     model: str,
     embedding_model: str,
     reasoning_effort: str,
+    max_output_tokens: int,
     cache: LLMCache | None,
     max_in_flight: int,
 ) -> Callable[[], AnswerQualityJudge]:
@@ -217,6 +218,7 @@ def _build_judge_factory(
             model,
             embedding_model,
             reasoning_effort=reasoning_effort,
+            max_output_tokens=max_output_tokens,
             cache=cache,
             max_in_flight=max_in_flight,
         )

tests/unit/test_benchmark_run.py

@@ -626,21 +626,46 @@ def fake_build_openai_ragas_judge(
         model: str,
         embedding_model: str,
         reasoning_effort: str = "medium",
+        max_output_tokens: int = 8192,
         api_key: str | None = None,
         cache: object = None,
         max_in_flight: int = 4,
     ) -> str:
-        calls.append((model, embedding_model, reasoning_effort, cache, max_in_flight))
+        calls.append(
+            (
+                model,
+                embedding_model,
+                reasoning_effort,
+                max_output_tokens,
+                cache,
+                max_in_flight,
+            )
+        )
         return "fake-openai-judge"
 
     monkeypatch.setattr(run_strategies, "build_openai_ragas_judge", fake_build_openai_ragas_judge)
 
     factory = run_strategies._build_judge_factory(
-        "openai", "gpt-5.4-mini-2026-03-17", "text-embedding-3-small", "medium", None, 4
+        "openai",
+        "gpt-5.4-mini-2026-03-17",
+        "text-embedding-3-small",
+        "medium",
+        8192,
+        None,
+        4,
     )
     factory()
 
-    assert calls == [("gpt-5.4-mini-2026-03-17", "text-embedding-3-small", "medium", None, 4)]
+    assert calls == [
+        (
+            "gpt-5.4-mini-2026-03-17",
+            "text-embedding-3-small",
+            "medium",
+            8192,
+            None,
+            4,
+        )
+    ]
 
 
 def test_build_judge_factory_constructs_a_gemini_judge(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -660,7 +685,13 @@ def fake_build_gemini_ragas_judge(
     monkeypatch.setattr(run_strategies, "build_gemini_ragas_judge", fake_build_gemini_ragas_judge)
 
     factory = run_strategies._build_judge_factory(
-        "gemini", "gemini-3.1-flash-lite", "gemini-embedding-001", "medium", None, 4
+        "gemini",
+        "gemini-3.1-flash-lite",
+        "gemini-embedding-001",
+        "medium",
+        8192,
+        None,
+        4,
     )
     factory()
 
@@ -670,7 +701,7 @@ def fake_build_gemini_ragas_judge(
 def test_build_judge_factory_fails_closed_for_an_unknown_provider() -> None:
     """An unrecognized provider fails fast rather than silently falling back to either adapter."""
     with pytest.raises(SystemExit, match="unknown judge provider"):
-        run_strategies._build_judge_factory("anthropic", "claude", "embed", "medium", None, 4)
+        run_strategies._build_judge_factory("anthropic", "claude", "embed", "medium", 8192, None, 4)
 
 
 def test_verify_resume_identity_passes_when_everything_matches() -> None:

tests/unit/test_ragas_judge.py

@@ -189,7 +189,7 @@ def test_versioned_gpt_snapshot_uses_responses_api_parameters() -> None:
     llm = object.__new__(_OpenAIResponsesInstructorLLM)
     llm.model = "gpt-5.4-mini-2026-03-17"
     llm.model_args = {
-        "max_tokens": 1024,
+        "max_tokens": 8192,
         "temperature": 0.01,
         "top_p": 0.1,
         "reasoning_effort": "medium",
@@ -198,11 +198,22 @@ def test_versioned_gpt_snapshot_uses_responses_api_parameters() -> None:
     mapped = llm._map_openai_params()
 
     assert mapped == {
-        "max_output_tokens": 1024,
+        "max_output_tokens": 8192,
         "reasoning": {"effort": "medium"},
     }
 
 
+def test_openai_judge_rejects_a_non_positive_output_budget() -> None:
+    """An invalid structured-output budget fails before client construction."""
+    with pytest.raises(ValueError, match="max_output_tokens must be positive"):
+        build_openai_ragas_judge(
+            "gpt-5.4-mini-2026-03-17",
+            "text-embedding-3-small",
+            max_output_tokens=0,
+            api_key="test-key",
+        )
+
+
 def test_openai_answer_relevancy_embeddings_use_an_async_client() -> None:
     """RAGAS AnswerRelevancy can call aembed_text() on the configured adapter."""
     embeddings = _build_async_openai_embeddings("test-key", "text-embedding-3-small")
@@ -225,3 +236,42 @@ def test_a_cache_hit_skips_the_metric_and_abstention_calls_entirely(tmp_path: Pa
     assert len(faithfulness.calls) == 1, "the second evaluate() made no additional metric call"
     assert len(answer_relevancy.calls) == 1
     assert len(abstention_llm.calls) == 1
+
+
+def test_output_budget_participates_in_judge_cache_identity(tmp_path: Path) -> None:
+    """A larger structured-output budget cannot reuse a result produced under a smaller cap."""
+    cache = FileLLMCache(tmp_path)
+    first_identity = ModelIdentity(
+        provider="openai",
+        model="model",
+        reasoning_effort="medium",
+        output_schema_version=1,
+        max_output_tokens=4096,
+    )
+    second_identity = ModelIdentity(
+        provider="openai",
+        model="model",
+        reasoning_effort="medium",
+        output_schema_version=1,
+        max_output_tokens=8192,
+    )
+    first_faithfulness = _FakeMetric(lambda **kwargs: _FakeMetricResult(0.9))
+    second_faithfulness = _FakeMetric(lambda **kwargs: _FakeMetricResult(0.9))
+
+    RagasJudge(
+        first_faithfulness,
+        _FakeMetric(lambda **kwargs: _FakeMetricResult(0.8)),
+        _FakeAbstentionLLM(),
+        first_identity,
+        cache=cache,
+    ).evaluate(_sample())
+    RagasJudge(
+        second_faithfulness,
+        _FakeMetric(lambda **kwargs: _FakeMetricResult(0.8)),
+        _FakeAbstentionLLM(),
+        second_identity,
+        cache=cache,
+    ).evaluate(_sample())
+
+    assert len(first_faithfulness.calls) == 1
+    assert len(second_faithfulness.calls) == 1