diff --git a/configs/experiments/benchmark-local-v01.yaml b/configs/experiments/benchmark-local-v01.yaml index 14a905e..b439644 100644 --- a/configs/experiments/benchmark-local-v01.yaml +++ b/configs/experiments/benchmark-local-v01.yaml @@ -35,6 +35,7 @@ judge: model: gpt-5.4-mini-2026-03-17 embedding_model: text-embedding-3-small reasoning_effort: medium + max_output_tokens: 8192 audit: enabled: false provider: openai diff --git a/configs/experiments/benchmark-v01.yaml b/configs/experiments/benchmark-v01.yaml index 3e38db1..612f631 100644 --- a/configs/experiments/benchmark-v01.yaml +++ b/configs/experiments/benchmark-v01.yaml @@ -43,6 +43,9 @@ judge: model: gpt-5.4-mini-2026-03-17 embedding_model: text-embedding-3-small reasoning_effort: medium + # Responses API budget includes hidden reasoning tokens and structured + # output. RAGAS's 1024-token default truncates long NLI judgments. + max_output_tokens: 8192 # Development fallback when no OpenAI credential is available (ADR-0018) - # labeled "exploratory_same_provider_judge" in the run record, since # generation is also Gemini-based: diff --git a/src/ragforge/evaluation/judge_ports.py b/src/ragforge/evaluation/judge_ports.py index 70b94a7..1dc3000 100644 --- a/src/ragforge/evaluation/judge_ports.py +++ b/src/ragforge/evaluation/judge_ports.py @@ -34,6 +34,7 @@ class ModelIdentity: model: str reasoning_effort: str | None output_schema_version: int + max_output_tokens: int | None = None @dataclass(frozen=True, slots=True) diff --git a/src/ragforge/evaluation/ragas_judge.py b/src/ragforge/evaluation/ragas_judge.py index e36c843..e62a5f7 100644 --- a/src/ragforge/evaluation/ragas_judge.py +++ b/src/ragforge/evaluation/ragas_judge.py @@ -208,6 +208,7 @@ def evaluate(self, sample: JudgeSample) -> JudgeResult: provider=self._identity.provider, model=self._identity.model, reasoning_effort=self._identity.reasoning_effort, + max_output_tokens=self._identity.max_output_tokens, output_schema_version=self._identity.output_schema_version, abstention_prompt_version=ABSTENTION_PROMPT_VERSION, question=sample.question, @@ -351,6 +352,7 @@ def build_openai_ragas_judge( api_key: str | None = None, cache: LLMCache | None = None, max_in_flight: int = _DEFAULT_MAX_IN_FLIGHT, + max_output_tokens: int = 8192, ) -> RagasJudge: """Construct a RagasJudge backed by real OpenAI models via ragas + instructor (ADR-0018). @@ -367,6 +369,9 @@ def build_openai_ragas_judge( reasoning_effort: Threaded straight into every underlying chat.completions.create() call (ragas.llms.InstructorLLM passes unknown kwargs through) - ADR-0018's "medium" default. + max_output_tokens: Responses API budget shared by hidden reasoning + and the structured output. RAGAS's 1024-token default is too + small for long NLI statement lists. api_key: Overrides OPENAI_API_KEY from the environment. cache: Optional LLMCache (ADR-0004), forwarded to the RagasJudge. max_in_flight: Bounds concurrent evaluate() calls to this provider, @@ -380,6 +385,8 @@ def build_openai_ragas_judge( raise GenerationError( "no OpenAI API key found: set OPENAI_API_KEY, or pass api_key explicitly" ) + if max_output_tokens <= 0: + raise ValueError("max_output_tokens must be positive") try: instructor_client = instructor.from_provider( f"openai/{llm_model_name}", @@ -396,12 +403,14 @@ def build_openai_ragas_judge( model=llm_model_name, provider="openai", reasoning_effort=reasoning_effort, + max_tokens=max_output_tokens, ) abstention_llm = _OpenAIResponsesInstructorLLM( client=instructor_client, model=llm_model_name, provider="openai", reasoning_effort=reasoning_effort, + max_tokens=max_output_tokens, system_prompt=_ABSTENTION_SYSTEM_PROMPT, ) return RagasJudge( @@ -413,6 +422,7 @@ def build_openai_ragas_judge( model=llm_model_name, reasoning_effort=reasoning_effort, output_schema_version=_OUTPUT_SCHEMA_VERSION, + max_output_tokens=max_output_tokens, ), cache=cache, max_in_flight=max_in_flight, diff --git a/src/ragforge/evaluation/run.py b/src/ragforge/evaluation/run.py index 97192c7..bb58b24 100644 --- a/src/ragforge/evaluation/run.py +++ b/src/ragforge/evaluation/run.py @@ -454,6 +454,7 @@ def _run() -> None: judge_model = config["judge"]["model"] judge_embedding_model = config["judge"]["embedding_model"] judge_reasoning_effort = config["judge"].get("reasoning_effort", "medium") + judge_max_output_tokens = config["judge"].get("max_output_tokens", 8192) audit_config = config.get("audit", {}) audit_enabled = audit_config.get("enabled", False) audit_provider = audit_config.get("provider", "openai") @@ -613,6 +614,7 @@ def _run() -> None: judge_model, judge_embedding_model, judge_reasoning_effort, + judge_max_output_tokens, cache, gemini_max_in_flight, ) diff --git a/src/ragforge/evaluation/run_strategies.py b/src/ragforge/evaluation/run_strategies.py index 57d859f..b1a81ec 100644 --- a/src/ragforge/evaluation/run_strategies.py +++ b/src/ragforge/evaluation/run_strategies.py @@ -194,6 +194,7 @@ def _build_judge_factory( model: str, embedding_model: str, reasoning_effort: str, + max_output_tokens: int, cache: LLMCache | None, max_in_flight: int, ) -> Callable[[], AnswerQualityJudge]: @@ -217,6 +218,7 @@ def _build_judge_factory( model, embedding_model, reasoning_effort=reasoning_effort, + max_output_tokens=max_output_tokens, cache=cache, max_in_flight=max_in_flight, ) diff --git a/tests/unit/test_benchmark_run.py b/tests/unit/test_benchmark_run.py index 1b7dc43..9809dd8 100644 --- a/tests/unit/test_benchmark_run.py +++ b/tests/unit/test_benchmark_run.py @@ -626,21 +626,46 @@ def fake_build_openai_ragas_judge( model: str, embedding_model: str, reasoning_effort: str = "medium", + max_output_tokens: int = 8192, api_key: str | None = None, cache: object = None, max_in_flight: int = 4, ) -> str: - calls.append((model, embedding_model, reasoning_effort, cache, max_in_flight)) + calls.append( + ( + model, + embedding_model, + reasoning_effort, + max_output_tokens, + cache, + max_in_flight, + ) + ) return "fake-openai-judge" monkeypatch.setattr(run_strategies, "build_openai_ragas_judge", fake_build_openai_ragas_judge) factory = run_strategies._build_judge_factory( - "openai", "gpt-5.4-mini-2026-03-17", "text-embedding-3-small", "medium", None, 4 + "openai", + "gpt-5.4-mini-2026-03-17", + "text-embedding-3-small", + "medium", + 8192, + None, + 4, ) factory() - assert calls == [("gpt-5.4-mini-2026-03-17", "text-embedding-3-small", "medium", None, 4)] + assert calls == [ + ( + "gpt-5.4-mini-2026-03-17", + "text-embedding-3-small", + "medium", + 8192, + None, + 4, + ) + ] def test_build_judge_factory_constructs_a_gemini_judge(monkeypatch: pytest.MonkeyPatch) -> None: @@ -660,7 +685,13 @@ def fake_build_gemini_ragas_judge( monkeypatch.setattr(run_strategies, "build_gemini_ragas_judge", fake_build_gemini_ragas_judge) factory = run_strategies._build_judge_factory( - "gemini", "gemini-3.1-flash-lite", "gemini-embedding-001", "medium", None, 4 + "gemini", + "gemini-3.1-flash-lite", + "gemini-embedding-001", + "medium", + 8192, + None, + 4, ) factory() @@ -670,7 +701,7 @@ def fake_build_gemini_ragas_judge( def test_build_judge_factory_fails_closed_for_an_unknown_provider() -> None: """An unrecognized provider fails fast rather than silently falling back to either adapter.""" with pytest.raises(SystemExit, match="unknown judge provider"): - run_strategies._build_judge_factory("anthropic", "claude", "embed", "medium", None, 4) + run_strategies._build_judge_factory("anthropic", "claude", "embed", "medium", 8192, None, 4) def test_verify_resume_identity_passes_when_everything_matches() -> None: diff --git a/tests/unit/test_ragas_judge.py b/tests/unit/test_ragas_judge.py index 99c712f..c365b60 100644 --- a/tests/unit/test_ragas_judge.py +++ b/tests/unit/test_ragas_judge.py @@ -189,7 +189,7 @@ def test_versioned_gpt_snapshot_uses_responses_api_parameters() -> None: llm = object.__new__(_OpenAIResponsesInstructorLLM) llm.model = "gpt-5.4-mini-2026-03-17" llm.model_args = { - "max_tokens": 1024, + "max_tokens": 8192, "temperature": 0.01, "top_p": 0.1, "reasoning_effort": "medium", @@ -198,11 +198,22 @@ def test_versioned_gpt_snapshot_uses_responses_api_parameters() -> None: mapped = llm._map_openai_params() assert mapped == { - "max_output_tokens": 1024, + "max_output_tokens": 8192, "reasoning": {"effort": "medium"}, } +def test_openai_judge_rejects_a_non_positive_output_budget() -> None: + """An invalid structured-output budget fails before client construction.""" + with pytest.raises(ValueError, match="max_output_tokens must be positive"): + build_openai_ragas_judge( + "gpt-5.4-mini-2026-03-17", + "text-embedding-3-small", + max_output_tokens=0, + api_key="test-key", + ) + + def test_openai_answer_relevancy_embeddings_use_an_async_client() -> None: """RAGAS AnswerRelevancy can call aembed_text() on the configured adapter.""" embeddings = _build_async_openai_embeddings("test-key", "text-embedding-3-small") @@ -225,3 +236,42 @@ def test_a_cache_hit_skips_the_metric_and_abstention_calls_entirely(tmp_path: Pa assert len(faithfulness.calls) == 1, "the second evaluate() made no additional metric call" assert len(answer_relevancy.calls) == 1 assert len(abstention_llm.calls) == 1 + + +def test_output_budget_participates_in_judge_cache_identity(tmp_path: Path) -> None: + """A larger structured-output budget cannot reuse a result produced under a smaller cap.""" + cache = FileLLMCache(tmp_path) + first_identity = ModelIdentity( + provider="openai", + model="model", + reasoning_effort="medium", + output_schema_version=1, + max_output_tokens=4096, + ) + second_identity = ModelIdentity( + provider="openai", + model="model", + reasoning_effort="medium", + output_schema_version=1, + max_output_tokens=8192, + ) + first_faithfulness = _FakeMetric(lambda **kwargs: _FakeMetricResult(0.9)) + second_faithfulness = _FakeMetric(lambda **kwargs: _FakeMetricResult(0.9)) + + RagasJudge( + first_faithfulness, + _FakeMetric(lambda **kwargs: _FakeMetricResult(0.8)), + _FakeAbstentionLLM(), + first_identity, + cache=cache, + ).evaluate(_sample()) + RagasJudge( + second_faithfulness, + _FakeMetric(lambda **kwargs: _FakeMetricResult(0.8)), + _FakeAbstentionLLM(), + second_identity, + cache=cache, + ).evaluate(_sample()) + + assert len(first_faithfulness.calls) == 1 + assert len(second_faithfulness.calls) == 1