From 2d020ca6d087675e9fdfc36a75212860eb78bbcd Mon Sep 17 00:00:00 2001 From: Bruno Vicco Date: Sun, 26 Jul 2026 15:28:23 -0300 Subject: [PATCH 1/2] fix(evaluation): harden cost-controlled benchmark resumes --- configs/experiments/benchmark-v01.yaml | 4 + docs/adr/0017-auditable-evidence-lineage.md | 12 +- scripts/verify_run.py | 44 +++- src/ragforge/evaluation/answer_harness.py | 45 ++-- src/ragforge/evaluation/artifact_writer.py | 25 ++- src/ragforge/evaluation/event_log.py | 25 +++ src/ragforge/evaluation/ragas_judge.py | 107 +++++++--- src/ragforge/evaluation/records.py | 17 ++ src/ragforge/evaluation/run.py | 218 ++++++++++++++++---- src/ragforge/evaluation/run_evidence.py | 57 ++++- src/ragforge/evaluation/run_manifest.py | 104 +++++++++- src/ragforge/evaluation/run_strategies.py | 13 +- src/ragforge/evaluation/split_builder.py | 72 +++++++ tests/unit/test_answer_harness.py | 45 +++- tests/unit/test_artifact_writer.py | 22 ++ tests/unit/test_benchmark_run.py | 96 ++++++++- tests/unit/test_event_log.py | 25 +++ tests/unit/test_ragas_judge.py | 55 ++++- tests/unit/test_records.py | 31 +++ tests/unit/test_run_evidence_summaries.py | 66 +++++- tests/unit/test_run_manifest.py | 84 ++++++++ tests/unit/test_split_builder.py | 54 ++++- tests/unit/test_verify_run.py | 48 +++-- 23 files changed, 1123 insertions(+), 146 deletions(-) diff --git a/configs/experiments/benchmark-v01.yaml b/configs/experiments/benchmark-v01.yaml index 612f631..5410b12 100644 --- a/configs/experiments/benchmark-v01.yaml +++ b/configs/experiments/benchmark-v01.yaml @@ -6,6 +6,10 @@ corpus: dataset: path: datasets/regrag-br/ split: test # dev split is reserved for router few-shot examples (ADR-0003) + # Cost-controlled comparison: deterministic, query-class-stratified sample. + # Remove max_questions only for the final full-test publication run. + max_questions: 60 + sampling_seed: regrag-br-benchmark-sample-v1 strategies: - dense - sparse_bm25 diff --git a/docs/adr/0017-auditable-evidence-lineage.md b/docs/adr/0017-auditable-evidence-lineage.md index 15962bf..6939394 100644 --- a/docs/adr/0017-auditable-evidence-lineage.md +++ b/docs/adr/0017-auditable-evidence-lineage.md @@ -217,7 +217,17 @@ Artifacts SHALL use: 4. atomic rename or transaction; 5. checksum generation after all files close. -Manifest status changes to `completed` only after checksum verification. +`artifact_root_hash` covers every final artifact except `manifest.json` and +`checksums.sha256`; excluding the manifest avoids a circular hash because the +root is stored inside that manifest. `checksums.sha256` SHALL still cover the +exact final `manifest.json` bytes. The checksum inventory is published before +the final atomic manifest rename, so manifest status changes to `completed` +only when the final inventory is ready. + +New and resumed auditable runs SHALL require a clean Git worktree, excluding +only untracked generated output under `artifacts/`, `experiments/`, and +`.ragforge/`. A resume SHALL preserve the original `started_at` and Git SHA +and fail closed when the current commit or another manifest identity differs. A failed run remains inspectable. diff --git a/scripts/verify_run.py b/scripts/verify_run.py index 31a372c..d7bf53c 100644 --- a/scripts/verify_run.py +++ b/scripts/verify_run.py @@ -21,12 +21,14 @@ import sys from pathlib import Path +from ragforge.evaluation.canonical_hash import canonical_json_hash from ragforge.evaluation.event_log import compute_event_hash from ragforge.evaluation.lineage_ports import EventEnvelope from ragforge.ingestion.snapshot import snapshot_hash ROOT = Path(__file__).resolve().parents[1] ARTIFACTS_DIR = ROOT / "artifacts" / "runs" +_MANIFEST_FILENAME = "manifest.json" def parse_args() -> argparse.Namespace: @@ -36,6 +38,20 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def load_recorded_checksums(artifacts_dir: Path) -> dict[str, str]: + """Load the standard checksum inventory into a relative-path mapping.""" + recorded: dict[str, str] = {} + checksums_path = artifacts_dir / "checksums.sha256" + for line in checksums_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + digest, separator, relative = line.partition(" ") + if not separator or not digest or not relative: + raise ValueError(f"invalid checksum line: {line!r}") + recorded[relative] = digest + return recorded + + def verify_checksums(artifacts_dir: Path) -> list[str]: """Return every problem found comparing checksums.sha256 against the files on disk. @@ -49,12 +65,10 @@ def verify_checksums(artifacts_dir: Path) -> list[str]: if not checksums_path.exists(): return [f"checksums.sha256 not found at {checksums_path}"] - recorded: dict[str, str] = {} - for line in checksums_path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - digest, _, relative = line.partition(" ") - recorded[relative] = digest + try: + recorded = load_recorded_checksums(artifacts_dir) + except ValueError as exc: + return [str(exc)] for relative, digest in recorded.items(): path = artifacts_dir / relative @@ -152,6 +166,24 @@ def verify_manifest(artifacts_dir: Path) -> list[str]: problems.append( f"manifest.json declares strategies with no question artifacts: {sorted(missing)}" ) + checksums_path = artifacts_dir / "checksums.sha256" + if checksums_path.exists(): + try: + recorded = load_recorded_checksums(artifacts_dir) + except ValueError as exc: + problems.append(str(exc)) + else: + root_inputs = { + relative: digest + for relative, digest in recorded.items() + if relative != _MANIFEST_FILENAME + } + expected_root_hash = canonical_json_hash(root_inputs) + if manifest.get("artifact_root_hash") != expected_root_hash: + problems.append( + "manifest.json artifact_root_hash does not match the final artifact " + "checksum inventory" + ) return problems diff --git a/src/ragforge/evaluation/answer_harness.py b/src/ragforge/evaluation/answer_harness.py index 1f9e860..d159c6b 100644 --- a/src/ragforge/evaluation/answer_harness.py +++ b/src/ragforge/evaluation/answer_harness.py @@ -12,6 +12,7 @@ from concurrent.futures import CancelledError from dataclasses import dataclass from statistics import mean +from typing import Protocol, runtime_checkable from ragforge.domain.models import Answer, Judgment, RetrievalResult from ragforge.domain.protocols import RetrievalStrategy @@ -25,6 +26,13 @@ _DEFAULT_MAX_WORKERS = 5 +@runtime_checkable +class _ClosableJudge(Protocol): + """Optional lifecycle exposed by judges that own network clients.""" + + def close(self) -> None: ... + + @dataclass(frozen=True, slots=True) class AnswerEvaluationResult: """Aggregate answer-quality metrics plus one AnswerRecord per attempted judgment (ADR-0012). @@ -64,16 +72,12 @@ def evaluate_answer_quality( then run concurrently across up to ``max_workers`` questions at once, since each question's Gemini calls are independent HTTP requests. - ``judge_factory`` builds one judge, not many: ``generator`` (a plain - synchronous HTTP client) is shared safely across worker threads, but - RagasJudge.evaluate() calls ragas's sync wrapper, which internally does - ``asyncio.run(self.ascore(...))`` - a fresh event loop per call, reusing - the same async client underneath. Calling that concurrently from - multiple threads against one shared judge corrupts its connection pool - (observed for real: thousands of leaked CLOSE_WAIT sockets and a stalled - run). Each worker thread instead lazily builds and caches its own judge - via ``judge_factory``, so no judge instance is ever touched from more - than one thread. + ``generator`` (a plain synchronous HTTP client) is shared safely across + worker threads. Each worker thread instead lazily builds and caches its + own judge via ``judge_factory`` so one RagasJudge and its persistent + asyncio event loop are never touched concurrently from multiple threads. + Judges exposing ``close()`` are closed after the worker pool finishes, + including when scoring raises. A retrieval, generation, or judge-scoring failure for one question is counted in "answer_errors" and excluded from the averages rather than @@ -143,12 +147,16 @@ def evaluate_answer_quality( retrieved.append((judgment, results)) thread_local = threading.local() + judges: list[AnswerQualityJudge] = [] + judges_lock = threading.Lock() def _judge_for_this_thread() -> AnswerQualityJudge: judge = getattr(thread_local, "judge", None) if judge is None: judge = judge_factory() thread_local.judge = judge + with judges_lock: + judges.append(judge) return judge def _score_one( @@ -197,12 +205,17 @@ def _on_scoring_result( consecutive_scoring_errors += 1 return consecutive_scoring_errors >= _MAX_CONSECUTIVE_ERRORS - outcomes = run_bounded( - retrieved, - lambda pair: _score_one(pair[0], pair[1]), - max_workers=max_workers, - on_result=_on_scoring_result, - ) + try: + outcomes = run_bounded( + retrieved, + lambda pair: _score_one(pair[0], pair[1]), + max_workers=max_workers, + on_result=_on_scoring_result, + ) + finally: + for judge in judges: + if isinstance(judge, _ClosableJudge): + judge.close() for (judgment, _results), outcome in zip(retrieved, outcomes, strict=True): if isinstance(outcome, BaseException): is_cancelled = isinstance(outcome, CancelledError) diff --git a/src/ragforge/evaluation/artifact_writer.py b/src/ragforge/evaluation/artifact_writer.py index 2a107d2..b4bfe2f 100644 --- a/src/ragforge/evaluation/artifact_writer.py +++ b/src/ragforge/evaluation/artifact_writer.py @@ -8,6 +8,7 @@ covers a partially-written file. """ +from collections.abc import Mapping from pathlib import Path from ragforge.ingestion.snapshot import snapshot_hash @@ -26,7 +27,11 @@ def write_atomic(path: Path, content: str) -> None: tmp_path.replace(path) -def compute_checksums(root: Path) -> dict[str, str]: +def compute_checksums( + root: Path, + *, + excluded_paths: frozenset[str] = frozenset(), +) -> dict[str, str]: """Return ``{relative_posix_path: sha256_hex}`` for every regular file under ``root``. The checksums file itself (if already present from a prior partial @@ -37,19 +42,23 @@ def compute_checksums(root: Path) -> dict[str, str]: if not path.is_file(): continue relative = path.relative_to(root).as_posix() - if relative == _CHECKSUMS_FILENAME: + if relative == _CHECKSUMS_FILENAME or relative in excluded_paths: continue checksums[relative] = snapshot_hash(path) return checksums -def write_checksums_file(root: Path) -> None: +def write_checksums_file( + root: Path, + checksums: Mapping[str, str] | None = None, +) -> None: """Write ``root/checksums.sha256`` in standard ``sha256sum`` format (verifiable externally). - Every existing file under ``root`` at call time is included - callers - should call this only after every other artifact for the run has been - written. + With no explicit mapping, every existing file under ``root`` at call + time is included. A precomputed mapping supports evidence finalization, + where the inventory for the intended final manifest is published before + that manifest's last atomic rename. """ - checksums = compute_checksums(root) - lines = [f"{digest} {relative}" for relative, digest in sorted(checksums.items())] + resolved_checksums = compute_checksums(root) if checksums is None else checksums + lines = [f"{digest} {relative}" for relative, digest in sorted(resolved_checksums.items())] write_atomic(root / _CHECKSUMS_FILENAME, "\n".join(lines) + "\n" if lines else "") diff --git a/src/ragforge/evaluation/event_log.py b/src/ragforge/evaluation/event_log.py index 00b31d4..6df630f 100644 --- a/src/ragforge/evaluation/event_log.py +++ b/src/ragforge/evaluation/event_log.py @@ -60,6 +60,31 @@ def __init__(self, run_id: str, path: Path) -> None: self._lock = threading.Lock() self._sequence = 0 self._previous_event_hash: str | None = None + self._restore_tail() + + def _restore_tail(self) -> None: + """Continue an existing run's sequence and hash chain when resuming.""" + if not self._path.exists(): + return + lines = self._path.read_text(encoding="utf-8").splitlines() + if not lines: + return + payload: object = json.loads(lines[-1]) + if not isinstance(payload, dict): + raise ValueError(f"last event in {self._path} must be a JSON object") + stored_run_id = payload.get("run_id") + sequence = payload.get("sequence") + event_hash = payload.get("event_hash") + if stored_run_id != self._run_id: + raise ValueError( + f"event log run_id mismatch: expected {self._run_id!r}, found {stored_run_id!r}" + ) + if isinstance(sequence, bool) or not isinstance(sequence, int) or sequence < 1: + raise ValueError(f"last event in {self._path} has an invalid sequence") + if not isinstance(event_hash, str) or not event_hash: + raise ValueError(f"last event in {self._path} has an invalid event_hash") + self._sequence = sequence + self._previous_event_hash = event_hash def emit( self, diff --git a/src/ragforge/evaluation/ragas_judge.py b/src/ragforge/evaluation/ragas_judge.py index e38a004..90b4e7d 100644 --- a/src/ragforge/evaluation/ragas_judge.py +++ b/src/ragforge/evaluation/ragas_judge.py @@ -52,6 +52,7 @@ evaluate() as a whole - the granularity available at this boundary. """ +import asyncio import json import math import os @@ -150,7 +151,7 @@ def _build_async_openai_embeddings( class _ScoredMetric(Protocol): """Shape shared by ragas.metrics.collections' single-turn metric classes.""" - def score(self, **kwargs: object) -> object: ... # returns a ragas MetricResult (has .value) + async def ascore(self, **kwargs: object) -> object: ... class _MetricResult(Protocol): @@ -163,11 +164,17 @@ class _MetricResult(Protocol): class _GeneratingLLM(Protocol): """Shape of ragas.llms.InstructorLLM this module actually calls directly (abstention).""" - def generate( + async def agenerate( self, prompt: str, response_model: type[_AbstentionOutput] ) -> _AbstentionOutput: ... +class _AsyncCloseable(Protocol): + """Minimal asynchronous resource lifecycle used by provider clients.""" + + async def close(self) -> None: ... + + class RagasJudge: """Scores a JudgeSample for Faithfulness, Answer Relevancy, and abstention appropriateness.""" @@ -179,6 +186,7 @@ def __init__( identity: ModelIdentity, cache: LLMCache | None = None, max_in_flight: int = _DEFAULT_MAX_IN_FLIGHT, + closeables: tuple[_AsyncCloseable, ...] = (), ) -> None: """Wire the judge to its already-constructed RAGAS metrics and abstention LLM. @@ -193,6 +201,8 @@ def __init__( caching - every evaluate() call reaches the real metrics. max_in_flight: Bounds concurrent evaluate() calls to this provider, process-wide (ADR-0014). + closeables: Provider clients that must close on this judge's + persistent event loop before it is shut down. """ self._faithfulness = faithfulness self._answer_relevancy = answer_relevancy @@ -200,6 +210,9 @@ def __init__( self._identity = identity self._cache = cache self._limiter = get_limiter(identity.provider, max_in_flight) + self._closeables = closeables + self._runner = asyncio.Runner() + self._closed = False @property def identity(self) -> ModelIdentity: @@ -233,32 +246,37 @@ def evaluate(self, sample: JudgeSample) -> JudgeResult: ) def _evaluate_uncached(self, sample: JudgeSample) -> JudgeResult: + if self._closed: + raise GenerationError("RAGAS judge is closed") try: with self._limiter: - faithfulness_score = _score_metric( - self._faithfulness, - "faithfulness", - user_input=sample.question, - response=sample.answer, - retrieved_contexts=list(sample.contexts), - ) - answer_relevancy_score = _score_metric( - self._answer_relevancy, - "answer_relevancy", - user_input=sample.question, - response=sample.answer, - ) - abstention = self._abstention_llm.generate( - _ABSTENTION_PROMPT_TEMPLATE.format( - question=sample.question, - answerable="não" if sample.unanswerable else "sim", - answer=sample.answer, - ), - response_model=_AbstentionOutput, - ) + return self._runner.run(self._evaluate_async(sample)) except Exception as exc: raise GenerationError(f"RAGAS judge scoring failed: {exc}") from exc + async def _evaluate_async(self, sample: JudgeSample) -> JudgeResult: + """Score all dimensions on one persistent event loop owned by this worker.""" + faithfulness_score = await _score_metric( + self._faithfulness, + "faithfulness", + user_input=sample.question, + response=sample.answer, + retrieved_contexts=list(sample.contexts), + ) + answer_relevancy_score = await _score_metric( + self._answer_relevancy, + "answer_relevancy", + user_input=sample.question, + response=sample.answer, + ) + abstention = await self._abstention_llm.agenerate( + _ABSTENTION_PROMPT_TEMPLATE.format( + question=sample.question, + answerable="não" if sample.unanswerable else "sim", + answer=sample.answer, + ), + response_model=_AbstentionOutput, + ) return JudgeResult( schema_version=_OUTPUT_SCHEMA_VERSION, faithfulness=MetricScore(score=faithfulness_score), @@ -268,12 +286,23 @@ def _evaluate_uncached(self, sample: JudgeSample) -> JudgeResult: ), ) + def close(self) -> None: + """Close provider clients on their owning loop, then release the loop.""" + if self._closed: + return + try: + for closeable in self._closeables: + self._runner.run(closeable.close()) + finally: + self._runner.close() + self._closed = True + -def _score_metric(metric: _ScoredMetric, name: str, **kwargs: object) -> float: +async def _score_metric(metric: _ScoredMetric, name: str, **kwargs: object) -> float: """Return one valid bounded score, retrying a semantically invalid result once.""" invalid_value: float | None = None for _attempt in range(_METRIC_SCORE_ATTEMPTS): - result = cast(_MetricResult, metric.score(**kwargs)) + result = cast(_MetricResult, await metric.ascore(**kwargs)) value = float(result.value) if math.isfinite(value) and 0.0 <= value <= 1.0: return value @@ -359,8 +388,11 @@ def build_gemini_ragas_judge( ragas_embeddings = GoogleEmbeddings(client=genai_client, model=embedding_model_name) return RagasJudge( - faithfulness=Faithfulness(llm=ragas_llm), - answer_relevancy=AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embeddings), + faithfulness=cast(_ScoredMetric, Faithfulness(llm=ragas_llm)), + answer_relevancy=cast( + _ScoredMetric, + AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embeddings), + ), abstention_llm=abstention_llm, identity=ModelIdentity( provider="gemini", @@ -416,13 +448,16 @@ def build_openai_ragas_judge( if max_output_tokens <= 0: raise ValueError("max_output_tokens must be positive") try: - instructor_client = instructor.from_provider( - f"openai/{llm_model_name}", - async_client=True, - api_key=key, + openai_client = AsyncOpenAI(api_key=key) + instructor_client = instructor.from_openai( + openai_client, mode=instructor.Mode.RESPONSES_TOOLS, ) - ragas_embeddings = _build_async_openai_embeddings(key, embedding_model_name) + embeddings_client = AsyncOpenAI(api_key=key) + ragas_embeddings = OpenAIEmbeddings( + client=embeddings_client, + model=embedding_model_name, + ) except Exception as exc: raise GenerationError(f"failed to create RAGAS judge client: {exc}") from exc @@ -442,8 +477,11 @@ def build_openai_ragas_judge( system_prompt=_ABSTENTION_SYSTEM_PROMPT, ) return RagasJudge( - faithfulness=Faithfulness(llm=ragas_llm), - answer_relevancy=AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embeddings), + faithfulness=cast(_ScoredMetric, Faithfulness(llm=ragas_llm)), + answer_relevancy=cast( + _ScoredMetric, + AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embeddings), + ), abstention_llm=abstention_llm, identity=ModelIdentity( provider="openai", @@ -454,4 +492,5 @@ def build_openai_ragas_judge( ), cache=cache, max_in_flight=max_in_flight, + closeables=(openai_client, embeddings_client), ) diff --git a/src/ragforge/evaluation/records.py b/src/ragforge/evaluation/records.py index 998164e..94cacaa 100644 --- a/src/ragforge/evaluation/records.py +++ b/src/ragforge/evaluation/records.py @@ -127,6 +127,23 @@ def append_records_jsonl(path: Path, records: list[QuestionRecord]) -> None: handle.write("\n") +def replace_strategy_records_jsonl( + path: Path, + strategy: str, + records: list[QuestionRecord], +) -> None: + """Atomically replace every persisted record for one retried strategy.""" + if any(record.strategy != strategy for record in records): + raise ValueError("all replacement records must belong to the declared strategy") + retained = [record for record in read_records_jsonl(path) if record.strategy != strategy] + temporary_path = path.with_suffix(f"{path.suffix}.tmp") + with temporary_path.open("w", encoding="utf-8") as handle: + for record in [*retained, *records]: + handle.write(json.dumps(record.to_json_dict(), ensure_ascii=False)) + handle.write("\n") + temporary_path.replace(path) + + def read_records_jsonl(path: Path) -> list[QuestionRecord]: """Load stored records, keeping the latest unique strategy/question pair.""" if not path.exists(): diff --git a/src/ragforge/evaluation/run.py b/src/ragforge/evaluation/run.py index bb58b24..c7adfe2 100644 --- a/src/ragforge/evaluation/run.py +++ b/src/ragforge/evaluation/run.py @@ -29,18 +29,21 @@ Document discovery, expected article counts, and source hashes come only from the corpus manifest (datasets/regrag-br/corpus_manifest.yaml) and the -question selection only from the versioned split -(datasets/regrag-br/split.json) - both ADR-0012. A preflight gate +question selection from the versioned split +(datasets/regrag-br/split.json), optionally followed by the deterministic +stratified cost cap declared in the experiment configuration - both +ADR-0012. A preflight gate (ragforge.evaluation.integrity) validates source hashes, split/golden-set agreement, and structural-reference resolution before any indexing starts, and fails the run closed (SystemExit) rather than silently indexing a reduced or drifted corpus. RAPTOR is built once per document, never across a document boundary, so a summary node can never blend unrelated norms. Every -selected question gets one immutable QuestionRecord per strategy - including +selected question gets one QuestionRecord per strategy - including unanswerable-class questions, which are excluded from ranking/citation averages (ADR-0018: they are still generated and judged, for abstention -appropriateness) but never dropped from coverage - appended to -experiments//records.jsonl as each strategy finishes. +appropriateness) but never dropped from coverage - atomically persisted to +experiments//records.jsonl as each strategy finishes. A retried +incomplete strategy replaces only its own stale records. The embedding provider is provider-neutral and config-driven (ADR-0013): ``embedding.provider: local`` (operational default, no credentials needed, @@ -169,11 +172,7 @@ from ragforge.domain.models import Chunk, Judgment from ragforge.domain.protocols import RetrievalStrategy from ragforge.embeddings.caching import CachedEmbeddingModel -from ragforge.evaluation.artifact_writer import ( - compute_checksums, - write_atomic, - write_checksums_file, -) +from ragforge.evaluation.artifact_writer import write_atomic from ragforge.evaluation.audit_metrics import compute_audit_report from ragforge.evaluation.audit_ports import AuditResult from ragforge.evaluation.canonical_hash import canonical_json_hash @@ -187,10 +186,11 @@ verify_structural_references, ) from ragforge.evaluation.judgments import load_judgments -from ragforge.evaluation.lineage_ports import GenerationLineage +from ragforge.evaluation.lineage_ports import GenerationLineage, RunManifest from ragforge.evaluation.manifest import load_corpus_manifest -from ragforge.evaluation.records import append_records_jsonl, read_records_jsonl +from ragforge.evaluation.records import read_records_jsonl, replace_strategy_records_jsonl from ragforge.evaluation.run_evidence import ( + finalize_evidence_directory, reject_if_evidence_dir_already_completed, write_question_artifacts, write_summaries, @@ -198,7 +198,8 @@ from ragforge.evaluation.run_lock import BenchmarkAlreadyRunningError, BenchmarkRunLock from ragforge.evaluation.run_manifest import ( build_initial_manifest, - finalize_manifest, + load_run_manifest, + require_clean_worktree, resolve_git_sha, ) from ragforge.evaluation.run_reporting import ( @@ -219,6 +220,7 @@ build_contextual_strategy, ) from ragforge.evaluation.split import Split, load_split +from ragforge.evaluation.split_builder import select_stratified_sample from ragforge.generation.auditing_answer_generator import AuditingAnswerGenerator from ragforge.generation.gemini_answer_generator import GeminiAnswerGenerator from ragforge.generation.gemini_contextualizer import GeminiContextualizer @@ -280,6 +282,7 @@ _DEFAULT_GEMINI_MAX_IN_FLIGHT = 4 _DEFAULT_EMBEDDING_CACHE_DIR = ".ragforge/cache/embeddings" _DEFAULT_INDEX_CACHE_DIR = ".ragforge/cache/indexes" +_DEFAULT_SAMPLING_SEED = "regrag-br-benchmark-sample-v1" _BASE_STRATEGY_LABELS = ( "dense", "sparse_bm25", @@ -390,6 +393,24 @@ def _select_split_judgments( return [by_id[question_id] for question_id in split_ids] +def _strategy_checkpoint_complete(metrics: Mapping[str, object], expected_answers: int) -> bool: + """Return whether a strategy checkpoint has complete, error-free answer coverage.""" + answer_n = metrics.get("answer_n") + answer_errors = metrics.get("answer_errors") + retrieval_errors = metrics.get("errors") + return ( + isinstance(answer_n, (int, float)) + and not isinstance(answer_n, bool) + and float(answer_n) == expected_answers + and isinstance(answer_errors, (int, float)) + and not isinstance(answer_errors, bool) + and float(answer_errors) == 0.0 + and isinstance(retrieval_errors, (int, float)) + and not isinstance(retrieval_errors, bool) + and float(retrieval_errors) == 0.0 + ) + + def _verify_resume_identity( previous: Mapping[str, object], index_namespace: str, @@ -433,16 +454,68 @@ def _verify_resume_identity( ) +def _verify_resume_manifest_identity( + previous: RunManifest, + *, + run_id: str, + git_sha: str, + corpus_hash: str, + dataset_hash: str, + split_hash: str, + configuration_hash: str, + models: dict[str, str], + strategies: tuple[str, ...], + execution: dict[str, object], +) -> None: + """Fail closed if resumed evidence differs from the run that started it.""" + expected: dict[str, object] = { + "run_id": run_id, + "status": "running", + "git_sha": git_sha, + "corpus_hash": corpus_hash, + "dataset_hash": dataset_hash, + "split_hash": split_hash, + "configuration_hash": configuration_hash, + "models": models, + "strategies": strategies, + "execution": execution, + } + mismatches = [ + f"{field}: {getattr(previous, field)!r} != {value!r}" + for field, value in expected.items() + if getattr(previous, field) != value + ] + if mismatches: + raise SystemExit( + "--resume evidence identity mismatch; start a new run:\n" + + "\n".join(f"- {mismatch}" for mismatch in mismatches) + ) + + def _run() -> None: """Index the real corpus with every strategy and score each against the golden set.""" args = parse_args() _reject_cache_mode(args.mode) args.config = args.config.resolve() + require_clean_worktree(ROOT) + current_git_sha = resolve_git_sha(ROOT) + if current_git_sha == "unknown": + raise SystemExit("auditable benchmark could not resolve the current Git commit") config = yaml.safe_load(args.config.read_text(encoding="utf-8")) requested_strategies = _validate_requested_strategies(config.get("strategies")) requested_strategy_set = set(requested_strategies) split_name = config["dataset"]["split"] + max_questions = config["dataset"].get("max_questions") + sampling_seed = config["dataset"].get("sampling_seed", _DEFAULT_SAMPLING_SEED) + if max_questions is not None and ( + isinstance(max_questions, bool) or not isinstance(max_questions, int) + ): + raise SystemExit("dataset.max_questions must be a positive integer") + if max_questions is not None and max_questions <= 0: + raise SystemExit("dataset.max_questions must be a positive integer") + if not isinstance(sampling_seed, str) or not sampling_seed: + raise SystemExit("dataset.sampling_seed must be a non-empty string") top_k = config["retrieval"]["top_k"] rerank_pool = config["retrieval"]["rerank_pool"] embedding_provider = config["embedding"]["provider"] @@ -484,6 +557,19 @@ def _run() -> None: except IntegrityError as exc: raise SystemExit(f"preflight integrity check failed:\n{exc}") from exc judgments = _select_split_judgments(split, judgments, split_name) + if max_questions is not None: + try: + judgments = select_stratified_sample( + judgments, + max_questions=max_questions, + seed=sampling_seed, + ) + except ValueError as exc: + raise SystemExit(f"invalid dataset sample: {exc}") from exc + print( + f"Selected {len(judgments)} stratified questions " + f"(max_questions={max_questions}, seed={sampling_seed})." + ) run_id = args.resume or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") run_dir = RESULTS_DIR / run_id @@ -493,6 +579,15 @@ def _run() -> None: artifacts_dir = ARTIFACTS_DIR / run_id reject_if_evidence_dir_already_completed(artifacts_dir) + existing_run_manifest: RunManifest | None = None + if args.resume is not None: + manifest_path = artifacts_dir / "manifest.json" + if not manifest_path.exists(): + raise SystemExit(f"--resume requires an existing evidence manifest at {manifest_path}") + try: + existing_run_manifest = load_run_manifest(manifest_path) + except (OSError, ValueError, json.JSONDecodeError) as exc: + raise SystemExit(f"invalid resume evidence manifest: {exc}") from exc print("Extracting and chunking real corpus documents...") documents = _load_documents(manifest) @@ -544,31 +639,59 @@ def _run() -> None: _verify_resume_identity( previous, index_namespace, generation_model, judge_provider, judge_model ) - run_metrics = previous["metrics"] + previous_metrics = previous["metrics"] + run_metrics = { + label: metrics + for label, metrics in previous_metrics.items() + if _strategy_checkpoint_complete(metrics, len(judgments)) + } + incomplete_labels = sorted(set(previous_metrics) - set(run_metrics)) print(f"Resuming {run_id}: {sorted(run_metrics)} already scored.") + if incomplete_labels: + print(f" retrying incomplete checkpoints: {incomplete_labels}") pending_strategy_set = requested_strategy_set - set(run_metrics) print("Writing ADR-0017 evidence manifest and snapshots...") - run_manifest = build_initial_manifest( - run_id=run_id, - git_sha=resolve_git_sha(), - corpus_hash=manifest.content_hash, - dataset_hash=snapshot_hash(JUDGMENTS_PATH), - split_hash=snapshot_hash(SPLIT_PATH), - configuration_hash=canonical_json_hash(config), - models={ - "embedding": f"{embedding_provider}/{embedding_model}", - "generation": generation_model, - "judge": f"{judge_provider}/{judge_model}", - "audit": f"{audit_provider}/{audit_model}" if audit_enabled else "disabled", - }, - strategies=requested_strategies, - execution=dict(execution_config), - ) - write_atomic( - artifacts_dir / "manifest.json", - json.dumps(dataclasses.asdict(run_manifest), ensure_ascii=False, indent=2), - ) + dataset_hash = snapshot_hash(JUDGMENTS_PATH) + split_hash = snapshot_hash(SPLIT_PATH) + configuration_hash = canonical_json_hash(config) + model_identities = { + "embedding": f"{embedding_provider}/{embedding_model}", + "generation": generation_model, + "judge": f"{judge_provider}/{judge_model}", + "audit": f"{audit_provider}/{audit_model}" if audit_enabled else "disabled", + } + manifest_execution = dict(execution_config) + if existing_run_manifest is not None: + _verify_resume_manifest_identity( + existing_run_manifest, + run_id=run_id, + git_sha=current_git_sha, + corpus_hash=manifest.content_hash, + dataset_hash=dataset_hash, + split_hash=split_hash, + configuration_hash=configuration_hash, + models=model_identities, + strategies=requested_strategies, + execution=manifest_execution, + ) + run_manifest = existing_run_manifest + else: + run_manifest = build_initial_manifest( + run_id=run_id, + git_sha=current_git_sha, + corpus_hash=manifest.content_hash, + dataset_hash=dataset_hash, + split_hash=split_hash, + configuration_hash=configuration_hash, + models=model_identities, + strategies=requested_strategies, + execution=manifest_execution, + ) + write_atomic( + artifacts_dir / "manifest.json", + json.dumps(dataclasses.asdict(run_manifest), ensure_ascii=False, indent=2), + ) write_atomic( artifacts_dir / "configuration.resolved.yaml", args.config.read_text(encoding="utf-8") ) @@ -576,6 +699,20 @@ def _run() -> None: artifacts_dir / "corpus-manifest.snapshot.yaml", MANIFEST_PATH.read_text(encoding="utf-8") ) write_atomic(artifacts_dir / "split.snapshot.json", SPLIT_PATH.read_text(encoding="utf-8")) + write_atomic( + artifacts_dir / "question-selection.snapshot.json", + json.dumps( + { + "schema_version": 1, + "split": split_name, + "max_questions": max_questions, + "sampling_seed": sampling_seed if max_questions is not None else None, + "question_ids": [judgment.question_id for judgment in judgments], + }, + ensure_ascii=False, + indent=2, + ), + ) event_log = EventLog(run_id, artifacts_dir / "events.jsonl") print( @@ -736,7 +873,7 @@ def _checkpoint() -> None: results_path.write_text(json.dumps(record, ensure_ascii=False, indent=2)) def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: - """Score ``strategy``, append its records.jsonl lines, then checkpoint results.json. + """Score ``strategy``, replace its records.jsonl lines, then checkpoint results.json. A no-op when ``label`` is already in ``run_metrics`` (--resume). The outer orchestration also excludes completed labels from stage @@ -749,6 +886,7 @@ def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: event_log.emit("strategy", "started", {"label": label}) try: metrics, records, candidate_lineage = _evaluate( + label, strategy, judgments, generator, @@ -771,7 +909,7 @@ def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: audit_results_by_strategy[label] = audit_results metrics = {**metrics, **compute_audit_report(audit_results)} run_metrics[label] = metrics - append_records_jsonl(records_path, records) + replace_strategy_records_jsonl(records_path, label, records) write_question_artifacts(artifacts_dir, label, records, candidate_lineage) write_summaries( artifacts_dir, @@ -1058,13 +1196,7 @@ def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: f"## Answer quality\n\n```\n{answer_quality_table}\n```\n", ) - artifact_checksums = compute_checksums(artifacts_dir) - write_checksums_file(artifacts_dir) - final_manifest = finalize_manifest(run_manifest, canonical_json_hash(artifact_checksums)) - write_atomic( - artifacts_dir / "manifest.json", - json.dumps(dataclasses.asdict(final_manifest), ensure_ascii=False, indent=2), - ) + finalize_evidence_directory(artifacts_dir, run_manifest) print(f"Evidence directory finalized at {artifacts_dir.relative_to(ROOT)}/") diff --git a/src/ragforge/evaluation/run_evidence.py b/src/ragforge/evaluation/run_evidence.py index be3f45c..6c0f662 100644 --- a/src/ragforge/evaluation/run_evidence.py +++ b/src/ragforge/evaluation/run_evidence.py @@ -6,14 +6,27 @@ """ import dataclasses +import hashlib import json from pathlib import Path -from ragforge.evaluation.artifact_writer import write_atomic +from ragforge.evaluation.artifact_writer import ( + compute_checksums, + write_atomic, + write_checksums_file, +) from ragforge.evaluation.audit_metrics import compute_audit_report from ragforge.evaluation.audit_ports import AuditResult -from ragforge.evaluation.lineage_ports import GenerationLineage, RetrievalCandidateLineage +from ragforge.evaluation.canonical_hash import canonical_json_hash +from ragforge.evaluation.lineage_ports import ( + GenerationLineage, + RetrievalCandidateLineage, + RunManifest, +) from ragforge.evaluation.records import QuestionRecord +from ragforge.evaluation.run_manifest import finalize_manifest + +_MANIFEST_FILENAME = "manifest.json" def _load_json_object(path: Path) -> dict[str, object]: @@ -48,6 +61,46 @@ def reject_if_evidence_dir_already_completed(artifacts_dir: Path) -> None: ) +def finalize_evidence_directory( + artifacts_dir: Path, + run_manifest: RunManifest, +) -> RunManifest: + """Atomically publish a completed manifest covered by the checksum inventory. + + ``artifact_root_hash`` covers every final artifact except + ``manifest.json`` and ``checksums.sha256``. Excluding the manifest from + that root avoids a self-reference because the root itself is stored in + the manifest. ``checksums.sha256`` still includes the exact final + manifest bytes, so post-completion changes remain detectable. + + The checksum inventory is written before the final manifest. A crash + before the last atomic rename therefore leaves the manifest in + ``running`` state rather than exposing a completed-but-unverifiable run. + """ + if run_manifest.status != "running": + raise ValueError("only a running manifest can be finalized") + artifact_checksums = compute_checksums( + artifacts_dir, + excluded_paths=frozenset({_MANIFEST_FILENAME}), + ) + final_manifest = finalize_manifest( + run_manifest, + canonical_json_hash(artifact_checksums), + ) + final_manifest_content = json.dumps( + dataclasses.asdict(final_manifest), + ensure_ascii=False, + indent=2, + ) + complete_checksums = { + **artifact_checksums, + _MANIFEST_FILENAME: hashlib.sha256(final_manifest_content.encode("utf-8")).hexdigest(), + } + write_checksums_file(artifacts_dir, complete_checksums) + write_atomic(artifacts_dir / _MANIFEST_FILENAME, final_manifest_content) + return final_manifest + + def write_question_artifacts( artifacts_dir: Path, label: str, diff --git a/src/ragforge/evaluation/run_manifest.py b/src/ragforge/evaluation/run_manifest.py index 1906eec..5f10cd4 100644 --- a/src/ragforge/evaluation/run_manifest.py +++ b/src/ragforge/evaluation/run_manifest.py @@ -10,17 +10,20 @@ check already uses for a different kind of mismatch. """ +import json import shutil import subprocess # nosec B404 from datetime import UTC, datetime +from pathlib import Path from ragforge.evaluation.lineage_ports import RunManifest _SCHEMA_VERSION = 1 _UNKNOWN_GIT_SHA = "unknown" +_GENERATED_OUTPUT_PREFIXES = ("artifacts/", "experiments/", ".ragforge/") -def resolve_git_sha() -> str: +def resolve_git_sha(repository_root: Path | None = None) -> str: """Return the current commit's full SHA, or "unknown" if it can't be determined. Never raises: git absent, not a repository, or any other failure all @@ -35,6 +38,7 @@ def resolve_git_sha() -> str: # user input reaches this command. result = subprocess.run( # noqa: S603 # nosec B603 [git_path, "rev-parse", "HEAD"], + cwd=repository_root, capture_output=True, text=True, timeout=5, @@ -46,6 +50,104 @@ def resolve_git_sha() -> str: return sha if sha else _UNKNOWN_GIT_SHA +def require_clean_worktree(repository_root: Path) -> None: + """Fail closed when reproducibility-relevant Git content is not committed. + + Untracked benchmark output directories are ignored because creating and + resuming runs necessarily populates them. Tracked modifications under + those directories still fail the gate. + + Raises: + SystemExit: If Git is unavailable, status cannot be read, or relevant + staged, unstaged, or untracked changes exist. + """ + git_path = shutil.which("git") + if git_path is None: + raise SystemExit("auditable benchmark requires Git, but git was not found") + try: + result = subprocess.run( # noqa: S603 # nosec B603 + [git_path, "status", "--porcelain=v1", "--untracked-files=all"], + cwd=repository_root, + capture_output=True, + text=True, + timeout=10, + check=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise SystemExit(f"failed to inspect Git worktree: {exc}") from exc + + dirty_entries = [] + for line in result.stdout.splitlines(): + if not line: + continue + path = line[3:] if len(line) > 3 else "" + is_generated_output = line.startswith("?? ") and path.startswith(_GENERATED_OUTPUT_PREFIXES) + if not is_generated_output: + dirty_entries.append(line) + if dirty_entries: + preview = "\n".join(f"- {entry}" for entry in dirty_entries[:20]) + raise SystemExit( + "auditable benchmark requires a clean Git worktree; commit or discard " + f"these changes before running:\n{preview}" + ) + + +def load_run_manifest(path: Path) -> RunManifest: + """Load and validate this project's own persisted manifest contract.""" + raw: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError(f"run manifest {path} must contain a JSON object") + + def _required_str(field: str) -> str: + value = raw.get(field) + if not isinstance(value, str) or not value: + raise ValueError(f"run manifest field {field!r} must be a non-empty string") + return value + + schema_version = raw.get("schema_version") + if isinstance(schema_version, bool) or not isinstance(schema_version, int): + raise ValueError("run manifest field 'schema_version' must be an integer") + completed_at_raw = raw.get("completed_at") + if completed_at_raw is not None and not isinstance(completed_at_raw, str): + raise ValueError("run manifest field 'completed_at' must be a string or null") + artifact_root_hash_raw = raw.get("artifact_root_hash") + if artifact_root_hash_raw is not None and not isinstance(artifact_root_hash_raw, str): + raise ValueError("run manifest field 'artifact_root_hash' must be a string or null") + + models_raw = raw.get("models") + if not isinstance(models_raw, dict) or any( + not isinstance(key, str) or not isinstance(value, str) for key, value in models_raw.items() + ): + raise ValueError("run manifest field 'models' must map strings to strings") + strategies_raw = raw.get("strategies") + if not isinstance(strategies_raw, list) or any( + not isinstance(value, str) for value in strategies_raw + ): + raise ValueError("run manifest field 'strategies' must be a list of strings") + execution_raw = raw.get("execution") + if not isinstance(execution_raw, dict) or any( + not isinstance(key, str) for key in execution_raw + ): + raise ValueError("run manifest field 'execution' must be an object") + + return RunManifest( + schema_version=schema_version, + run_id=_required_str("run_id"), + status=_required_str("status"), + git_sha=_required_str("git_sha"), + started_at=_required_str("started_at"), + completed_at=completed_at_raw, + corpus_hash=_required_str("corpus_hash"), + dataset_hash=_required_str("dataset_hash"), + split_hash=_required_str("split_hash"), + configuration_hash=_required_str("configuration_hash"), + models={str(key): str(value) for key, value in models_raw.items()}, + strategies=tuple(str(value) for value in strategies_raw), + execution={str(key): value for key, value in execution_raw.items()}, + artifact_root_hash=artifact_root_hash_raw, + ) + + def build_initial_manifest( *, run_id: str, diff --git a/src/ragforge/evaluation/run_strategies.py b/src/ragforge/evaluation/run_strategies.py index b1a81ec..87584b3 100644 --- a/src/ragforge/evaluation/run_strategies.py +++ b/src/ragforge/evaluation/run_strategies.py @@ -273,6 +273,7 @@ def build_contextual_strategy( def _evaluate( + strategy_label: str, strategy: RetrievalStrategy, judgments: list[Judgment], generator: AnswerGenerator, @@ -281,13 +282,15 @@ def _evaluate( answer_quality_workers: int, embedding_identity_hash: str | None = None, ) -> tuple[dict[str, float], list[QuestionRecord], list[RetrievalCandidateLineage]]: - """Score ``strategy`` for retrieval ranking and answer quality alike (ADR-0002/0007). + """Score ``strategy`` under its canonical experiment label (ADR-0002/0007). Merges evaluate_strategy's and evaluate_answer_quality's per-question records into one QuestionRecord per judgment (ADR-0012), alongside the same aggregate metrics dict this returned before. ``embedding_identity_hash`` (ADR-0017), when given, is forwarded to evaluate_strategy to populate - per-candidate retrieval lineage. + per-candidate retrieval lineage. ``strategy_label`` deliberately comes + from the experiment configuration: implementation names such as + ``sparse`` and ``hybrid`` are not stable result identifiers. """ retrieval_result = evaluate_strategy( strategy, judgments, k=top_k, embedding_identity_hash=embedding_identity_hash @@ -300,6 +303,10 @@ def _evaluate( k=top_k, max_workers=answer_quality_workers, ) - records = merge_question_records(strategy.name, retrieval_result.records, answer_result.records) + records = merge_question_records( + strategy_label, + retrieval_result.records, + answer_result.records, + ) metrics = {**retrieval_result.metrics, **answer_result.metrics} return metrics, records, retrieval_result.candidate_lineage diff --git a/src/ragforge/evaluation/split_builder.py b/src/ragforge/evaluation/split_builder.py index ccba752..5cdc16f 100644 --- a/src/ragforge/evaluation/split_builder.py +++ b/src/ragforge/evaluation/split_builder.py @@ -1,11 +1,14 @@ """Deterministic stratified split construction for RegRAG-BR (ADR-0003).""" import hashlib +import math from collections import defaultdict from ragforge.domain.models import Judgment from ragforge.evaluation.split import Split +_SAMPLE_ALGORITHM_VERSION = "stratified-capacity-v1" + def build_stratified_split( judgments: list[Judgment], @@ -68,3 +71,72 @@ def build_stratified_split( validation=validation, test=test, ) + + +def select_stratified_sample( + judgments: list[Judgment], + *, + max_questions: int, + seed: str, +) -> list[Judgment]: + """Select an exact-size deterministic sample while preserving every query class. + + One slot is reserved for each class, then the remaining capacity is + apportioned proportionally using the largest-remainder method. Selection + within each class is ranked by a versioned SHA-256 score; returned + judgments retain their original split order. + + Args: + judgments: Judgments already selected from one declared split. + max_questions: Exact sample size, unless the split is smaller. + seed: Versioned sampling seed recorded in the resolved configuration. + + Raises: + ValueError: If the requested size cannot represent every query class. + """ + if max_questions <= 0: + raise ValueError("max_questions must be positive") + if max_questions >= len(judgments): + return list(judgments) + + by_class: dict[str, list[Judgment]] = defaultdict(list) + for judgment in judgments: + if judgment.query.query_class is None: + raise ValueError(f"judgment {judgment.question_id!r} has no query class") + by_class[judgment.query.query_class.value].append(judgment) + if max_questions < len(by_class): + raise ValueError( + f"max_questions must be at least {len(by_class)} to represent every query class" + ) + + remaining = max_questions - len(by_class) + total_capacity = len(judgments) - len(by_class) + allocation: dict[str, int] = dict.fromkeys(by_class, 1) + remainders: list[tuple[float, str]] = [] + allocated_extra = 0 + for query_class, class_judgments in sorted(by_class.items()): + capacity = len(class_judgments) - 1 + ideal_extra = remaining * capacity / total_capacity + extra = min(capacity, math.floor(ideal_extra)) + allocation[query_class] += extra + allocated_extra += extra + remainders.append((ideal_extra - extra, query_class)) + for _remainder, query_class in sorted(remainders, key=lambda item: (-item[0], item[1])): + if allocated_extra >= remaining: + break + if allocation[query_class] < len(by_class[query_class]): + allocation[query_class] += 1 + allocated_extra += 1 + + selected_ids: set[str] = set() + for query_class, class_judgments in sorted(by_class.items()): + ranked = sorted( + class_judgments, + key=lambda judgment: hashlib.sha256( + ( + f"{_SAMPLE_ALGORITHM_VERSION}:{seed}:{query_class}:{judgment.question_id}" + ).encode() + ).hexdigest(), + ) + selected_ids.update(judgment.question_id for judgment in ranked[: allocation[query_class]]) + return [judgment for judgment in judgments if judgment.question_id in selected_ids] diff --git a/tests/unit/test_answer_harness.py b/tests/unit/test_answer_harness.py index 5ee3ceb..3e7f4ee 100644 --- a/tests/unit/test_answer_harness.py +++ b/tests/unit/test_answer_harness.py @@ -154,6 +154,15 @@ def evaluate(self, sample: JudgeSample) -> JudgeResult: return _judge_result() +class _ClosableTrackingJudge(_FactoryTrackingJudge): + def __init__(self) -> None: + super().__init__() + self.close_calls = 0 + + def close(self) -> None: + self.close_calls += 1 + + def test_evaluate_answer_quality_averages_citation_accuracy_across_judgments() -> None: """One perfectly-cited answer and one uncited answer average to 0.5 citation accuracy.""" generator = _FakeGenerator( @@ -423,11 +432,9 @@ def test_evaluate_answer_quality_never_shares_one_judge_instance_across_threads( """judge_factory builds a separate judge per worker thread, never shared across threads. Regression test: a real live run shared one RagasJudge across concurrent - worker threads. Each RagasJudge.evaluate() call does asyncio.run(...) - internally (a fresh event loop per call, reusing the same async client - underneath); calling that concurrently from multiple threads against one - shared judge corrupted its connection pool - observed for real as - thousands of leaked CLOSE_WAIT sockets and a stalled run. + worker threads. Concurrent use of one async client and event loop + corrupted its connection pool - observed for real as thousands of leaked + CLOSE_WAIT sockets and a stalled run. """ count = 10 generator = _FakeGenerator( @@ -450,3 +457,31 @@ def factory() -> _FactoryTrackingJudge: assert len(built_instances) <= 3, "at most one judge instance per worker thread" for judge in built_instances: assert len(judge.thread_ids) == 1, "a judge instance was used from more than one thread" + + +def test_evaluate_answer_quality_closes_every_thread_local_judge() -> None: + """Every worker-owned judge releases its persistent loop and provider clients.""" + count = 6 + generator = _FakeGenerator( + {f"q{i}": Answer(text=f"answer {i}", citations=(ART_1,)) for i in range(count)} + ) + built_instances: list[_ClosableTrackingJudge] = [] + lock = threading.Lock() + + def factory() -> _ClosableTrackingJudge: + with lock: + judge = _ClosableTrackingJudge() + built_instances.append(judge) + return judge + + evaluate_answer_quality( + _FakeStrategy(), + [_judgment(f"q{i}", ART_1) for i in range(count)], + generator, + factory, + k=5, + max_workers=3, + ) + + assert built_instances + assert all(judge.close_calls == 1 for judge in built_instances) diff --git a/tests/unit/test_artifact_writer.py b/tests/unit/test_artifact_writer.py index 36af932..69cefcb 100644 --- a/tests/unit/test_artifact_writer.py +++ b/tests/unit/test_artifact_writer.py @@ -63,6 +63,19 @@ def test_compute_checksums_excludes_the_checksums_file_itself(tmp_path: Path) -> assert "a.txt" in checksums +def test_compute_checksums_excludes_explicit_non_root_artifacts(tmp_path: Path) -> None: + """A self-referential manifest can be omitted from the artifact root calculation.""" + (tmp_path / "manifest.json").write_text("manifest", encoding="utf-8") + (tmp_path / "report.json").write_text("report", encoding="utf-8") + + checksums = compute_checksums( + tmp_path, + excluded_paths=frozenset({"manifest.json"}), + ) + + assert checksums == {"report.json": snapshot_hash(tmp_path / "report.json")} + + def test_write_checksums_file_matches_compute_checksums(tmp_path: Path) -> None: """The written checksums.sha256 file's contents exactly reflect compute_checksums' output.""" (tmp_path / "a.txt").write_text("aaa", encoding="utf-8") @@ -76,6 +89,15 @@ def test_write_checksums_file_matches_compute_checksums(tmp_path: Path) -> None: assert parsed == expected +def test_write_checksums_file_accepts_a_precomputed_final_inventory(tmp_path: Path) -> None: + """Finalization can list the intended final manifest before publishing it.""" + write_checksums_file(tmp_path, {"manifest.json": "abc123"}) + + assert (tmp_path / "checksums.sha256").read_text(encoding="utf-8") == ( + "abc123 manifest.json\n" + ) + + def test_write_checksums_file_on_empty_directory_writes_empty_file(tmp_path: Path) -> None: """An empty root produces an empty (not missing) checksums.sha256.""" write_checksums_file(tmp_path) diff --git a/tests/unit/test_benchmark_run.py b/tests/unit/test_benchmark_run.py index 9809dd8..4a369e7 100644 --- a/tests/unit/test_benchmark_run.py +++ b/tests/unit/test_benchmark_run.py @@ -34,8 +34,11 @@ _resolve_embedding_cache_dir, _resolve_index_cache_dir, _select_split_judgments, + _strategy_checkpoint_complete, _validate_requested_strategies, + _verify_resume_manifest_identity, ) +from ragforge.evaluation.run_manifest import build_initial_manifest from ragforge.evaluation.run_reporting import ( build_run_record, format_answer_quality_table, @@ -123,6 +126,91 @@ def test_select_split_judgments_uses_declared_partition_order() -> None: assert [judgment.question_id for judgment in selected] == ["q3", "q1"] +def test_strategy_checkpoint_requires_complete_error_free_answer_coverage() -> None: + """Resume skips only a strategy whose answer evaluation fully succeeded.""" + assert _strategy_checkpoint_complete( + {"answer_n": 60.0, "answer_errors": 0.0, "errors": 0.0}, + expected_answers=60, + ) + + +@pytest.mark.parametrize( + "metrics", + [ + {"answer_n": 0.0, "answer_errors": 10.0, "errors": 0.0}, + {"answer_n": 59.0, "answer_errors": 1.0, "errors": 0.0}, + {"answer_n": 60.0, "answer_errors": 0.0, "errors": 1.0}, + {"answer_n": True, "answer_errors": 0.0, "errors": 0.0}, + {"answer_n": 60.0, "answer_errors": 0.0}, + ], +) +def test_strategy_checkpoint_retries_incomplete_or_failed_results( + metrics: dict[str, object], +) -> None: + """Quota failures and partial checkpoints must be retried on --resume.""" + assert not _strategy_checkpoint_complete(metrics, expected_answers=60) + + +def test_resume_manifest_accepts_the_original_running_identity() -> None: + """Resume preserves the original started_at when every identity remains equal.""" + manifest = build_initial_manifest( + run_id="run-1", + git_sha="abc123", + corpus_hash="corpus", + dataset_hash="dataset", + split_hash="split", + configuration_hash="config", + models={"judge": "openai/model"}, + strategies=("dense",), + execution={"workers": 2}, + ) + + _verify_resume_manifest_identity( + manifest, + run_id="run-1", + git_sha="abc123", + corpus_hash="corpus", + dataset_hash="dataset", + split_hash="split", + configuration_hash="config", + models={"judge": "openai/model"}, + strategies=("dense",), + execution={"workers": 2}, + ) + + assert manifest.status == "running" + assert manifest.completed_at is None + + +def test_resume_manifest_rejects_a_different_git_commit() -> None: + """A clean but different commit cannot continue evidence from the original code.""" + manifest = build_initial_manifest( + run_id="run-1", + git_sha="original", + corpus_hash="corpus", + dataset_hash="dataset", + split_hash="split", + configuration_hash="config", + models={}, + strategies=("dense",), + execution={}, + ) + + with pytest.raises(SystemExit, match="git_sha"): + _verify_resume_manifest_identity( + manifest, + run_id="run-1", + git_sha="different", + corpus_hash="corpus", + dataset_hash="dataset", + split_hash="split", + configuration_hash="config", + models={}, + strategies=("dense",), + execution={}, + ) + + class _FakeEmbedder: name = "fake-embedder" dimensions = 3 @@ -464,8 +552,8 @@ def evaluate(self, sample: JudgeSample) -> JudgeResult: ) -def test_evaluate_merges_retrieval_and_answer_records_tagged_with_the_strategy_name() -> None: - """_evaluate returns metrics from both harnesses plus one merged record per question.""" +def test_evaluate_tags_records_with_the_canonical_experiment_label() -> None: + """Configured aliases override internal implementation names in persisted records.""" judgments = [ Judgment( question_id="q1", @@ -477,6 +565,7 @@ def test_evaluate_merges_retrieval_and_answer_records_tagged_with_the_strategy_n ] metrics, records, candidate_lineage = _evaluate( + "configured-label", _FakeStrategyForEvaluate(), judgments, _FakeGeneratorForEvaluate(), @@ -488,7 +577,7 @@ def test_evaluate_merges_retrieval_and_answer_records_tagged_with_the_strategy_n assert metrics["recall_at_k"] == 1.0 assert metrics["citation_accuracy"] == 1.0 assert len(records) == 1 - assert records[0].strategy == "fake-strategy" + assert records[0].strategy == "configured-label" assert records[0].retrieval_status == "succeeded" assert records[0].generation_status == "succeeded" assert candidate_lineage == [], "no embedding_identity_hash was passed, so no lineage collected" @@ -507,6 +596,7 @@ def test_evaluate_populates_candidate_lineage_when_embedding_identity_hash_is_gi ] _, _, candidate_lineage = _evaluate( + "configured-label", _FakeStrategyForEvaluate(), judgments, _FakeGeneratorForEvaluate(), diff --git a/tests/unit/test_event_log.py b/tests/unit/test_event_log.py index ee9a929..e3778c5 100644 --- a/tests/unit/test_event_log.py +++ b/tests/unit/test_event_log.py @@ -5,6 +5,8 @@ import json from pathlib import Path +import pytest + from ragforge.evaluation.event_log import EventLog, compute_event_hash from ragforge.evaluation.lineage_ports import EventEnvelope @@ -43,6 +45,29 @@ def test_emit_writes_one_json_line_per_event(tmp_path: Path) -> None: assert [json.loads(line)["sequence"] for line in lines] == [1, 2] +def test_reopened_log_continues_the_existing_sequence_and_hash_chain(tmp_path: Path) -> None: + """--resume appends to ADR-0017 evidence instead of starting a second chain.""" + path = tmp_path / "events.jsonl" + original = EventLog("run-1", path) + original.emit("indexing", "started", {"stage": "base"}) + previous = original.emit("indexing", "completed", {"stage": "base"}) + + resumed = EventLog("run-1", path) + appended = resumed.emit("strategy", "started", {"label": "dense"}) + + assert appended.sequence == 3 + assert appended.previous_event_hash == previous.event_hash + + +def test_reopened_log_rejects_a_different_run_id(tmp_path: Path) -> None: + """Evidence from one run cannot be extended under another run identity.""" + path = tmp_path / "events.jsonl" + EventLog("run-1", path).emit("indexing", "started", {"stage": "base"}) + + with pytest.raises(ValueError, match="run_id mismatch"): + EventLog("run-2", path) + + def test_emit_defaults_correlation_id_to_the_run_id(tmp_path: Path) -> None: """When no correlation_id is given, it defaults to the run's own run_id.""" log = EventLog("run-42", tmp_path / "events.jsonl") diff --git a/tests/unit/test_ragas_judge.py b/tests/unit/test_ragas_judge.py index 1b93192..1f6631a 100644 --- a/tests/unit/test_ragas_judge.py +++ b/tests/unit/test_ragas_judge.py @@ -1,5 +1,6 @@ """Tests for RagasJudge, using fakes for RAGAS metrics and the abstention LLM (no network).""" +import asyncio from collections.abc import Callable from pathlib import Path from typing import Any @@ -32,7 +33,7 @@ def __init__(self, handler: Callable[..., _FakeMetricResult]) -> None: self._handler = handler self.calls: list[dict[str, Any]] = [] - def score(self, **kwargs: Any) -> _FakeMetricResult: + async def ascore(self, **kwargs: Any) -> _FakeMetricResult: self.calls.append(kwargs) return self._handler(**kwargs) @@ -49,11 +50,19 @@ def __init__(self, appropriate: bool = True, rationale: str = "ok") -> None: self._rationale = rationale self.calls: list[tuple[str, Any]] = [] - def generate(self, prompt: str, response_model: Any) -> Any: + async def agenerate(self, prompt: str, response_model: Any) -> Any: self.calls.append((prompt, response_model)) return _FakeAbstentionResult(self._appropriate, self._rationale) +class _TrackingCloseable: + def __init__(self) -> None: + self.loop_ids: list[int] = [] + + async def close(self) -> None: + self.loop_ids.append(id(asyncio.get_running_loop())) + + def _sample( question: str = "pergunta", answer: str = "resposta", *, unanswerable: bool = False ) -> JudgeSample: @@ -84,6 +93,46 @@ def test_evaluate_returns_faithfulness_answer_relevancy_and_abstention() -> None assert result.schema_version == 2 +def test_evaluate_reuses_one_event_loop_and_closes_clients_on_that_loop() -> None: + """Async clients never cross the short-lived loops that caused live-run failures.""" + evaluation_loop_ids: list[int] = [] + + def record_loop(**kwargs: Any) -> _FakeMetricResult: + evaluation_loop_ids.append(id(asyncio.get_running_loop())) + return _FakeMetricResult(1.0) + + closeable = _TrackingCloseable() + judge = RagasJudge( + _FakeMetric(record_loop), + _FakeMetric(record_loop), + _FakeAbstentionLLM(), + _IDENTITY, + closeables=(closeable,), + ) + + judge.evaluate(_sample(question="q1")) + judge.evaluate(_sample(question="q2")) + judge.close() + judge.close() + + assert len(set(evaluation_loop_ids)) == 1 + assert closeable.loop_ids == [evaluation_loop_ids[0]] + + +def test_evaluate_rejects_use_after_close() -> None: + """A closed judge fails explicitly instead of leaking an un-awaited coroutine.""" + judge = RagasJudge( + _FakeMetric(lambda **kwargs: _FakeMetricResult(1.0)), + _FakeMetric(lambda **kwargs: _FakeMetricResult(1.0)), + _FakeAbstentionLLM(), + _IDENTITY, + ) + judge.close() + + with pytest.raises(GenerationError, match="closed"): + judge.evaluate(_sample()) + + def test_identity_property_returns_the_configured_identity() -> None: """.identity exposes the exact judge configuration, for the run manifest.""" judge = RagasJudge( @@ -189,7 +238,7 @@ def test_evaluate_raises_generation_error_when_the_abstention_call_fails() -> No """A failure in this project's own abstention call is also translated to GenerationError.""" class _FailingAbstentionLLM: - def generate(self, prompt: str, response_model: Any) -> Any: + async def agenerate(self, prompt: str, response_model: Any) -> Any: raise RuntimeError("boom") judge = RagasJudge( diff --git a/tests/unit/test_records.py b/tests/unit/test_records.py index 4fed0bd..b5cb3ce 100644 --- a/tests/unit/test_records.py +++ b/tests/unit/test_records.py @@ -10,6 +10,8 @@ RetrievalRecord, append_records_jsonl, merge_question_records, + read_records_jsonl, + replace_strategy_records_jsonl, ) @@ -105,3 +107,32 @@ def test_append_records_jsonl_is_idempotent_for_a_resumed_strategy(tmp_path: Pat append_records_jsonl(path, records) assert len(path.read_text(encoding="utf-8").splitlines()) == 1 + + +def test_replace_strategy_records_jsonl_removes_failed_resume_records(tmp_path: Path) -> None: + """Retrying a strategy replaces stale outcomes while preserving other strategies.""" + path = tmp_path / "records.jsonl" + stale = merge_question_records( + "dense", + [_retrieval("q1", status="failed", error="quota")], + [_answer("q1", status="failed", error="quota")], + ) + preserved = merge_question_records( + "sparse_bm25", + [_retrieval("q1")], + [_answer("q1")], + ) + replacement = merge_question_records( + "dense", + [_retrieval("q1")], + [_answer("q1")], + ) + append_records_jsonl(path, [*stale, *preserved]) + + replace_strategy_records_jsonl(path, "dense", replacement) + + records = read_records_jsonl(path) + assert len(records) == 2 + by_strategy = {record.strategy: record for record in records} + assert by_strategy["dense"].errors == () + assert by_strategy["sparse_bm25"] == preserved[0] diff --git a/tests/unit/test_run_evidence_summaries.py b/tests/unit/test_run_evidence_summaries.py index dce936c..eeeb2da 100644 --- a/tests/unit/test_run_evidence_summaries.py +++ b/tests/unit/test_run_evidence_summaries.py @@ -1,9 +1,17 @@ """Tests for resumable benchmark summary persistence.""" +import dataclasses +import json from pathlib import Path +import pytest + +from ragforge.evaluation.artifact_writer import write_atomic +from ragforge.evaluation.canonical_hash import canonical_json_hash from ragforge.evaluation.lineage_ports import GenerationLineage -from ragforge.evaluation.run_evidence import write_summaries +from ragforge.evaluation.run_evidence import finalize_evidence_directory, write_summaries +from ragforge.evaluation.run_manifest import build_initial_manifest +from ragforge.ingestion.snapshot import snapshot_hash def _lineage(model: str) -> GenerationLineage: @@ -45,3 +53,59 @@ def test_write_summaries_merges_prior_strategies_on_resume(tmp_path: Path) -> No generation = (tmp_path / "summaries" / "generation.json").read_text(encoding="utf-8") assert '"dense"' in generation assert '"sparse"' in generation + + +def test_finalize_evidence_covers_the_final_manifest_without_a_circular_root( + tmp_path: Path, +) -> None: + """The completed manifest verifies while its root excludes only self-reference.""" + manifest = build_initial_manifest( + run_id="run-1", + git_sha="abc123", + corpus_hash="corpus", + dataset_hash="dataset", + split_hash="split", + configuration_hash="config", + models={}, + strategies=("dense",), + execution={}, + ) + write_atomic( + tmp_path / "manifest.json", + json.dumps(dataclasses.asdict(manifest), ensure_ascii=False, indent=2), + ) + write_atomic(tmp_path / "report.json", "{}") + + final_manifest = finalize_evidence_directory(tmp_path, manifest) + + recorded = { + relative: digest + for digest, relative in ( + line.split(" ", 1) + for line in (tmp_path / "checksums.sha256").read_text(encoding="utf-8").splitlines() + ) + } + assert final_manifest.status == "completed" + assert recorded["manifest.json"] == snapshot_hash(tmp_path / "manifest.json") + assert final_manifest.artifact_root_hash == canonical_json_hash( + {relative: digest for relative, digest in recorded.items() if relative != "manifest.json"} + ) + + +def test_finalize_evidence_rejects_an_already_completed_manifest(tmp_path: Path) -> None: + """Completed evidence cannot be silently checksummed and overwritten again.""" + manifest = build_initial_manifest( + run_id="run-1", + git_sha="abc123", + corpus_hash="corpus", + dataset_hash="dataset", + split_hash="split", + configuration_hash="config", + models={}, + strategies=("dense",), + execution={}, + ) + completed = finalize_evidence_directory(tmp_path, manifest) + + with pytest.raises(ValueError, match="running manifest"): + finalize_evidence_directory(tmp_path, completed) diff --git a/tests/unit/test_run_manifest.py b/tests/unit/test_run_manifest.py index 846e45a..07e6f4f 100644 --- a/tests/unit/test_run_manifest.py +++ b/tests/unit/test_run_manifest.py @@ -1,6 +1,9 @@ """Tests for run manifest lifecycle (ADR-0017).""" +import dataclasses +import json import subprocess +from pathlib import Path import pytest @@ -8,7 +11,9 @@ from ragforge.evaluation.run_manifest import ( build_initial_manifest, finalize_manifest, + load_run_manifest, reject_if_already_completed, + require_clean_worktree, resolve_git_sha, ) @@ -70,6 +75,85 @@ def test_build_initial_manifest_starts_running_with_no_completion_fields() -> No assert manifest.run_id == "run-1" +def test_load_run_manifest_preserves_the_original_start_identity(tmp_path: Path) -> None: + """A resumed run reloads, rather than reconstructs, its running manifest.""" + manifest = _manifest() + path = tmp_path / "manifest.json" + path.write_text(json.dumps(dataclasses.asdict(manifest)), encoding="utf-8") + + loaded = load_run_manifest(path) + + assert loaded == manifest + assert loaded.started_at == manifest.started_at + assert loaded.git_sha == manifest.git_sha + + +def test_require_clean_worktree_allows_only_untracked_generated_outputs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Run output directories do not prevent a later clean resume.""" + monkeypatch.setattr("ragforge.evaluation.run_manifest.shutil.which", lambda _name: "git") + result = subprocess.CompletedProcess( + args=["git"], + returncode=0, + stdout=( + "?? artifacts/runs/run-1/report.json\n" + "?? experiments/run-1/results.json\n" + "?? .ragforge/cache/indexes/marker.json\n" + ), + stderr="", + ) + monkeypatch.setattr( + "ragforge.evaluation.run_manifest.subprocess.run", + lambda *_args, **_kwargs: result, + ) + + require_clean_worktree(tmp_path) + + +def test_require_clean_worktree_rejects_staged_unstaged_or_untracked_source( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The manifest Git SHA may not conceal code or documentation changes.""" + monkeypatch.setattr("ragforge.evaluation.run_manifest.shutil.which", lambda _name: "git") + result = subprocess.CompletedProcess( + args=["git"], + returncode=0, + stdout=" M src/ragforge/app.py\nA wiki/Home.md\n?? configs/local.yaml\n", + stderr="", + ) + monkeypatch.setattr( + "ragforge.evaluation.run_manifest.subprocess.run", + lambda *_args, **_kwargs: result, + ) + + with pytest.raises(SystemExit, match="clean Git worktree"): + require_clean_worktree(tmp_path) + + +def test_require_clean_worktree_rejects_a_tracked_output_change( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Only untracked generated output is exempt; tracked evidence remains protected.""" + monkeypatch.setattr("ragforge.evaluation.run_manifest.shutil.which", lambda _name: "git") + result = subprocess.CompletedProcess( + args=["git"], + returncode=0, + stdout=" M experiments/published/results.json\n", + stderr="", + ) + monkeypatch.setattr( + "ragforge.evaluation.run_manifest.subprocess.run", + lambda *_args, **_kwargs: result, + ) + + with pytest.raises(SystemExit, match="clean Git worktree"): + require_clean_worktree(tmp_path) + + def test_finalize_manifest_marks_completed_and_sets_artifact_root_hash() -> None: """finalize_manifest returns a copy with status="completed" and the given root hash set.""" manifest = _manifest() diff --git a/tests/unit/test_split_builder.py b/tests/unit/test_split_builder.py index 8aa7086..6f204d3 100644 --- a/tests/unit/test_split_builder.py +++ b/tests/unit/test_split_builder.py @@ -1,7 +1,11 @@ """Tests for deterministic stratified RegRAG-BR split construction.""" +from collections import Counter + +import pytest + from ragforge.domain.models import Judgment, Query, QueryClass -from ragforge.evaluation.split_builder import build_stratified_split +from ragforge.evaluation.split_builder import build_stratified_split, select_stratified_sample def _judgment(question_id: str, query_class: QueryClass) -> Judgment: @@ -57,3 +61,51 @@ def test_build_stratified_split_rejects_invalid_ratio() -> None: assert "validation_ratio" in str(exc) else: raise AssertionError("expected ValueError") + + +def test_select_stratified_sample_is_exact_deterministic_and_preserves_order() -> None: + """The cost cap is exact, reproducible, class-aware, and keeps split order.""" + judgments = [ + _judgment(f"{query_class.value}-{index}", query_class) + for query_class in QueryClass + for index in range(10) + ] + + first = select_stratified_sample(judgments, max_questions=20, seed="stable") + second = select_stratified_sample(judgments, max_questions=20, seed="stable") + + assert first == second + assert len(first) == 20 + assert [judgments.index(item) for item in first] == sorted( + judgments.index(item) for item in first + ) + counts = Counter(item.query.query_class for item in first) + assert set(counts) == set(QueryClass) + assert max(counts.values()) - min(counts.values()) <= 1 + + +def test_select_stratified_sample_apportions_uneven_classes_proportionally() -> None: + """Larger classes receive more of the remaining capacity without excluding small ones.""" + judgments = [ + *[_judgment(f"exact-{index}", QueryClass.EXACT_FACTUAL) for index in range(8)], + *[_judgment(f"global-{index}", QueryClass.GLOBAL) for index in range(2)], + ] + + sampled = select_stratified_sample(judgments, max_questions=5, seed="stable") + + counts = Counter(item.query.query_class for item in sampled) + assert counts == {QueryClass.EXACT_FACTUAL: 4, QueryClass.GLOBAL: 1} + + +def test_select_stratified_sample_rejects_a_cap_smaller_than_the_class_count() -> None: + """A cost cap may not silently remove a query class from comparison.""" + judgments = [_judgment(f"{query_class.value}-0", query_class) for query_class in QueryClass] + [ + _judgment("extra", QueryClass.EXACT_FACTUAL) + ] + + with pytest.raises(ValueError, match="represent every query class"): + select_stratified_sample( + judgments, + max_questions=len(QueryClass) - 1, + seed="stable", + ) diff --git a/tests/unit/test_verify_run.py b/tests/unit/test_verify_run.py index e05c6c5..edebcbc 100644 --- a/tests/unit/test_verify_run.py +++ b/tests/unit/test_verify_run.py @@ -1,5 +1,6 @@ """Tests for scripts/verify_run.py: checksum, event-chain, and manifest verification (ADR-0017).""" +import dataclasses import importlib.util import json import sys @@ -10,6 +11,8 @@ from ragforge.evaluation.artifact_writer import write_atomic, write_checksums_file from ragforge.evaluation.event_log import EventLog +from ragforge.evaluation.run_evidence import finalize_evidence_directory +from ragforge.evaluation.run_manifest import build_initial_manifest _SCRIPT_PATH = Path(__file__).resolve().parents[2] / "scripts" / "verify_run.py" @@ -34,25 +37,19 @@ def _build_clean_run(artifacts_dir: Path, *, run_id: str = "run-1") -> Path: write_atomic(run_dir / "questions" / "base.json", json.dumps({"question_id": "q1"})) - manifest = { - "schema_version": 1, - "run_id": run_id, - "status": "completed", - "git_sha": "abc123", - "started_at": "2026-01-01T00:00:00+00:00", - "completed_at": "2026-01-01T00:01:00+00:00", - "corpus_hash": "corpus-hash", - "dataset_hash": "dataset-hash", - "split_hash": "split-hash", - "configuration_hash": "config-hash", - "models": {}, - "strategies": ["base"], - "execution": {}, - "artifact_root_hash": None, - } - write_atomic(run_dir / "manifest.json", json.dumps(manifest)) - - write_checksums_file(run_dir) + manifest = build_initial_manifest( + run_id=run_id, + git_sha="abc123", + corpus_hash="corpus-hash", + dataset_hash="dataset-hash", + split_hash="split-hash", + configuration_hash="config-hash", + models={}, + strategies=("base",), + execution={}, + ) + write_atomic(run_dir / "manifest.json", json.dumps(dataclasses.asdict(manifest))) + finalize_evidence_directory(run_dir, manifest) return run_dir @@ -166,6 +163,19 @@ def test_verify_manifest_detects_a_strategy_with_no_question_artifact(tmp_path: assert any("raptor" in problem for problem in problems) +def test_verify_manifest_detects_an_inconsistent_artifact_root_hash(tmp_path: Path) -> None: + """Recomputing file checksums cannot hide a manifest/root inconsistency.""" + run_dir = _build_clean_run(tmp_path) + manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8")) + manifest["artifact_root_hash"] = "incorrect" + write_atomic(run_dir / "manifest.json", json.dumps(manifest)) + write_checksums_file(run_dir) + + problems = verify_run.verify_manifest(run_dir) + + assert any("artifact_root_hash" in problem for problem in problems) + + def test_main_exits_zero_and_prints_ok_for_a_clean_run( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: From b98f970a7bf6eea576e3612c6d558b302b924bde Mon Sep 17 00:00:00 2001 From: Bruno Vicco Date: Sun, 26 Jul 2026 15:28:35 -0300 Subject: [PATCH 2/2] docs(wiki): document embeddings and retrieval strategies --- wiki/Embeddings-pt-BR.md | 278 ++++++++++++++++++++++++ wiki/Embeddings.md | 266 +++++++++++++++++++++++ wiki/Estrategias-de-Recuperacao.md | 333 +++++++++++++++++++++++++++++ wiki/Home.md | 91 ++++++++ wiki/Retrieval-Strategies.md | 333 +++++++++++++++++++++++++++++ wiki/_Sidebar.md | 13 ++ 6 files changed, 1314 insertions(+) create mode 100644 wiki/Embeddings-pt-BR.md create mode 100644 wiki/Embeddings.md create mode 100644 wiki/Estrategias-de-Recuperacao.md create mode 100644 wiki/Home.md create mode 100644 wiki/Retrieval-Strategies.md create mode 100644 wiki/_Sidebar.md diff --git a/wiki/Embeddings-pt-BR.md b/wiki/Embeddings-pt-BR.md new file mode 100644 index 0000000..9df5c57 --- /dev/null +++ b/wiki/Embeddings-pt-BR.md @@ -0,0 +1,278 @@ +# Embeddings no RAGForge + +[English](Embeddings) · [Início](Home) · [Estratégias de recuperação](Estrategias-de-Recuperacao) + +> Retrato da documentação em 2026-07-25. O estado dos modelos vem da +> configuração do repositório; ele não significa que todos os candidatos já +> concluíram o experimento atual sobre o RegRAG-BR. + +## O que é um embedding + +Um embedding é um vetor numérico que representa o conteúdo semântico de uma +entrada. O RAGForge gera um vetor para o `retrieval_text` de cada chunk e o +armazena no PostgreSQL com pgvector. Durante uma consulta, o mesmo modelo +vetoriza a pergunta e o pgvector ordena os chunks por distância cosseno. + +Esse é um desenho de **bi-encoder**: documentos e perguntas são codificados +separadamente. Ele escala para o corpus porque os vetores dos documentos são +calculados antes das consultas. É diferente do cross-encoder usado no reranking, +que lê cada par pergunta–chunk em conjunto e, por isso, só é aplicado a um +conjunto pequeno de candidatos. + +```text +retrieval_text do documento ──modelo de embedding──> vetor ──┐ + ├─ similaridade cosseno ─> ranking +texto da pergunta ────────────modelo de embedding──> vetor ──┘ +``` + +O projeto usa hoje uma única operação `embed()` para documentos e perguntas. Uma +instrução de consulta específica por modelo e versionada está planejada, mas não +implementada. + +## Dense, sparse e multimodal: significados diferentes + +| Termo | Significado | Comportamento no RAGForge | +|---|---|---| +| Embedding denso | Vetor de ponto flutuante com largura fixa, cujo significado é distribuído entre as dimensões | Implementado com Gemini ou Sentence Transformers | +| Representação esparsa | Pesos de termos ou atributos em que a maioria dos valores é zero | O RAGForge **não** gera embeddings esparsos por modelo; sua estratégia sparse é BM25 | +| Embedding multimodal | Mídias diferentes compartilham o mesmo espaço vetorial | `gemini-embedding-2` oferece a capacidade, mas o RAGForge envia apenas texto | +| Embedding Matryoshka/MRL | Modelo treinado para manter informação útil quando a dimensão de saída é reduzida | Gemini é solicitado com 1.536 dimensões; os modelos locais usam a largura informada pelo modelo | + +BM25 é às vezes agrupado com “embeddings esparsos” em explicações genéricas de +RAG, mas isso seria impreciso para esta implementação. A estratégia +`sparse_bm25` envia o texto da pergunta diretamente ao OpenSearch e não chama +nenhum modelo de embedding. + +## Inventário de modelos + +### Candidatos de embedding para recuperação + +| Modelo | Execução | Largura usada ou declarada | Estado no repositório | Uso no RAGForge | +|---|---:|---:|---|---| +| `gemini-embedding-001` | API Gemini hospedada | 1.536 | Seleção canônica, provisória e aguardando revalidação | Vetores densos de texto; configurado na matriz publicável | +| `gemini-embedding-2` | API Gemini hospedada | 1.536 | Candidato aguardando revalidação | Apenas texto, apesar da capacidade multimodal do modelo | +| `Qwen/Qwen3-Embedding-0.6B` | Sentence Transformers local | 1.024 | Default operacional local; ainda não avaliado no experimento isolado | Vetores densos sem credencial de embedding em `make bench-live-local` | +| `BAAI/bge-m3` | Sentence Transformers local | 1.024 | Candidato aguardando revalidação | Somente saída densa; as saídas sparse e no estilo ColBERT não são usadas | +| `intfloat/multilingual-e5-large-instruct` | Sentence Transformers local | 1.024 | Candidato de controle ainda não avaliado | Saída densa sem a instrução específica de consulta | + +### `gemini-embedding-001` + +- O benchmark canônico solicita 1.536 dimensões. O modelo oferece dimensões + flexíveis até 3.072. +- A redução mantém o vetor abaixo do limite de 2.000 dimensões do índice HNSW + do pgvector para o tipo `vector`. +- As chamadas são hospedadas, tarifadas e exigem `GEMINI_API_KEY` ou + `GOOGLE_API_KEY`. +- O adaptador processa lotes de até 100 textos, repete falhas transitórias, + limita chamadas concorrentes e pode manter cache por texto. +- O adaptador registra `normalize=false`; a comparação no pgvector continua + usando distância cosseno. + +O experimento isolado de PT-BR selecionou este modelo para a matriz canônica, +mas o arquivo atual do experimento marca a seleção como provisória e pendente de +revalidação. Ela não deve ser apresentada como um vencedor permanente. + +### `gemini-embedding-2` + +- O provedor o descreve como um modelo unificado para texto, imagem, vídeo, + áudio e PDF. A porta de embedding do RAGForge aceita apenas texto; as outras + modalidades não são exercitadas. +- O projeto solicita 1.536 dimensões, assim como no `gemini-embedding-001`. +- Uma sondagem direta do projeto observou que uma requisição `embed_content` + com vários textos retornava somente um embedding. Por isso, o adaptador envia + um texto por requisição para este modelo. A indexação fica mais lenta e + produz muito mais chamadas. +- Ele é candidato de comparação, não o modelo canônico. + +### `Qwen/Qwen3-Embedding-0.6B` + +- É a alternativa operacional sem provedor de embedding em + `benchmark-local-v01.yaml`. +- Executa por `SentenceTransformerEmbedder`, informa 1.024 dimensões e devolve + vetores normalizados por L2. +- O model card o descreve como multilíngue, sensível a instruções e compatível + com redução de dimensão Matryoshka. O RAGForge usa hoje a largura completa e + não envia instrução de consulta. +- A configuração não fixa uma revisão imutável do Hugging Face. O adaptador + registra a revisão não resolvida como `main`, suficiente para exploração, + mas abaixo da meta de reprodutibilidade da ADR-0013. +- “Local” vale somente para a etapa de embedding. O benchmark completo ainda + chama Gemini para contextualização, resumos, extração do GraphRAG e geração + de respostas, e OpenAI para o judge canônico. + +### `BAAI/bge-m3` + +- O modelo pode produzir representações densas, sparse aprendidas e multi-vector + no estilo ColBERT. O adaptador do RAGForge chama o `encode()` padrão do + Sentence Transformers; portanto, somente a representação densa de 1.024 + dimensões participa da comparação. +- Os vetores são normalizados por L2 antes da indexação. +- O repositório registra CPU como caminho estável na máquina de desenvolvimento + atual. Uma execução em MPS esgotou a memória; essa é uma observação daquela + máquina, não uma restrição geral do modelo. +- O resultado da comparação em PT-BR precisa ser revalidado. + +### `intfloat/multilingual-e5-large-instruct` + +- É o candidato local de controle. +- Seu model card exige uma instrução de tarefa na pergunta e alerta para perda + de desempenho sem ela. +- O RAGForge vetoriza hoje perguntas e documentos pelo mesmo método `embed()` e + registra o hash de uma instrução vazia. Um resultado obtido agora mediria + tanto o modelo quanto a integração incompleta do projeto. +- Está declarado na matriz experimental, mas ainda não foi avaliado nela. + +### `text-embedding-3-small`: somente avaliação + +O `text-embedding-3-small` está configurado no judge OpenAI canônico para a +métrica Answer Relevancy do RAGAS. Ele **não** cria o índice de recuperação, +não recupera chunks e não participa da comparação de embeddings Dense/Hybrid. + +Essa distinção é importante: + +```text +embedding de recuperação -> encontra evidências +embedding do judge -> ajuda a medir se a resposta atende à pergunta +``` + +O fallback de judge Gemini usa de forma semelhante o +`gemini-embedding-001` para Answer Relevancy, mas essa alternativa é rotulada +como exploratória porque o gerador de respostas também usa Gemini. + +## Normalização, dimensões e similaridade + +O adaptador local chama: + +```python +model.encode(..., normalize_embeddings=True) +``` + +Para vetores de norma unitária, produto escalar e similaridade cosseno geram a +mesma ordenação. O adaptador Gemini não normaliza no cliente. A camada de +armazenamento usa sempre o operador de cosseno do pgvector +(`vector_cosine_ops` e `<=>`), que considera as magnitudes ao comparar os +vetores. + +As dimensões fazem parte do contrato do espaço vetorial: + +- a largura da coluna pgvector é fixa quando a tabela é criada; +- duas larguras iguais não tornam modelos diferentes compatíveis; +- mudar modelo, revisão, dimensões, normalização ou instrução exige outro + índice; +- a largura nativa de 3.072 do Gemini é reduzida para 1.536 porque índices HNSW + do pgvector sobre o tipo `vector` aceitam até 2.000 dimensões. + +## Identidade, cache e isolamento de índices + +O RAGForge identifica um espaço de embedding por: + +```text +provedor ++ modelo ++ revisão ++ dimensões ++ normalização ++ hash da instrução de consulta ++ runtime +``` + +O namespace do índice também inclui o hash do corpus, a configuração de +chunking e o schema do texto de recuperação. Um fingerprint separado inclui +todo `source_text`, `retrieval_text`, IDs estruturais, metadados e a identidade +do produtor de texto sintético. + +Consequências: + +- vetores não são reutilizados só porque dois modelos têm a mesma largura; +- o cache de embeddings usa a identidade completa e o hash do texto; +- índices base, contextual, SAC e RAPTOR ficam isolados; +- índices parciais não recebem marcador de conclusão reutilizável; +- falha em um provedor não troca silenciosamente o embedding durante a + execução. + +Limitação atual: as revisões dos modelos locais não são fixadas no YAML, e +todas as instruções de consulta compartilham hoje o hash de uma string vazia. +Essas lacunas precisam ser fechadas antes de alegar reprodutibilidade exata +entre máquinas para modelos sensíveis a instruções. + +## Configuração + +Embedding hospedado canônico: + +```yaml +embedding: + provider: gemini + model: gemini-embedding-001 + dimensions: 1536 +``` + +Alternativa local: + +```yaml +embedding: + provider: local + model: Qwen/Qwen3-Embedding-0.6B + dimensions: 1024 + device: cpu +``` + +O runner aceita somente os provedores de recuperação `local` e `gemini`. Não +há fallback automático entre eles. + +## Como comparar e escolher um modelo + +O protocolo do RAGForge mantém constantes corpus, split, chunking, texto de +recuperação, top-k, julgamentos e métricas. Ele varia o embedding e mede Dense e +Hybrid+RRF, as estratégias afetadas mais diretamente pela qualidade vetorial. + +Execução de um candidato: + +```bash +uv run python configs/experiments/run_embeddings_ptbr.py \ + --model Qwen/Qwen3-Embedding-0.6B +``` + +Exemplo hospedado: + +```bash +GEMINI_API_KEY=... uv run python \ + configs/experiments/run_embeddings_ptbr.py \ + --model gemini-embedding-001 +``` + +A comparação produz Recall@k, Precision@k, nDCG@k e MRR. Antes de promover um +novo vencedor, também devem ser registrados revisão imutável, instrução efetiva, +dispositivo, precisão numérica, throughput, latência, memória, tamanho do índice +e custo do provedor. + +## Privacidade e impacto operacional + +- Embedding local mantém os textos de chunks e perguntas na máquina durante + essa etapa. +- Embedding Gemini hospedado envia texto de recuperação e perguntas a um + provedor externo. +- O corpus atual contém atos oficiais públicos; isso não representa autorização + automática para enviar futuros documentos privados. +- Uma execução com dados privados exige decisão explícita de processamento. O + inventário geral de privacidade do repositório ainda não está completo. +- Chaves de API e conteúdo não devem ser registrados em logs. As credenciais + são lidas de variáveis de ambiente. + +## Fontes + +### RAGForge + +- [Porta e adaptadores de embedding](https://github.com/brunovicco/ragforge/tree/main/src/ragforge/embeddings) +- [ADR-0005: escopo da comparação](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0005-embedding-comparison-scope.md) +- [ADR-0013: backends neutros de provedor](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0013-provider-neutral-embedding-backends.md) +- [Configuração do experimento PT-BR](https://github.com/brunovicco/ragforge/blob/main/configs/experiments/embeddings-ptbr.yaml) + +### Documentação primária dos modelos e armazenamento + +- [Google: `gemini-embedding-001`](https://ai.google.dev/gemini-api/docs/models/gemini-embedding-001) +- [Google: `gemini-embedding-2`](https://ai.google.dev/gemini-api/docs/models/gemini-embedding-2?hl=pt-br) +- [Model card do Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B) +- [Model card do BGE-M3](https://huggingface.co/BAAI/bge-m3) +- [Model card do Multilingual E5 Large Instruct](https://huggingface.co/intfloat/multilingual-e5-large-instruct) +- [OpenAI: `text-embedding-3-small`](https://developers.openai.com/api/docs/models/text-embedding-3-small) +- [pgvector: tipos, dimensões e HNSW](https://github.com/pgvector/pgvector) diff --git a/wiki/Embeddings.md b/wiki/Embeddings.md new file mode 100644 index 0000000..0accdfc --- /dev/null +++ b/wiki/Embeddings.md @@ -0,0 +1,266 @@ +# Embeddings in RAGForge + +[Português](Embeddings-pt-BR) · [Home](Home) · [Retrieval strategies](Retrieval-Strategies) + +> Documentation snapshot: 2026-07-25. Model status comes from the repository +> configuration, not from a claim that every candidate has completed the current +> RegRAG-BR experiment. + +## What an embedding is + +An embedding is a numerical vector that represents the semantic content of an input. +RAGForge embeds each chunk's `retrieval_text` and stores the vector in PostgreSQL with +pgvector. At query time, the same model embeds the question, and pgvector orders chunks +by cosine distance. + +This is a **bi-encoder** design: documents and queries are encoded separately. It scales +to a full corpus because document vectors are computed before query time. It is different +from the project's cross-encoder reranker, which reads each query–chunk pair jointly and +is therefore applied only to a small candidate pool. + +```text +document retrieval_text ──embedding model──> vector ──┐ + ├─ cosine similarity ─> ranking +question text ────────────embedding model──> vector ──┘ +``` + +The project currently uses one `embed()` operation for both documents and queries. A +versioned, model-specific query instruction is planned but not implemented. + +## Dense, sparse, and multimodal: different meanings + +| Term | Meaning | RAGForge behavior | +|---|---|---| +| Dense embedding | A fixed-width floating-point vector in which meaning is distributed across dimensions | Implemented through Gemini or Sentence Transformers | +| Sparse representation | Mostly-zero term or feature weights | RAGForge does **not** generate sparse model embeddings; its sparse strategy is BM25 | +| Multimodal embedding | Different media share one vector space | `gemini-embedding-2` supports it, but RAGForge currently sends text only | +| Matryoshka/MRL embedding | A model trained so useful shorter vectors can be obtained by reducing output dimensions | Gemini is requested at 1,536 dimensions; local models currently use their reported width | + +BM25 is often grouped with “sparse embeddings” in broad RAG discussions, but that would +be inaccurate for this implementation. The `sparse_bm25` strategy sends query text +directly to OpenSearch and performs no embedding call. + +## Model inventory + +### Retrieval embedding candidates + +| Model | Runtime | Width used or declared | Repository status | How RAGForge uses it | +|---|---:|---:|---|---| +| `gemini-embedding-001` | Hosted Gemini API | 1,536 | Canonical selection, provisional pending revalidation | Text-only dense vectors; configured for the publishable matrix | +| `gemini-embedding-2` | Hosted Gemini API | 1,536 | Candidate pending revalidation | Text input only, despite the model's multimodal capability | +| `Qwen/Qwen3-Embedding-0.6B` | Local Sentence Transformers | 1,024 | Operational local default; isolated experiment not yet evaluated | Credential-free dense vectors for `make bench-live-local` | +| `BAAI/bge-m3` | Local Sentence Transformers | 1,024 | Candidate pending revalidation | Dense output only; BGE-M3's sparse and ColBERT-style outputs are not used | +| `intfloat/multilingual-e5-large-instruct` | Local Sentence Transformers | 1,024 | Control candidate, not yet evaluated | Dense output without the model-specific query instruction | + +### `gemini-embedding-001` + +- The canonical benchmark requests 1,536 output dimensions. The model supports flexible + output dimensions up to 3,072. +- The reduction keeps the vector below pgvector HNSW's 2,000-dimension limit for the + `vector` type. +- Calls are hosted, metered, and require `GEMINI_API_KEY` or `GOOGLE_API_KEY`. +- The adapter batches up to 100 texts, retries transient failures, limits concurrent + provider calls, and can cache vectors per text. +- The adapter records `normalize=false`; pgvector still compares vectors with cosine + distance. + +The repository's isolated PT-BR comparison selected this model for the canonical matrix, +but the current experiment file marks that selection as provisional pending revalidation. +It should not be presented as a permanent winner. + +### `gemini-embedding-2` + +- The provider describes it as a unified text, image, video, audio, and PDF embedding + model. RAGForge's embedding port is text-only, so none of the non-text modalities are + exercised. +- The project requests 1,536 dimensions, just as it does for + `gemini-embedding-001`. +- A direct project probe found that a multi-text `embed_content` request returned one + embedding instead of one per input. The adapter therefore sends one text per request + for this model. This is slower and creates many more requests during indexing. +- It is a comparison candidate, not the canonical model. + +### `Qwen/Qwen3-Embedding-0.6B` + +- This is the provider-free operational alternative in + `benchmark-local-v01.yaml`. +- It runs through `SentenceTransformerEmbedder`, reports 1,024 dimensions, and returns + L2-normalized vectors. +- The model card describes it as multilingual, instruction-aware, and capable of + Matryoshka dimension reduction. RAGForge currently uses the full reported width and + does not send a query instruction. +- The configuration does not pin an immutable Hugging Face revision. The adapter records + the unresolved revision as `main`, which is adequate for exploration but weaker than + the reproducibility target in ADR-0013. +- “Local” only applies to the embedding stage. The full benchmark still calls Gemini for + contextualization, summarization, GraphRAG extraction, and answer generation, and + OpenAI for the canonical judge. + +### `BAAI/bge-m3` + +- The model can produce dense, learned sparse, and ColBERT-style multi-vector + representations. The RAGForge adapter calls Sentence Transformers' standard + `encode()`, so only the 1,024-dimensional dense representation participates in the + comparison. +- Vectors are L2-normalized before indexing. +- The repository records CPU as the stable execution path on the current development + machine. An MPS run exhausted memory; that is a machine-specific observation, not a + general model restriction. +- Its PT-BR comparison result must be revalidated. + +### `intfloat/multilingual-e5-large-instruct` + +- This is the local control candidate. +- Its model card requires a task instruction on the query and warns of degraded + performance without one. +- RAGForge currently embeds query and document text through the same `embed()` method and + records an empty query-instruction hash. A score produced today would therefore measure + the project's incomplete integration as well as the model. +- It is declared in the experiment matrix but has not yet been evaluated there. + +### `text-embedding-3-small`: evaluation only + +`text-embedding-3-small` is configured for RAGAS Answer Relevancy in the canonical +OpenAI judge. It does **not** create the retrieval index, retrieve chunks, or participate +in the Dense/Hybrid embedding comparison. + +Keeping this distinction matters: + +```text +retrieval embedding -> finds evidence +judge embedding -> helps score whether the generated answer addresses the question +``` + +The Gemini judge fallback similarly uses `gemini-embedding-001` for Answer Relevancy, +but that fallback is labeled exploratory because the answer generator is also Gemini. + +## Normalization, dimensions, and similarity + +The local adapter calls: + +```python +model.encode(..., normalize_embeddings=True) +``` + +For unit-normalized vectors, dot product and cosine similarity produce the same ordering. +The Gemini adapter does not normalize client-side. The storage layer consistently uses +pgvector's cosine operator (`vector_cosine_ops` and `<=>`), which accounts for vector +magnitudes when comparing them. + +Dimensions are part of the vector-space contract: + +- the pgvector column width is fixed when the table is created; +- equal widths do not make two models compatible; +- changing model, revision, dimensions, normalization, or instruction requires a + different index; +- Gemini's 3,072 native width is reduced to 1,536 because pgvector HNSW indexes over the + `vector` type support up to 2,000 dimensions. + +## Identity, cache, and index isolation + +RAGForge identifies an embedding space with: + +```text +provider ++ model ++ revision ++ dimensions ++ normalization ++ query-instruction hash ++ runtime +``` + +The index namespace also includes the corpus hash, chunking configuration, and retrieval +text schema. A separate index fingerprint hashes every chunk's source text, retrieval +text, structural IDs, metadata, and the synthetic-text producer identity. + +Consequences: + +- vectors are never reused merely because two models have the same width; +- cached embeddings are keyed by complete identity and the input-text hash; +- contextual, SAC, RAPTOR, and base indexes are isolated; +- partial indexes do not receive a reusable completion marker; +- a provider failure does not silently switch the run to another embedding model. + +Current limitation: local model revisions are not pinned by the YAML configuration, and +all query instructions currently share the hash of an empty string. Those gaps must be +closed before claiming exact cross-machine reproducibility for instruction-aware models. + +## Configuration + +Canonical hosted embedding: + +```yaml +embedding: + provider: gemini + model: gemini-embedding-001 + dimensions: 1536 +``` + +Local embedding alternative: + +```yaml +embedding: + provider: local + model: Qwen/Qwen3-Embedding-0.6B + dimensions: 1024 + device: cpu +``` + +The runner supports only `local` and `gemini` retrieval providers. There is no automatic +fallback between them. + +## Choosing and comparing a model + +RAGForge's comparison protocol holds corpus, split, chunking, retrieval text, top-k, +judgments, and metrics constant. It varies the embedding and measures Dense and +Hybrid+RRF, the strategies most directly affected by vector quality. + +Run one candidate with: + +```bash +uv run python configs/experiments/run_embeddings_ptbr.py \ + --model Qwen/Qwen3-Embedding-0.6B +``` + +Hosted example: + +```bash +GEMINI_API_KEY=... uv run python \ + configs/experiments/run_embeddings_ptbr.py \ + --model gemini-embedding-001 +``` + +The comparison reports Recall@k, Precision@k, nDCG@k, and MRR. Before promoting a new +winner, also record immutable model revisions, the effective query instruction, device, +precision, throughput, latency, memory, index size, and provider cost. + +## Privacy and operational impact + +- Local embedding keeps chunk and query text on the machine for the embedding stage. +- Hosted Gemini embedding sends retrieval text and questions to an external provider. +- The current corpus contains public official acts, but this does not imply authorization + to send future private documents externally. +- A private-data run needs an explicit data-processing decision; the repository's general + privacy inventory is not yet complete. +- Neither API keys nor content should be logged. Credentials are read from environment + variables. + +## Sources + +### RAGForge + +- [Embedding port and adapters](https://github.com/brunovicco/ragforge/tree/main/src/ragforge/embeddings) +- [ADR-0005: embedding comparison scope](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0005-embedding-comparison-scope.md) +- [ADR-0013: provider-neutral backends](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0013-provider-neutral-embedding-backends.md) +- [PT-BR experiment configuration](https://github.com/brunovicco/ragforge/blob/main/configs/experiments/embeddings-ptbr.yaml) + +### Primary model and storage documentation + +- [Google: `gemini-embedding-001`](https://ai.google.dev/gemini-api/docs/models/gemini-embedding-001) +- [Google: `gemini-embedding-2`](https://ai.google.dev/gemini-api/docs/models/gemini-embedding-2) +- [Qwen3-Embedding-0.6B model card](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B) +- [BGE-M3 model card](https://huggingface.co/BAAI/bge-m3) +- [Multilingual E5 Large Instruct model card](https://huggingface.co/intfloat/multilingual-e5-large-instruct) +- [OpenAI: `text-embedding-3-small`](https://developers.openai.com/api/docs/models/text-embedding-3-small) +- [pgvector: vector types, dimensions, and HNSW](https://github.com/pgvector/pgvector) diff --git a/wiki/Estrategias-de-Recuperacao.md b/wiki/Estrategias-de-Recuperacao.md new file mode 100644 index 0000000..bc1129c --- /dev/null +++ b/wiki/Estrategias-de-Recuperacao.md @@ -0,0 +1,333 @@ +# Estratégias de recuperação e modelos auxiliares + +[English](Retrieval-Strategies) · [Início](Home) · [Embeddings](Embeddings-pt-BR) + +> Retrato da documentação em 2026-07-25. A configuração principal contém 10 +> rótulos de estratégia. O README agrupa-os em oito famílias mais amplas; SAC e +> SAC+Contextual são variantes experimentais explícitas no runner. + +## Visão de ponta a ponta + +```mermaid +flowchart LR + A["Documento oficial"] --> B["Extração"] + B --> C["Chunking estrutural jurídico"] + C --> D["source_text"] + D --> E{"Variação do texto de recuperação"} + E -->|baseline| F["source_text"] + E -->|contextual| G["contexto do chunk + fonte"] + E -->|SAC| H["resumo do documento + fonte"] + E -->|SAC + contextual| I["resumo + contexto + fonte"] + F --> J["Índices Dense / BM25"] + G --> J + H --> J + I --> J + J --> K["Recuperar evidências top-k"] + K --> L["Gerar resposta citada a partir de source_text"] + L --> M["Judge independente e auditoria opcional"] +``` + +O chunker jurídico divide o texto pela hierarquia normativa — artigo, +parágrafo, inciso e alínea — e atribui um ID estrutural estável a cada unidade. +Nas variantes base, Contextual e SAC: + +- `source_text` é extraído do documento oficial; +- `retrieval_text` pode conter contexto ou resumo gerado; +- a indexação usa `retrieval_text`; +- geração de resposta e citações usam `source_text`. + +RAPTOR é a exceção que precisa permanecer explícita: ele cria nós sintéticos de +resumo cujo `source_text` é texto gerado. Esses nós podem entrar no contexto de +geração e carregam os IDs estruturais agregados dos filhos. Eles são uma abstração +para recuperação, não redação jurídica autoritativa. + +## Matriz de estratégias + +| Rótulo na configuração | Fonte dos candidatos | Ordenação | Embedding? | Trabalho adicional de modelo | +|---|---|---|---|---| +| `dense` | Chunks base | Similaridade cosseno no pgvector | Sim | Nenhum | +| `sparse_bm25` | Chunks base | BM25 no OpenSearch | Não | Nenhum | +| `hybrid_rrf` | Dense + BM25 | Reciprocal Rank Fusion | Sim, no ramo dense | Nenhum | +| `reranked` | Top 50 do Hybrid | Score do cross-encoder, depois top 5 | Sim, na primeira etapa | Cross-encoder local | +| `contextual` | Contexto por chunk + fonte | Hybrid RRF | Sim | Uma chamada Gemini por chunk | +| `parent_child` | Hits dense dos filhos | Score dense, depois expansão ao pai | Sim | Nenhum | +| `sac` | Resumo do documento + fonte | Similaridade cosseno Dense | Sim | Um resumo Gemini por documento | +| `sac_contextual` | Resumo + contexto do chunk + fonte | Similaridade cosseno Dense | Sim | Enriquecimento por documento e por chunk | +| `raptor` | Folhas + resumos recursivos | Similaridade cosseno Dense | Sim | Resumos Gemini para cada grupo da árvore | +| `graphrag` | Grafo e chunks do LightRAG | Ordem do LightRAG, mapeada para `1/rank` | Sim | Extração Gemini de entidades e relações | + +O `top_k` padrão é 5, e o conjunto de reranking tem 50 candidatos. + +## 1. Dense + +Dense é o baseline semântico: + +1. vetorizar o `retrieval_text` de cada chunk; +2. armazenar os vetores em uma coluna pgvector; +3. criar índice HNSW com `vector_cosine_ops`; +4. vetorizar a pergunta com o mesmo modelo; +5. devolver os `top_k` vetores mais próximos. + +Pontos fortes: + +- reconhece paráfrases e redações semanticamente relacionadas; +- não exige que os termos exatos da pergunta apareçam no chunk; +- serve de camada de recuperação para SAC e RAPTOR. + +Trade-offs: + +- a qualidade depende do modelo e da integração do embedding; +- identificadores exatos, números de artigo e termos raros podem ser perdidos; +- mudar o espaço vetorial exige reconstruir o índice. + +## 2. Sparse BM25 + +`sparse_bm25` faz busca lexical no OpenSearch com o analisador `brazilian`. +BM25 ordena textos usando frequência do termo, frequência inversa no corpus e +normalização pelo tamanho do documento. + +Pontos fortes: + +- adequado para termos exatos, identificadores jurídicos, artigos e siglas; +- não tem custo de inferência de modelo; +- independe do embedding denso selecionado. + +Trade-offs: + +- paráfrases com pouca sobreposição lexical são mais difíceis; +- tokenização e análise linguística influenciam o resultado; +- é busca lexical clássica, não embedding sparse aprendido. + +O índice sparse pesquisa `retrieval_text`, mas devolve o `source_text` +autoritativo associado. + +## 3. Hybrid + RRF + +Hybrid executa Dense e BM25 separadamente e combina suas posições com +Reciprocal Rank Fusion: + +```text +RRF(chunk) = Σ 1 / (60 + rank) +``` + +Um chunk presente nos dois rankings acumula as duas contribuições. O RRF compara +posições em vez dos scores brutos, evitando comparar diretamente escalas +diferentes de BM25 e cosseno. + +No RAGForge, cada ramo recebe a profundidade solicitada pelo chamador. No +`hybrid_rrf` comum, ela é top-k; dentro de `reranked`, ela é o conjunto maior de +reranking. + +## 4. Reranked + +A estratégia usa duas etapas: + +```text +Top 50 do Hybrid + -> cross-encoder(pergunta, chunk) para cada candidato + -> ordenar pelo score do cross-encoder + -> top 5 +``` + +O modelo é `cross-encoder/ms-marco-MiniLM-L-6-v2`. Diferente de um +bi-encoder, o cross-encoder lê a pergunta e o chunk em conjunto, permitindo +interações mais finas entre seus tokens. Como é mais caro por candidato, ele +não é executado sobre todo o corpus. + +Limitação importante: o modelo foi treinado para ranking de passagens MS MARCO, +e seu model card é orientado a inglês. Ele não foi selecionado pela comparação +dedicada de modelos em PT-BR. Seu desempenho em português jurídico brasileiro +precisa, portanto, de validação empírica. + +## 5. Contextual Retrieval + +Para cada chunk, `gemini-3.1-flash-lite` gera uma explicação curta que o situa +no documento-fonte completo: + +```text +retrieval_text = contexto específico do chunk + source_text +``` + +O RAGForge indexa o texto enriquecido nos índices Dense e BM25 e recupera com +Hybrid+RRF. Assim, implementa contextual embeddings e contextual BM25. + +Ponto forte: + +- recupera informação perdida quando uma disposição jurídica não repete o + assunto, a autoridade ou o escopo da norma. + +Custos e riscos: + +- uma chamada real de LLM por chunk durante a preparação do índice; +- o contexto gerado pode estar errado ou destacar demais uma interpretação; +- o contextualizador atual não usa o cache persistente de LLM empregado em + algumas outras etapas; uma retomada pode repetir o trabalho; +- somente `source_text`, nunca o texto sintético, segue para a geração. + +## 6. Parent-child + +Parent-child, ou small-to-big, pesquisa chunks jurídicos detalhados e devolve um +pai autoritativo maior: + +```text +pesquisar parágrafo/inciso -> devolver o artigo pai +``` + +A relação vem da hierarquia jurídica real produzida pelo chunker, não de uma +janela arbitrária de caracteres. Pais duplicados são removidos. Quando vários +filhos bem posicionados têm o mesmo pai, a estratégia pode devolver menos de +`top_k`, pois não repõe resultados após a deduplicação. + +A implementação atual usa Dense — não Hybrid — como retriever interno. + +## 7. Summary-Augmented Chunking (SAC) + +SAC gera um resumo para cada versão imutável de documento com +`gemini-3.1-flash-lite` e prefixa o mesmo resumo em todos os chunks daquele +documento: + +```text +retrieval_text = resumo do documento + source_text +``` + +O objetivo é reduzir Document-Level Retrieval Mismatch: recuperar uma cláusula +localmente plausível da norma errada. O rótulo `sac` usa Dense para isolar o +efeito do resumo do documento. + +Trade-offs: + +- uma chamada de geração por documento, em vez de uma por chunk; +- pode melhorar a discriminação entre documentos; +- um erro no resumo se repete em todos os chunks do documento; +- o prefixo comum pode reduzir a discriminação entre disposições da mesma norma; +- SAC continua experimental até demonstrar ganho em mais de uma família de + embedding sem regressão material no recall estrutural. + +## 8. SAC + Contextual + +A composição preserva os dois níveis de contexto: + +```text +retrieval_text = + resumo do documento + + contexto específico do chunk + + source_text +``` + +Ela reutiliza os chunks já contextualizados e aplica o resumo do documento por +cima. Em seguida, usa Dense. Rótulo e fingerprint próprios impedem que o +resultado seja apresentado como Dense comum ou SAC isolado. + +## 9. RAPTOR + +RAPTOR adiciona nós recursivamente resumidos acima dos chunks originais e +pesquisa todos os níveis juntos, no modo “collapsed tree”. + +A implementação do RAGForge é deliberadamente mínima: + +- agrupa nós na ordem do documento, cinco por vez; +- resume cada grupo com `gemini-3.1-flash-lite`; +- repete até cinco níveis ou até chegar a uma raiz; +- constrói uma árvore separada por documento; +- reúne folhas e resumos em um único índice Dense. + +Esse **não** é o algoritmo completo do artigo RAPTOR. Não há redução UMAP nem +clustering semântico por mistura gaussiana. Artigos adjacentes, mas de assuntos +diferentes, podem ser resumidos juntos, e os nós de resumo gerados são +devolvidos como conteúdo recuperado. Como esses nós armazenam o resumo gerado em +`source_text`, eles também podem chegar ao gerador de resposta; os IDs estruturais +dos filhos não tornam a redação gerada autoritativa. A simplificação e esse risco +para a qualidade da evidência devem permanecer explícitos nas comparações. + +## 10. GraphRAG + +O RAGForge integra o LightRAG: + +1. preserva os limites dos chunks jurídicos pelo ponto de extensão de chunking + do LightRAG; +2. usa o embedding de recuperação configurado para os vetores do LightRAG; +3. usa `gemini-3.1-flash-lite` para extrair entidades e relações; +4. consulta no modo `local` por padrão; +5. relaciona o conteúdo devolvido aos chunks do RAGForge por texto exato. + +Limitações atuais: + +- o mapeamento por texto exato pode descartar resultados reformatados ou sem + correspondência; +- o LightRAG não expõe score nativo por chunk nesse caminho; o RAGForge registra + `1/rank`; +- caminhos do grafo, confiança das entidades e proveniência das relações não + entram na métrica de recuperação; +- a indexação faz várias chamadas de LLM por chunk e tem custo bem maior; +- os modos `global`, `hybrid`, `mix` e `naive` são aceitos pelo adaptador, mas + o benchmark principal fixa `local`. + +Temporal GraphRAG é uma estratégia futura e separada. Ela não está na matriz +atual porque o corpus ainda não possui evidência temporal com versões +qualificadas. + +## Modelos auxiliares: o que é e o que não é embedding + +| Etapa | Modelo | Papel | Embedding de recuperação? | +|---|---|---|---| +| Indexação/consulta Dense | Modelo Gemini ou local configurado | Produzir vetores de recuperação | Sim | +| Reranking | `cross-encoder/ms-marco-MiniLM-L-6-v2` | Avaliar pares pergunta–chunk em conjunto | Não | +| Contextual Retrieval | `gemini-3.1-flash-lite` | Gerar contexto específico por chunk | Não | +| SAC | `gemini-3.1-flash-lite` | Gerar um resumo por versão de documento | Não | +| RAPTOR | `gemini-3.1-flash-lite` | Gerar nós de resumo recursivos | Não | +| Extração do GraphRAG | `gemini-3.1-flash-lite` | Extrair entidades e relações | Não | +| Geração de resposta | `gemini-3.1-flash-lite` configurado | Produzir resposta com citações | Não | +| Judge canônico | `gpt-5.4-mini-2026-03-17` | Medir fidelidade, relevância e abstenção | Não | +| Relevância do judge | `text-embedding-3-small` | Apoiar Answer Relevancy do RAGAS | Sim, mas somente para avaliação | +| Auditoria semântica opcional | `gpt-5.4-mini-2026-03-17` | Verificar suporte e reescrever no máximo uma vez | Não | + +O judge canônico é independente do gerador Gemini, mas seus scores continuam +sem validação até que o exercício planejado de calibração humana atinja a +concordância exigida. + +## Quais provedores são contatados? + +| Comando | Embedding de recuperação | Outros provedores live | +|---|---|---| +| `make bench-live` | Gemini | Geração/enriquecimento Gemini + judge OpenAI | +| `make bench-live-local` | Qwen local | Geração/enriquecimento Gemini + judge OpenAI | +| `make bench` | Replay por cache pretendido | Não implementado; o comando falha de forma explícita | + +Portanto, `make bench-live-local` não é um benchmark offline. Ele remove o +provedor externo apenas da etapa de embedding de recuperação. + +## Avaliação + +As estratégias compartilham julgamentos por unidade estrutural e produzem: + +- Recall@k; +- Precision@k; +- nDCG@k; +- MRR; +- métricas de mismatch no nível de documento para variantes aplicáveis; +- cobertura e falhas. + +A geração de resposta é avaliada separadamente por Citation Accuracy, +Faithfulness, Answer Relevancy e comportamento de abstenção. Uma estratégia +pode recuperar bem e ainda gerar resposta ruim; as duas camadas não devem ser +reduzidas a um score único sem definição. + +## Fontes + +### RAGForge + +- [Runner principal do benchmark](https://github.com/brunovicco/ragforge/blob/main/src/ragforge/evaluation/run.py) +- [Implementações de recuperação](https://github.com/brunovicco/ragforge/tree/main/src/ragforge/retrieval) +- [ADR-0006: chunking estrutural jurídico](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0006-legal-structural-chunker.md) +- [ADR-0010: escopo do GraphRAG](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0010-graphrag-evaluation-scope.md) +- [ADR-0015: SAC](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0015-summary-augmented-chunking.md) + +### Referências primárias + +- [OpenSearch: busca BM25](https://docs.opensearch.org/latest/search-plugins/keyword-search/) +- [OpenSearch: Reciprocal Rank Fusion](https://docs.opensearch.org/latest/search-plugins/search-pipelines/score-ranker-processor/) +- [Anthropic: Contextual Retrieval](https://www.anthropic.com/engineering/contextual-retrieval) +- [Artigo RAPTOR](https://arxiv.org/abs/2401.18059) +- [Artigo SAC](https://aclanthology.org/2025.nllp-1.3/) +- [Repositório LightRAG](https://github.com/HKUDS/LightRAG) +- [Model card do cross-encoder](https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2) diff --git a/wiki/Home.md b/wiki/Home.md new file mode 100644 index 0000000..a67b35d --- /dev/null +++ b/wiki/Home.md @@ -0,0 +1,91 @@ +# RAGForge Wiki + +> Documentation snapshot: 2026-07-25 · RAGForge v0.1 in development + +[Português](#português) · [English](#english) + +## Português + +O RAGForge é uma plataforma experimental para comparar estratégias de +Retrieval-Augmented Generation (RAG) sobre documentos financeiros e regulatórios +brasileiros. Esta wiki explica o que o projeto **implementa hoje**, separando três +conceitos que costumam ser confundidos: + +- **modelo de embedding**: transforma texto em um vetor denso; +- **método de recuperação**: decide como os candidatos são encontrados e ordenados; +- **enriquecimento de texto do Contextual/SAC**: altera apenas o texto usado para + indexação, sem transformar esse conteúdo sintético em evidência jurídica. + +### Comece por aqui + +- [Embeddings (PT-BR)](Embeddings-pt-BR): conceitos, modelos candidatos, configuração, + identidade, cache, privacidade e limitações atuais. +- [Estratégias de recuperação (PT-BR)](Estrategias-de-Recuperacao): Dense, BM25, + Hybrid+RRF, Reranked, Contextual Retrieval, Parent-child, SAC, RAPTOR e GraphRAG, + além dos modelos auxiliares. +- [Embeddings (English)](Embeddings) +- [Retrieval strategies (English)](Retrieval-Strategies) + +### Estado resumido + +| Item | Estado atual | +|---|---| +| Embedding da matriz canônica | `gemini-embedding-001`, 1.536 dimensões | +| Alternativa local sem credencial de embedding | `Qwen/Qwen3-Embedding-0.6B`, 1.024 dimensões | +| Seleção do modelo canônico | Provisória, aguardando revalidação do experimento PT-BR | +| Busca lexical | BM25 no OpenSearch com analisador `brazilian` | +| Busca vetorial | Distância cosseno em índice HNSW do pgvector | +| Estratégias configuradas | 10 rótulos, de `dense` a `graphrag` | +| Geração e enriquecimentos | `gemini-3.1-flash-lite` | +| Judge canônico | OpenAI `gpt-5.4-mini-2026-03-17`; ainda não calibrado com humanos | + +> “Canônico” significa “configuração escolhida para a matriz publicável”. Não significa que +> a comparação esteja definitivamente encerrada. O arquivo +> `configs/experiments/embeddings-ptbr.yaml` marca a seleção como +> `provisional_pending_revalidation`. + +## English + +RAGForge is an experimental platform for comparing Retrieval-Augmented Generation +(RAG) strategies over Brazilian financial and regulatory documents. This wiki +documents what the project **implements today**, keeping three often-confused +concepts separate: + +- **embedding model**: turns text into a dense vector; +- **retrieval method**: determines how candidates are found and ranked; +- **Contextual/SAC retrieval-text enrichment**: changes only the text used for + indexing, without turning that synthetic content into legal evidence. + +### Start here + +- [Embeddings](Embeddings): concepts, candidate models, configuration, identity, + caching, privacy, and current limitations. +- [Retrieval strategies](Retrieval-Strategies): Dense, BM25, Hybrid+RRF, Reranked, + Contextual Retrieval, Parent-child, SAC, RAPTOR, and GraphRAG, plus the supporting + models. +- [Embeddings (Português)](Embeddings-pt-BR) +- [Estratégias de recuperação (Português)](Estrategias-de-Recuperacao) + +### Status at a glance + +| Item | Current state | +|---|---| +| Canonical-matrix embedding | `gemini-embedding-001`, 1,536 dimensions | +| Local embedding alternative | `Qwen/Qwen3-Embedding-0.6B`, 1,024 dimensions | +| Canonical model selection | Provisional, pending PT-BR experiment revalidation | +| Lexical retrieval | OpenSearch BM25 with the `brazilian` analyzer | +| Vector retrieval | Cosine distance over a pgvector HNSW index | +| Configured strategies | 10 labels, from `dense` through `graphrag` | +| Generation and enrichment | `gemini-3.1-flash-lite` | +| Canonical judge | OpenAI `gpt-5.4-mini-2026-03-17`; not yet human-calibrated | + +> “Canonical” means “selected for the publishable matrix.” It does not mean the +> comparison is permanently closed. `configs/experiments/embeddings-ptbr.yaml` +> records the selection as `provisional_pending_revalidation`. + +## Project sources + +- [Main benchmark configuration](https://github.com/brunovicco/ragforge/blob/main/configs/experiments/benchmark-v01.yaml) +- [Local benchmark configuration](https://github.com/brunovicco/ragforge/blob/main/configs/experiments/benchmark-local-v01.yaml) +- [PT-BR embedding experiment](https://github.com/brunovicco/ragforge/blob/main/configs/experiments/embeddings-ptbr.yaml) +- [Architecture Decision Records](https://github.com/brunovicco/ragforge/tree/main/docs/adr) diff --git a/wiki/Retrieval-Strategies.md b/wiki/Retrieval-Strategies.md new file mode 100644 index 0000000..0d658c3 --- /dev/null +++ b/wiki/Retrieval-Strategies.md @@ -0,0 +1,333 @@ +# Retrieval strategies and supporting models + +[Português](Estrategias-de-Recuperacao) · [Home](Home) · [Embeddings](Embeddings) + +> Documentation snapshot: 2026-07-25. The main configuration contains 10 +> strategy labels. The project README groups them into eight broader families; +> SAC and SAC+Contextual are explicit experimental variants in the runner. + +## End-to-end view + +```mermaid +flowchart LR + A["Official document"] --> B["Extraction"] + B --> C["Legal structural chunking"] + C --> D["source_text"] + D --> E{"Retrieval-text variant"} + E -->|baseline| F["source_text"] + E -->|contextual| G["chunk context + source"] + E -->|SAC| H["document summary + source"] + E -->|SAC + contextual| I["summary + context + source"] + F --> J["Dense / BM25 indexes"] + G --> J + H --> J + I --> J + J --> K["Retrieve top-k evidence"] + K --> L["Generate cited answer from source_text"] + L --> M["Independent judge and optional audit"] +``` + +The legal structural chunker splits by normative hierarchy—article, paragraph, +item, and sub-item—and gives each unit a stable structural ID. For the base, +Contextual, and SAC variants: + +- `source_text` is extracted from the official document; +- `retrieval_text` may contain generated context or summaries; +- indexing uses `retrieval_text`; +- answer generation and citations use `source_text`. + +RAPTOR is the exception that must remain explicit: it creates synthetic summary +nodes whose `source_text` is generated text. Those nodes can enter answer-generation +context and carry the aggregated structural IDs of their children. They provide a +retrieval abstraction, not authoritative legal wording. + +## Strategy matrix + +| Configuration label | Candidate source | Ranking | Embedding? | Extra model work | +|---|---|---|---|---| +| `dense` | Base chunks | pgvector cosine similarity | Yes | None | +| `sparse_bm25` | Base chunks | OpenSearch BM25 | No | None | +| `hybrid_rrf` | Dense + BM25 | Reciprocal Rank Fusion | Yes, on dense branch | None | +| `reranked` | Hybrid top 50 | Cross-encoder score, then top 5 | Yes, during first stage | Local cross-encoder | +| `contextual` | Per-chunk context + source | Hybrid RRF | Yes | One Gemini context call per chunk | +| `parent_child` | Dense child hits | Dense score, then parent expansion | Yes | None | +| `sac` | Document summary + source | Dense cosine similarity | Yes | One Gemini summary per document | +| `sac_contextual` | Summary + chunk context + source | Dense cosine similarity | Yes | Document and chunk enrichment | +| `raptor` | Leaves + recursive summaries | Dense cosine similarity | Yes | Gemini summaries for each tree group | +| `graphrag` | LightRAG graph and chunks | LightRAG order, mapped to `1/rank` | Yes | Gemini entity/relation extraction | + +The default `top_k` is 5, and the reranking pool is 50. + +## 1. Dense + +Dense retrieval is the semantic baseline: + +1. embed every chunk's `retrieval_text`; +2. store vectors in a pgvector column; +3. build an HNSW index using `vector_cosine_ops`; +4. embed the question with the same model; +5. return the nearest `top_k` vectors. + +Strengths: + +- recognizes paraphrases and semantically related wording; +- does not require exact query terms to occur in the chunk; +- is reusable as the retrieval layer for SAC and RAPTOR. + +Trade-offs: + +- quality depends on the embedding model and integration; +- exact identifiers, article numbers, and rare terms may be missed; +- changing the embedding space requires rebuilding the index. + +## 2. Sparse BM25 + +`sparse_bm25` performs lexical retrieval in OpenSearch with the `brazilian` +analyzer. BM25 ranks text using term frequency, inverse document frequency, and +document-length normalization. + +Strengths: + +- effective for exact terms, legal identifiers, article numbers, and acronyms; +- has no model inference cost; +- is independent from the selected dense embedding. + +Trade-offs: + +- semantic paraphrases with little lexical overlap are harder to find; +- tokenization and language analysis affect results; +- this is classical lexical search, not a learned sparse embedding. + +The sparse store searches `retrieval_text` but returns the associated +authoritative `source_text`. + +## 3. Hybrid + RRF + +Hybrid retrieval runs Dense and BM25 independently, then combines their ranks +with Reciprocal Rank Fusion: + +```text +RRF(chunk) = Σ 1 / (60 + rank) +``` + +A chunk appearing in both rankings accumulates both contributions. RRF compares +ranks instead of raw scores, avoiding a direct comparison between BM25 and cosine +score scales. + +In RAGForge, each branch is asked for the same candidate depth requested by the +caller. For ordinary `hybrid_rrf` that is top-k; for the reranked strategy it is +the wider rerank pool. + +## 4. Reranked + +The reranked strategy is a two-stage pipeline: + +```text +Hybrid top 50 + -> cross-encoder(query, chunk) for every candidate + -> sort by cross-encoder score + -> top 5 +``` + +The model is `cross-encoder/ms-marco-MiniLM-L-6-v2`. Unlike a bi-encoder, a +cross-encoder jointly reads the question and chunk, enabling finer interaction +between their tokens. It is more expensive per candidate, so it is not run over +the whole corpus. + +Important limitation: the model was trained for MS MARCO passage ranking and its +model card is English-oriented. It was not selected through the project's +dedicated PT-BR model comparison. Its performance on Brazilian legal Portuguese +therefore needs empirical validation. + +## 5. Contextual Retrieval + +For every chunk, `gemini-3.1-flash-lite` generates a short explanation that +locates it within the complete source document: + +```text +retrieval_text = chunk-specific context + source_text +``` + +RAGForge then indexes the enriched text in both Dense and BM25 stores and +retrieves with Hybrid+RRF. This implements both contextual embeddings and +contextual BM25. + +Strength: + +- restores information lost when a legal provision does not repeat the norm's + subject, authority, or scope. + +Costs and risks: + +- one real LLM call per chunk during index preparation; +- generated context can be wrong or overemphasize one interpretation; +- the current contextualizer is not wired to the persistent LLM cache used by + some other stages, so resume can repeat this work; +- only `source_text`, never the generated blurb, is sent to answer generation. + +## 6. Parent-child + +Parent-child, or small-to-big retrieval, searches fine-grained legal chunks and +returns a larger authoritative parent: + +```text +search paragraph/item -> return its parent article +``` + +The relationship comes from the real legal hierarchy produced by the chunker, +not an arbitrary character window. Duplicate parents are removed. If several +high-ranked children share a parent, the strategy can return fewer than +`top_k` results because it does not backfill after deduplication. + +The current implementation uses Dense—not Hybrid—as its inner retriever. + +## 7. Summary-Augmented Chunking (SAC) + +SAC generates one summary for each immutable document version with +`gemini-3.1-flash-lite`, then prefixes the same summary to every chunk from that +document: + +```text +retrieval_text = document summary + source_text +``` + +It targets Document-Level Retrieval Mismatch: finding a locally plausible clause +from the wrong norm. The `sac` label uses Dense retrieval to isolate the effect +of the document summary. + +Trade-offs: + +- one generation call per document rather than per chunk; +- the summary can improve document discrimination; +- one summary error is repeated across every chunk in that document; +- the common prefix can weaken within-document discrimination; +- SAC remains experimental until improvement is measured across embedding + families without a material structural-recall regression. + +## 8. SAC + Contextual + +This composition preserves both levels of context: + +```text +retrieval_text = + document summary + + chunk-specific context + + source_text +``` + +It reuses the already contextualized chunks and applies the document summary on +top. The strategy then uses Dense retrieval. A distinct strategy label and index +fingerprint prevent the result from being reported as ordinary Dense or SAC. + +## 9. RAPTOR + +RAPTOR adds recursively summarized nodes above the original chunks and searches +all levels together—the “collapsed tree” mode. + +The RAGForge implementation is intentionally minimal: + +- groups nodes in source-document order, five at a time; +- summarizes each group with `gemini-3.1-flash-lite`; +- repeats up to five levels or until one root remains; +- builds a separate tree per document; +- flattens leaves and summaries into one Dense index. + +This is **not** the full RAPTOR paper algorithm. It does not apply UMAP reduction +or Gaussian-mixture semantic clustering. Adjacent but topically different +articles can be summarized together, and generated summary nodes are returned as +retrieval content. Because those nodes store the generated summary as +`source_text`, they can also reach the answer generator; their child structural +IDs do not make the generated wording authoritative. The simplification and this +evidence-quality risk must remain visible in comparisons. + +## 10. GraphRAG + +RAGForge integrates LightRAG: + +1. preserve the project's legal chunk boundaries through LightRAG's custom + chunking extension; +2. use the configured retrieval embedder for LightRAG vectors; +3. use `gemini-3.1-flash-lite` for entity and relation extraction; +4. query in LightRAG `local` mode by default; +5. map returned chunk content back to RAGForge chunks by exact text matching. + +Current limitations: + +- exact-text mapping can drop results that LightRAG reformats or cannot match; +- LightRAG exposes no native per-chunk relevance score in this path, so RAGForge + records `1/rank`; +- graph paths, entity confidence, and relationship provenance are not included + in the retrieval metric; +- indexing makes multiple LLM calls per chunk and is materially more expensive; +- `global`, `hybrid`, `mix`, and `naive` modes are accepted by the adapter, but + the main benchmark fixes `local`. + +Temporal GraphRAG is a future, separate strategy. It is not implemented in the +current matrix because the corpus does not yet have version-qualified temporal +evidence. + +## Supporting models: what is and is not an embedding + +| Stage | Model | Role | Retrieval embedding? | +|---|---|---|---| +| Dense indexing/query | Configured Gemini or local model | Produce retrieval vectors | Yes | +| Reranking | `cross-encoder/ms-marco-MiniLM-L-6-v2` | Jointly score query–chunk pairs | No | +| Contextual Retrieval | `gemini-3.1-flash-lite` | Generate chunk-specific context | No | +| SAC | `gemini-3.1-flash-lite` | Generate one summary per document version | No | +| RAPTOR | `gemini-3.1-flash-lite` | Generate recursive summary nodes | No | +| GraphRAG extraction | `gemini-3.1-flash-lite` | Extract entities and relations | No | +| Answer generation | Configured `gemini-3.1-flash-lite` | Produce a cited answer | No | +| Canonical judge | `gpt-5.4-mini-2026-03-17` | Score faithfulness, relevancy, and abstention | No | +| Judge relevancy | `text-embedding-3-small` | Support RAGAS Answer Relevancy | Yes, but evaluation-only | +| Optional semantic audit | `gpt-5.4-mini-2026-03-17` | Verify claim support and rewrite once | No | + +The canonical judge is independent from the Gemini answer generator, but its +scores remain unvalidated until the planned human-calibration exercise reaches +the required agreement. + +## Which providers are contacted? + +| Command | Retrieval embedding | Other live providers | +|---|---|---| +| `make bench-live` | Gemini | Gemini generation/enrichment + OpenAI judge | +| `make bench-live-local` | Local Qwen | Gemini generation/enrichment + OpenAI judge | +| `make bench` | Intended cache replay | Not implemented; the command currently fails closed | + +Therefore, `make bench-live-local` is not an offline benchmark. It only removes +the external provider from the retrieval-embedding stage. + +## Evaluation + +Retrieval strategies share structural-unit judgments and report: + +- Recall@k; +- Precision@k; +- nDCG@k; +- MRR; +- document-level mismatch metrics for relevant variants; +- coverage and failures. + +Answer generation is evaluated separately for Citation Accuracy, Faithfulness, +Answer Relevancy, and abstention behavior. A strategy can retrieve well and +still produce a poor answer; the two layers should not be collapsed into one +undocumented score. + +## Sources + +### RAGForge + +- [Main benchmark runner](https://github.com/brunovicco/ragforge/blob/main/src/ragforge/evaluation/run.py) +- [Retrieval implementations](https://github.com/brunovicco/ragforge/tree/main/src/ragforge/retrieval) +- [ADR-0006: legal structural chunking](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0006-legal-structural-chunker.md) +- [ADR-0010: GraphRAG scope](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0010-graphrag-evaluation-scope.md) +- [ADR-0015: SAC](https://github.com/brunovicco/ragforge/blob/main/docs/adr/0015-summary-augmented-chunking.md) + +### Primary references + +- [OpenSearch: BM25 keyword search](https://docs.opensearch.org/latest/search-plugins/keyword-search/) +- [OpenSearch: Reciprocal Rank Fusion](https://docs.opensearch.org/latest/search-plugins/search-pipelines/score-ranker-processor/) +- [Anthropic: Contextual Retrieval](https://www.anthropic.com/engineering/contextual-retrieval) +- [RAPTOR paper](https://arxiv.org/abs/2401.18059) +- [SAC paper](https://aclanthology.org/2025.nllp-1.3/) +- [LightRAG repository](https://github.com/HKUDS/LightRAG) +- [Cross-encoder model card](https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2) diff --git a/wiki/_Sidebar.md b/wiki/_Sidebar.md new file mode 100644 index 0000000..5d8f8b1 --- /dev/null +++ b/wiki/_Sidebar.md @@ -0,0 +1,13 @@ +## RAGForge + +- [Home / Início](Home) + +### Português + +- [Embeddings](Embeddings-pt-BR) +- [Estratégias e modelos](Estrategias-de-Recuperacao) + +### English + +- [Embeddings](Embeddings) +- [Strategies and models](Retrieval-Strategies)