diff --git a/.gitignore b/.gitignore index 8b5391a..92fb53c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ htmlcov/ dist/ build/ .venv/ +.ragforge/ # Secrets and local configuration .env diff --git a/Makefile b/Makefile index afc8bcb..011e256 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install lint type test quality bench bench-live api dashboard infra-up infra-down +.PHONY: install lint type test quality bench bench-live bench-live-local api dashboard infra-up infra-down install: uv sync --all-groups @@ -24,6 +24,9 @@ bench: bench-live: uv run python -m ragforge.evaluation.run --mode live --config configs/experiments/benchmark-v01.yaml +bench-live-local: + uv run python -m ragforge.evaluation.run --mode live --config configs/experiments/benchmark-local-v01.yaml + api: uv run uvicorn apps.api.main:app --reload diff --git a/README.md b/README.md index 0435e64..a1f5180 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # RAGForge +*Leia isto em [Português](README.pt-BR.md).* + **Adaptive RAG benchmarking platform for Brazilian financial and regulatory documents.** RAGForge is being built to benchmark sparse, dense, hybrid, contextual, hierarchical (RAPTOR), graph (GraphRAG) and corrective strategies - measuring answer quality, retrieval precision, latency and cost on **RegRAG-BR**, a 230-question golden dataset over CMN/BCB and CVM norms. @@ -23,7 +25,7 @@ Most RAG comparisons are anecdotal. RAGForge treats the question "*which RAG str | 7 | RAPTOR | Recursive summary tree (minimal impl.) | Implemented | | 8 | GraphRAG | LightRAG adapter (local + global) | Implemented | -Cross-cutting: **Adaptive Router** (rules + few-shot, planned), **Corrective workflow** (evidence evaluator with retry / reformulation / insufficient-evidence declaration, LangGraph, planned), **governance** (answer → chunk → article citation tracing via Citation Accuracy, implemented; full audit trail beyond the metric, planned), **observability** (Langfuse metadata-only tracing implemented; OpenTelemetry planned). +Cross-cutting: **Adaptive Router** (rules + few-shot, planned), **Corrective workflow** (evidence evaluator with retry / reformulation / insufficient-evidence declaration, LangGraph, planned), **governance** (answer → chunk → article citation tracing via Citation Accuracy, plus a post-generation semantic-support audit with bounded rewrite and a tamper-evident evidence trail per run, implemented), **observability** (Langfuse metadata-only tracing implemented; OpenTelemetry planned). ## Status @@ -37,10 +39,13 @@ is actually running today versus what the design targets - see the [PR history]( | All 8 benchmarked retrieval strategies (Dense through GraphRAG) | Implemented | | Evaluation harness + structural-coverage judgments (ADR-0002) | Implemented | | Observability (Langfuse, metadata-only) | Implemented | -| Answer generation + Citation Accuracy + RAGAS judge (ADR-0007) | Implemented - judge uncalibrated (ADR-0007 kappa exercise pending) | +| Answer generation + Citation Accuracy | Implemented | +| Independent LLM judge - Faithfulness/Answer Relevancy + abstention (ADR-0018) | Implemented - uncalibrated (ADR-0007 kappa exercise pending) | +| Post-generation citation/semantic-support audit + bounded rewrite (ADR-0016) | Implemented - off by default (`audit.enabled: false`) | +| Auditable, tamper-evident run evidence directory (ADR-0017) | Implemented - `artifacts/runs//`, verified via `scripts/verify_run.py` | | Main benchmark runner (`make bench-live`, all 8 strategies + answer quality) | Implemented - live mode only | | Adaptive Router, Corrective workflow | Planned | -| RegRAG-BR golden set | 230 questions published (5 of 6 corpus documents curated) | +| RegRAG-BR golden set | 230 questions published: 36 validation/dev + 194 test | | API / dashboard apps | Planned (scaffolding only) | | `make bench` (cached, bit-for-bit replay, ADR-0004) | Planned - needs a versioned LLM call cache, not built yet | @@ -48,13 +53,25 @@ is actually running today versus what the design targets - see the [PR history]( ```bash uv sync --all-groups -make infra-up # Postgres+pgvector, OpenSearch (docker compose profiles) -GEMINI_API_KEY=... make bench-live # real run, all 8 strategies, real API cost -make dashboard # benchmark view + side-by-side strategy Arena +make infra-up # Postgres+pgvector, OpenSearch +GEMINI_API_KEY=... OPENAI_API_KEY=... make bench-live +make bench-live-local # same matrix, local Qwen embeddings +make dashboard # benchmark + side-by-side Arena ``` `make bench-live` calls real providers (embeddings, contextualization, RAPTOR summarization, GraphRAG entity extraction - see the strategy table above). `make bench` (deterministic, zero-cost replay from a versioned LLM cache) is the target design per [ADR-0004](docs/adr/0004-benchmark-reproducibility-policy.md), but that cache layer doesn't exist yet - only live mode is implemented. +The canonical publishable matrix uses `gemini-embedding-001`, selected by the isolated PT-BR +embedding comparison (ADR-0005). `make bench-live-local` uses +`Qwen/Qwen3-Embedding-0.6B` as ADR-0013's credential-free embedding alternative; it is a +separately identified run, not a silent fallback or a relabeling of the quality winner. + +Both live commands persist per-text embeddings and completed indexes under the ignored +`.ragforge/cache/` directory. A deterministic index fingerprint includes corpus-derived text, +embedding identity, and synthetic-text producer identity; partial indexes are never marked +reusable. `--resume ` skips strategies already checkpointed. A repository lock rejects +concurrent benchmark processes before they can mutate shared indexes. + ## Key design decisions All non-obvious choices are recorded as [ADRs](docs/adr/README.md). The load-bearing ones: @@ -65,6 +82,8 @@ All non-obvious choices are recorded as [ADRs](docs/adr/README.md). The load-bea - [ADR-0006](docs/adr/0006-legal-structural-chunker.md) - domain-aware chunking by legal hierarchy (Art./§/inciso) with stable structural IDs. - [ADR-0007](docs/adr/0007-llm-judge-calibration-ptbr.md) - the LLM judge is calibrated against human evaluation in PT-BR and the agreement is published. - [ADR-0011](docs/adr/0011-structural-id-collision-in-amended-norms.md) - structural IDs that collide across amendment history/appended annexes are excluded from golden-set citations, not fixed at the chunker level. +- [ADR-0016](docs/adr/0016-post-generation-citation-audit.md) - a semantic-support verifier and at most one bounded rewrite catch unsupported claims a citation-existence check alone would miss. +- [ADR-0017](docs/adr/0017-auditable-evidence-lineage.md) - every published score traces back to a hash-chained, tamper-evident evidence directory per run - exact inputs, model identities, and retrieval candidates, not just the aggregate metric. ## Repository layout @@ -82,7 +101,7 @@ The core is framework-free: `RetrievalStrategy` is a Protocol; LLM SDKs are bann ## Dataset - RegRAG-BR -230 questions (7 query classes) over selected CMN/BCB resolutions (4,893, risk management, Open Finance, AML) and CVM/CMN norms, with article-level relevance judgments and reference answers, published under CC-BY-4.0 with a datasheet. Norms are official acts (art. 8, I, Law 9,610/98 - not copyright-protected). +230 questions (7 query classes) over selected CMN/BCB resolutions (4,893, risk management, Open Finance, AML) and CVM/CMN norms, with article-level relevance judgments and reference answers, published under CC-BY-4.0 with a datasheet. The deterministic stratified split reserves 36 questions for router development/validation and 194 for official test metrics. Norms are official acts (art. 8, I, Law 9,610/98 - not copyright-protected). Published (`datasets/regrag-br/judgments.json`): 230 hand-curated questions, each with a reference answer, verified against the real parsed text of 5 corpus documents (LC-105/2001, RES-CMN-4893/2021, RES-CMN-5274/2025, LEI-13709/2018-LGPD, ICVM-607/2019). A 6th corpus document, LEI-6385/1976, is not yet curated. Structural IDs known to be ambiguous in the real source text - amendment-history and appended-annex artifacts in 3 of the 5 documents - are excluded from citation; see [ADR-0011](docs/adr/0011-structural-id-collision-in-amended-norms.md). diff --git a/README.pt-BR.md b/README.pt-BR.md new file mode 100644 index 0000000..d2157d3 --- /dev/null +++ b/README.pt-BR.md @@ -0,0 +1,121 @@ +# RAGForge + +*Read this in [English](README.md).* + +**Plataforma adaptativa de benchmark de RAG para documentos financeiros e regulatórios brasileiros.** + +O RAGForge está sendo construído para comparar estratégias sparse, dense, hybrid, contextual, hierárquica (RAPTOR), grafo (GraphRAG) e corretiva - medindo qualidade de resposta, precisão de recuperação, latência e custo sobre o **RegRAG-BR**, um golden dataset de 230 perguntas sobre normas do CMN/BCB e da CVM. + +> 🚧 v0.1 em desenvolvimento. Veja [Status](#status) para o que já está implementado versus planejado. + +## Por que isso existe + +A maioria das comparações de RAG é anedótica. O RAGForge trata a pergunta "*qual estratégia de RAG eu deveria usar?*" como um experimento: 8 estratégias avaliadas × 7 classes de pergunta, com um roteador adaptativo pensado para ser avaliado contra um **oráculo empírico**, e todo número publicado reproduzível bit a bit a partir de um cache versionado de chamadas de LLM. Veja [Status](#status) para o que já foi construído. + +## Estratégias avaliadas + +| # | Estratégia | Abordagem | Status | +| --- | ---------- | ---------- | -------- | +| 1 | Dense (baseline) | pgvector, top-k fixo | Implementado | +| 2 | Sparse BM25 | OpenSearch, analisador `brazilian` | Implementado | +| 3 | Hybrid + RRF | BM25 + dense + Reciprocal Rank Fusion | Implementado | +| 4 | Reranked | Hybrid top-50 → cross-encoder → top-5 | Implementado | +| 5 | Contextual Retrieval | Contexto via LLM por chunk + prompt caching | Implementado | +| 6 | Parent-child / multi-vector | Busca em chunks pequenos, entrega a seção | Implementado | +| 7 | RAPTOR | Árvore recursiva de resumos (impl. mínima) | Implementado | +| 8 | GraphRAG | Adapter LightRAG (local + global) | Implementado | + +Transversais: **Roteador Adaptativo** (regras + few-shot, planejado), **Fluxo corretivo** (avaliador de evidência com retry / reformulação / declaração de evidência insuficiente, LangGraph, planejado), **governança** (rastreamento resposta → chunk → artigo via Citation Accuracy, mais uma auditoria de suporte semântico pós-geração com reescrita limitada e uma trilha de evidência à prova de adulteração por execução, implementado), **observabilidade** (tracing do Langfuse apenas com metadados implementado; OpenTelemetry planejado). + +## Status + +O RAGForge está em desenvolvimento ativo (v0.1, ver a nota de checkpoint acima). Esta seção acompanha o que +está de fato rodando hoje versus o que é meta de design - veja o [histórico de PRs](../../pulls?q=is%3Apr) para como cada linha chegou lá. + +| Componente | Status | +|---|---| +| Chunker estrutural jurídico (ADR-0006) | Implementado | +| Pipeline de ingestão (extração, hash de snapshot) | Implementado | +| As 8 estratégias de recuperação avaliadas (Dense até GraphRAG) | Implementado | +| Harness de avaliação + julgamentos de cobertura estrutural (ADR-0002) | Implementado | +| Observabilidade (Langfuse, apenas metadados) | Implementado | +| Geração de resposta + Citation Accuracy | Implementado | +| Judge de LLM independente - Faithfulness/Answer Relevancy + abstenção (ADR-0018) | Implementado - não calibrado (exercício de kappa da ADR-0007 pendente) | +| Auditoria de citação/suporte semântico pós-geração + reescrita limitada (ADR-0016) | Implementado - desligado por padrão (`audit.enabled: false`) | +| Diretório de evidência auditável e à prova de adulteração por execução (ADR-0017) | Implementado - `artifacts/runs//`, verificado via `scripts/verify_run.py` | +| Runner principal do benchmark (`make bench-live`, 8 estratégias + qualidade de resposta) | Implementado - apenas modo live | +| Roteador Adaptativo, Fluxo corretivo | Planejado | +| Golden set RegRAG-BR | 230 perguntas: 36 validation/dev + 194 test | +| Apps de API / dashboard | Planejado (apenas scaffolding) | +| `make bench` (replay determinístico, bit a bit, ADR-0004) | Planejado - precisa de um cache versionado de chamadas de LLM, ainda não construído | + +## Início rápido + +```bash +uv sync --all-groups +make infra-up # Postgres+pgvector, OpenSearch +GEMINI_API_KEY=... OPENAI_API_KEY=... make bench-live +make bench-live-local # mesma matriz, embeddings Qwen locais +make dashboard # benchmark + Arena lado a lado +``` + +`make bench-live` chama provedores reais (embeddings, contextualização, sumarização do RAPTOR, extração de entidades do GraphRAG - ver a tabela de estratégias acima). `make bench` (replay determinístico e sem custo a partir de um cache versionado de LLM) é o design-alvo segundo a [ADR-0004](docs/adr/0004-benchmark-reproducibility-policy.md), mas essa camada de cache ainda não existe - apenas o modo live está implementado. + +A matriz canônica e publicável usa `gemini-embedding-001`, selecionado pela +comparação isolada de embeddings em PT-BR (ADR-0005). `make bench-live-local` +usa `Qwen/Qwen3-Embedding-0.6B` como alternativa operacional sem credenciais +de embedding prevista pela ADR-0013; é uma execução identificada separadamente, +não um fallback silencioso nem uma troca do vencedor de qualidade. + +Os dois comandos live persistem embeddings por texto e índices completos no +diretório ignorado `.ragforge/cache/`. O fingerprint inclui o texto derivado do +corpus, a identidade do embedding e o produtor de texto sintético; índices +parciais nunca são marcados como reutilizáveis. `--resume ` pula +estratégias já concluídas, e um lock do repositório impede benchmarks simultâneos. + +## Decisões de design relevantes + +Todas as escolhas não óbvias são registradas como [ADRs](docs/adr/README.md). As mais estruturantes: + +- [ADR-0002](docs/adr/0002-article-level-relevance-judgments.md) - julgamentos de relevância no **nível de artigo da norma**, para que as métricas de recuperação continuem comparáveis entre estratégias que fragmentam o texto de formas diferentes (ou nem retornam chunks). +- [ADR-0003](docs/adr/0003-empirical-router-oracle.md) - o roteador é avaliado contra um **oráculo empírico por pergunta** (melhor estratégia medida, não presumida), com uma divisão dev/test que evita vazamento de few-shot. +- [ADR-0004](docs/adr/0004-benchmark-reproducibility-policy.md) - o `make bench` reproduz a partir de um cache versionado de LLM: reprodução bit a bit, custo zero de API. +- [ADR-0006](docs/adr/0006-legal-structural-chunker.md) - chunking sensível ao domínio pela hierarquia jurídica (Art./§/inciso), com IDs estruturais estáveis. +- [ADR-0007](docs/adr/0007-llm-judge-calibration-ptbr.md) - o judge de LLM é calibrado contra avaliação humana em PT-BR e a concordância é publicada. +- [ADR-0011](docs/adr/0011-structural-id-collision-in-amended-norms.md) - IDs estruturais que colidem entre histórico de emendas/anexos anexados são excluídos das citações do golden set, não corrigidos no nível do chunker. +- [ADR-0016](docs/adr/0016-post-generation-citation-audit.md) - um verificador de suporte semântico e no máximo uma reescrita limitada capturam alegações sem suporte que uma checagem de mera existência da citação deixaria passar. +- [ADR-0017](docs/adr/0017-auditable-evidence-lineage.md) - todo score publicado é rastreável até um diretório de evidência encadeado por hash e à prova de adulteração, por execução - entradas exatas, identidades de modelo e candidatos de recuperação, não só a métrica agregada. + +## Estrutura do repositório + +``` +apps/ # api/ (FastAPI) e dashboard/ (Streamlit: benchmark + Arena) +src/ragforge/ # domain/ (núcleo livre de framework) · ingestion/ chunking/ embeddings/ + # retrieval/ reranking/ routing/ generation/ evaluation/ governance/ +datasets/ # corpus/ (snapshot versionado) + regrag-br/ (golden set, CC-BY-4.0) +experiments/ # resultados versionados + cache de LLM por run-id +configs/ # configs declarativas de experimentos - todo número do README nasce aqui +docs/adr/ # architecture decision records +``` + +O núcleo é livre de framework: `RetrievalStrategy` é um Protocol; SDKs de LLM são banidos dos pacotes centrais por uma guarda de arquitetura no CI (`scripts/validate_architecture.py`, limites em `pyproject.toml`). + +## Dataset - RegRAG-BR + +230 perguntas (7 classes de consulta) sobre resoluções selecionadas do CMN/BCB (4.893, gestão de risco, Open Finance, PLD/FT) e normas da CVM/CMN, com julgamentos de relevância no nível de artigo e respostas de referência, publicadas sob CC-BY-4.0 com um datasheet. O split determinístico e estratificado reserva 36 perguntas para desenvolvimento/validação do roteador e 194 para as métricas oficiais de teste. As normas são atos oficiais (art. 8º, I, Lei 9.610/98 - não protegidos por direito autoral). + +Publicado (`datasets/regrag-br/judgments.json`): 230 perguntas curadas manualmente, cada uma com uma resposta de referência, verificadas contra o texto real extraído de 5 documentos do corpus (LC-105/2001, RES-CMN-4893/2021, RES-CMN-5274/2025, LEI-13709/2018-LGPD, ICVM-607/2019). Um 6º documento do corpus, a LEI-6385/1976, ainda não foi curado. IDs estruturais conhecidos por serem ambíguos no texto-fonte real - artefatos de histórico de emendas e anexos anexados em 3 dos 5 documentos - são excluídos de citação; ver [ADR-0011](docs/adr/0011-structural-id-collision-in-amended-norms.md). + +## Desenvolvimento + +```bash +uv sync --all-groups +uv run pytest +uv run python scripts/quality_gate.py # ruff, mypy, pytest (≥80% do core), bandit, pip-audit, guarda de arquitetura +``` + +Estruturado com [claude-python-engineering-harness](https://github.com/brunovicco/claude-python-engineering-harness) ([ADR-0009](docs/adr/0009-scaffold-via-engineering-harness.md)). + +## Licença + +Código: MIT · Dataset (RegRAG-BR): CC-BY-4.0 diff --git a/configs/experiments/benchmark-local-v01.yaml b/configs/experiments/benchmark-local-v01.yaml new file mode 100644 index 0000000..14a905e --- /dev/null +++ b/configs/experiments/benchmark-local-v01.yaml @@ -0,0 +1,56 @@ +# Provider-free embedding variant of benchmark-v01.yaml (ADR-0013). +# It keeps the same corpus, split, strategies, generator, and independent +# judge as the canonical benchmark; only the embedding backend changes. +run_id: null +corpus: + snapshot: datasets/corpus/ + hash: null +dataset: + path: datasets/regrag-br/ + split: test +strategies: + - dense + - sparse_bm25 + - hybrid_rrf + - reranked + - contextual + - parent_child + - sac + - sac_contextual + - raptor + - graphrag +embedding: + provider: local + model: Qwen/Qwen3-Embedding-0.6B + dimensions: 1024 + # CPU is the stable default on the current development machine. Override + # with cuda on a compatible NVIDIA host; device is execution metadata, not + # a different semantic embedding identity. + device: cpu +generation: + model: gemini-3.1-flash-lite + temperature: 0 +judge: + provider: openai + model: gpt-5.4-mini-2026-03-17 + embedding_model: text-embedding-3-small + reasoning_effort: medium +audit: + enabled: false + provider: openai + model: gpt-5.4-mini-2026-03-17 + reasoning_effort: medium +retrieval: + top_k: 5 + rerank_pool: 50 +llm_cache: + mode: live +execution: + answer_quality_workers: 5 + gemini_max_in_flight: 4 + embedding_cache_dir: .ragforge/cache/embeddings + index_cache_dir: .ragforge/cache/indexes +pricing: + generation: + input_per_million_usd: null + output_per_million_usd: null diff --git a/configs/experiments/benchmark-v01.yaml b/configs/experiments/benchmark-v01.yaml index 19f69a0..3e38db1 100644 --- a/configs/experiments/benchmark-v01.yaml +++ b/configs/experiments/benchmark-v01.yaml @@ -18,24 +18,14 @@ strategies: - raptor - graphrag embedding: - # ADR-0013: local is the operational default - no credentials needed, - # provider-free execution - regardless of the embeddings-ptbr experiment's - # (ADR-0005) quality result (Dense recall@5=0.947 for gemini-embedding-001 - # vs. BAAI/bge-m3's 0.789 on the real golden set; see - # configs/experiments/embeddings-ptbr.yaml and - # experiments/embeddings-ptbr/runs.jsonl). Qwen3-Embedding-0.6B is not a - # predeclared quality winner - the full RegRAG-BR experiment determines - # relative quality (ADR-0013's own acceptance criteria). - provider: local - model: Qwen/Qwen3-Embedding-0.6B - # Matryoshka truncation size; only used when provider: gemini (pgvector's - # HNSW index caps at 2000 dimensions). Ignored for provider: local - the - # model reports its own native dimension. + # Canonical publishable configuration (ADR-0005/ADR-0013): the isolated + # PT-BR comparison selected gemini-embedding-001 on retrieval quality. + # configs/experiments/benchmark-local-v01.yaml keeps Qwen as the + # credential-free operational alternative without relabeling it a winner. + provider: gemini + model: gemini-embedding-001 + # Matryoshka truncation keeps the vector within pgvector HNSW's limit. dimensions: 1536 - # To use the hosted comparator instead: - # provider: gemini - # model: gemini-embedding-001 - # (requires GEMINI_API_KEY or GOOGLE_API_KEY) generation: # Not chosen via a dedicated comparison (unlike the embedding model, # ADR-0005) - a placeholder, not a data-driven winner, matching the @@ -76,7 +66,7 @@ retrieval: top_k: 5 rerank_pool: 50 llm_cache: - mode: cache # cache | live (ADR-0004) + mode: live # full deterministic replay remains future ADR-0004 work execution: # ADR-0014. Both optional - omitting this whole section falls back to the # same defaults. Not the full execution.stages.* schema from the ADR (a @@ -85,3 +75,11 @@ execution: # Gemini ProviderLimiter shared by the embedder/generator/judge. answer_quality_workers: 5 gemini_max_in_flight: 4 + embedding_cache_dir: .ragforge/cache/embeddings + index_cache_dir: .ragforge/cache/indexes +pricing: + # Prices are deliberately declared by the experiment, never hard-coded in + # code. Fill these with the provider's dated prices to publish cost. + generation: + input_per_million_usd: null + output_per_million_usd: null diff --git a/configs/experiments/embeddings-ptbr.yaml b/configs/experiments/embeddings-ptbr.yaml index b58f25e..63a8e8e 100644 --- a/configs/experiments/embeddings-ptbr.yaml +++ b/configs/experiments/embeddings-ptbr.yaml @@ -7,7 +7,7 @@ candidates: - name: BAAI/bge-m3 kind: open provider: sentence-transformers - status: evaluated + status: pending_revalidation dimensions: 1024 notes: > Multilingual, PT-BR capable. Ran on CPU (device=cpu) - the MPS @@ -17,7 +17,7 @@ candidates: - name: gemini-embedding-001 kind: proprietary provider: gemini - status: evaluated + status: pending_revalidation dimensions: 1536 notes: > Requires GEMINI_API_KEY or GOOGLE_API_KEY in the environment - never @@ -28,7 +28,7 @@ candidates: - name: gemini-embedding-2 kind: proprietary provider: gemini - status: evaluated + status: pending_revalidation dimensions: 1536 notes: > Google's first natively multimodal embedding model (GA name; the @@ -75,12 +75,9 @@ dataset: retrieval: top_k: 5 -# Full metrics per run: experiments/embeddings-ptbr/runs.jsonl. Headline -# dense recall@5 on the real golden set (2026-07-22, 465 chunks, n=19 -# judged queries): BAAI/bge-m3=0.789, gemini-embedding-001=0.947, -# gemini-embedding-2=0.763. -# -# Decision (2026-07-22): gemini-embedding-001, on retrieval quality -# (dense/hybrid recall@5=0.947, clearly ahead of both other candidates). -# Frozen into configs/experiments/benchmark-v01.yaml per ADR-0005. -winner: gemini-embedding-001 +# Previous run artifacts were deliberately invalidated before the v0.2 +# stratified split and runner corrections. New metrics will be appended to +# experiments/embeddings-ptbr/runs.jsonl by the dedicated runner. +selection: + model: gemini-embedding-001 + status: provisional_pending_revalidation diff --git a/datasets/regrag-br/split.json b/datasets/regrag-br/split.json index f44c178..be4affe 100644 --- a/datasets/regrag-br/split.json +++ b/datasets/regrag-br/split.json @@ -2,34 +2,63 @@ "schema_version": 1, "dataset_version": "0.2", "train": [], - "validation": [], - "test": [ + "validation": [ "q001", - "q002", "q003", + "q007", + "q008", + "q014", + "q016", + "q021", + "q026", + "q045", + "q058", + "q061", + "q062", + "q067", + "q073", + "q074", + "q075", + "q076", + "q081", + "q092", + "q095", + "q098", + "q099", + "q102", + "q110", + "q113", + "q116", + "q118", + "q127", + "q158", + "q172", + "q173", + "q179", + "q189", + "q195", + "q201", + "q217" + ], + "test": [ + "q002", "q004", "q005", "q006", - "q007", - "q008", "q009", "q010", "q011", "q012", "q013", - "q014", "q015", - "q016", "q017", "q018", "q019", "q020", - "q021", "q022", "q023", "q024", "q025", - "q026", "q027", "q028", "q029", @@ -48,7 +77,6 @@ "q042", "q043", "q044", - "q045", "q046", "q047", "q048", @@ -61,30 +89,21 @@ "q055", "q056", "q057", - "q058", "q059", "q060", - "q061", - "q062", "q063", "q064", "q065", "q066", - "q067", "q068", "q069", "q070", "q071", "q072", - "q073", - "q074", - "q075", - "q076", "q077", "q078", "q079", "q080", - "q081", "q082", "q083", "q084", @@ -95,17 +114,12 @@ "q089", "q090", "q091", - "q092", "q093", "q094", - "q095", "q096", "q097", - "q098", - "q099", "q100", "q101", - "q102", "q103", "q104", "q105", @@ -113,15 +127,11 @@ "q107", "q108", "q109", - "q110", "q111", "q112", - "q113", "q114", "q115", - "q116", "q117", - "q118", "q119", "q120", "q121", @@ -130,7 +140,6 @@ "q124", "q125", "q126", - "q127", "q128", "q129", "q130", @@ -161,7 +170,6 @@ "q155", "q156", "q157", - "q158", "q159", "q160", "q161", @@ -175,14 +183,11 @@ "q169", "q170", "q171", - "q172", - "q173", "q174", "q175", "q176", "q177", "q178", - "q179", "q180", "q181", "q182", @@ -192,19 +197,16 @@ "q186", "q187", "q188", - "q189", "q190", "q191", "q192", "q193", "q194", - "q195", "q196", "q197", "q198", "q199", "q200", - "q201", "q202", "q203", "q204", @@ -220,7 +222,6 @@ "q214", "q215", "q216", - "q217", "q218", "q219", "q220", diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7dcc13e..84f564f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,15 @@ ## Context -Describe the business capability owned by this service and its upstream and downstream dependencies. +RAGForge is an experimental evaluation platform, not an end-user RAG service. +It compares retrieval strategies over the versioned RegRAG-BR regulatory +corpus, generates grounded answers, scores retrieval and answer quality, and +produces tamper-evident evidence for every publishable run. + +External dependencies are official corpus snapshots, hosted Gemini/OpenAI +model APIs, Postgres+pgvector, and OpenSearch. LightRAG uses repository-local +storage. The future API, dashboard, adaptive router, and corrective workflow +consume benchmark results but are not production entrypoints in v0.1. ## Layers @@ -51,4 +59,20 @@ domain -> no outer layer ## Diagrams -Add C4 context/container diagrams and sequence diagrams for critical flows. +```text +corpus manifest + split + judgments + -> integrity gate + -> extraction and legal structural chunking + -> embedding and strategy-specific indexing + -> retrieval over the frozen test split + -> cited answer generation + -> independent judge and optional semantic audit + -> aggregate metrics + per-question evidence + checksums +``` + +The current composition root is `ragforge.evaluation.run`. `domain/` owns +framework-free query, chunk, judgment, and retrieval contracts. Retrieval, +generation, embedding, and storage packages implement boundary behavior. +`evaluation/` coordinates the benchmark and owns its evidence/reporting +contracts. `application/`, `entrypoints/`, and `apps/` remain scaffolding for +future product surfaces. diff --git a/docs/adr/0003-empirical-router-oracle.md b/docs/adr/0003-empirical-router-oracle.md index c35bf23..773dddd 100644 --- a/docs/adr/0003-empirical-router-oracle.md +++ b/docs/adr/0003-empirical-router-oracle.md @@ -11,7 +11,11 @@ The original plan defined the "oracle" as the a-priori mapping `query class → 1. **Empirical per-question oracle.** Since the benchmark already runs all 8 strategies over all questions, the oracle is defined *ex post*: for each question, the strategy with the best composite score (RAGAS quality primary; cost as tie-breaker). The a-priori class mapping is demoted to a **testable hypothesis** - and the "where intuition was wrong" table becomes a report section. 2. **Router metrics:** routing accuracy vs the empirical oracle; **regret** (per-question quality delta between chosen strategy and oracle); comparison of three policies: adaptive router vs best-fixed-strategy vs oracle (ceiling). -3. **Anti-leakage split.** Before curation, a stratified-by-class split: `dev` (~15%, sole source of few-shot examples and rule tuning) and `test` (~85%, frozen; all README numbers come from it). Versioned in `datasets/regrag-br/splits.json`. +3. **Anti-leakage split.** A deterministic stratified-by-class split: + `validation` (~15%, treated as dev and the sole source of future few-shot + examples/rule tuning) and `test` (~85%, frozen; all README numbers come + from it). Versioned in `datasets/regrag-br/split.json`; `train` remains + empty until a learned router is introduced. ## Consequences diff --git a/docs/adr/0005-embedding-comparison-scope.md b/docs/adr/0005-embedding-comparison-scope.md index 3c68f35..f5e6df2 100644 --- a/docs/adr/0005-embedding-comparison-scope.md +++ b/docs/adr/0005-embedding-comparison-scope.md @@ -15,6 +15,14 @@ The embedding comparison is an **isolated experiment**, not a dimension of the m 2. The winner is **frozen** for all remaining strategies and the main benchmark matrix. The choice and its supporting numbers are recorded in `configs/experiments/embeddings-ptbr.yaml`. 3. The main matrix reports a single embedding configuration, declared in the README. +### Clarification after ADR-0013 + +`gemini-embedding-001` remains the quality-selected embedding for the canonical, +publishable matrix. ADR-0013 adds `Qwen/Qwen3-Embedding-0.6B` as a separate local +operational configuration so the embedding stage can run without provider credentials. +The local default is not a replacement winner and its results must carry a distinct +configuration identity. + ## Consequences - Controlled scope: one full indexing pass instead of two; RAPTOR and GraphRAG (highest-effort items) index once. diff --git a/docs/adr/0013-provider-neutral-embedding-backends.md b/docs/adr/0013-provider-neutral-embedding-backends.md index 44ea391..e993ae5 100644 --- a/docs/adr/0013-provider-neutral-embedding-backends.md +++ b/docs/adr/0013-provider-neutral-embedding-backends.md @@ -70,7 +70,11 @@ Provider SDK types SHALL remain in adapters. | Local control | `intfloat/multilingual-e5-large-instruct` | Local | | Hosted comparator | configured stable Gemini embedding model | Gemini API | -The local default is not a predeclared quality winner. The full RegRAG-BR experiment determines relative quality. +The local default is not a predeclared quality winner. ADR-0005's isolated +Dense/Hybrid comparison selected `gemini-embedding-001` for the canonical +publishable matrix. Qwen is the separately configured operational default for +provider-free embedding runs. A future full RegRAG-BR embedding comparison may +change that quality decision, but must do so through a new recorded experiment. ### Exact model revision diff --git a/docs/adr/README.md b/docs/adr/README.md index 7ca79bd..9a6c612 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -15,11 +15,11 @@ Format: [MADR-style](https://adr.github.io/). ADRs are immutable once accepted; | [0009](0009-scaffold-via-engineering-harness.md) | Scaffold and quality gates via claude-python-engineering-harness | Accepted | | [0010](0010-graphrag-evaluation-scope.md) | GraphRAG evaluation scope and provenance recovery | Accepted | | [0011](0011-structural-id-collision-in-amended-norms.md) | Structural-ID collisions in amended norms | Accepted | -| [0012](0012-benchmark-integrity-and-end-to-end-evaluation.md) | Enforce benchmark corpus integrity and end-to-end evaluation | Proposed | -| [0013](0013-provider-neutral-embedding-backends.md) | Adopt provider-neutral embedding backends with a local default | Proposed | -| [0014](0014-bounded-parallel-benchmark-execution.md) | Use bounded, deterministic parallel execution for benchmark stages | Proposed | -| [0015](0015-summary-augmented-chunking.md) | Evaluate Summary-Augmented Chunking as a separate retrieval strategy | Proposed | -| [0016](0016-post-generation-citation-audit.md) | Add bounded post-generation citation and support auditing | Proposed | -| [0017](0017-auditable-evidence-lineage.md) | Produce auditable and tamper-evident evidence lineage for every run | Proposed | -| [0018](0018-independent-llm-judge-provider.md) | Use an independent and calibrated OpenAI LLM judge | Proposed | +| [0012](0012-benchmark-integrity-and-end-to-end-evaluation.md) | Enforce benchmark corpus integrity and end-to-end evaluation | Accepted | +| [0013](0013-provider-neutral-embedding-backends.md) | Adopt provider-neutral embedding backends with a local default | Accepted | +| [0014](0014-bounded-parallel-benchmark-execution.md) | Use bounded, deterministic parallel execution for benchmark stages | Accepted | +| [0015](0015-summary-augmented-chunking.md) | Evaluate Summary-Augmented Chunking as a separate retrieval strategy | Accepted | +| [0016](0016-post-generation-citation-audit.md) | Add bounded post-generation citation and support auditing | Accepted | +| [0017](0017-auditable-evidence-lineage.md) | Produce auditable and tamper-evident evidence lineage for every run | Accepted | +| [0018](0018-independent-llm-judge-provider.md) | Use an independent and calibrated OpenAI LLM judge | Accepted | | [0019](0019-temporal-graphrag-experimental-strategy.md) | Add Temporal GraphRAG only as a future experimental strategy | Proposed | diff --git a/experiments/20260723T001223Z/results.json b/experiments/20260723T001223Z/results.json deleted file mode 100644 index bfd95fe..0000000 --- a/experiments/20260723T001223Z/results.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "run_id": "20260723T001223Z", - "mode": "live", - "config_path": "configs/experiments/benchmark-v01.yaml", - "embedding": { - "model": "gemini-embedding-001", - "dimensions": 1536 - }, - "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", - "contextualizer_model": "gemini-3.1-flash-lite", - "summarizer_model": "gemini-3.1-flash-lite", - "graphrag_llm_model": "gemini-3.1-flash-lite", - "graphrag_mode": "local", - "k": 5, - "n_chunks": 465, - "metrics": { - "dense": { - "recall_at_k": 0.9473684210526315, - "precision_at_k": 0.26315789473684215, - "ndcg_at_k": 0.8254757518001955, - "mrr": 0.7894736842105263, - "k": 5.0, - "n": 19.0 - }, - "sparse_bm25": { - "recall_at_k": 0.6578947368421053, - "precision_at_k": 0.18947368421052632, - "ndcg_at_k": 0.49659401793880914, - "mrr": 0.43859649122807015, - "k": 5.0, - "n": 19.0 - }, - "hybrid_rrf": { - "recall_at_k": 0.9473684210526315, - "precision_at_k": 0.26315789473684215, - "ndcg_at_k": 0.7582697131525727, - "mrr": 0.6903508771929825, - "k": 5.0, - "n": 19.0 - }, - "reranked": { - "recall_at_k": 0.7631578947368421, - "precision_at_k": 0.2105263157894737, - "ndcg_at_k": 0.6522771443053881, - "mrr": 0.6403508771929824, - "k": 5.0, - "n": 19.0 - }, - "parent_child": { - "recall_at_k": 0.9473684210526315, - "precision_at_k": 0.3026315789473684, - "ndcg_at_k": 0.8449005016122241, - "mrr": 0.8157894736842105, - "k": 5.0, - "n": 19.0 - }, - "contextual": { - "recall_at_k": 0.9078947368421053, - "precision_at_k": 0.2736842105263158, - "ndcg_at_k": 0.7534777550074311, - "mrr": 0.7254385964912281, - "k": 5.0, - "n": 19.0 - }, - "raptor": { - "recall_at_k": 0.9605263157894737, - "precision_at_k": 0.5052631578947369, - "ndcg_at_k": 0.8048019775970073, - "mrr": 0.7912280701754386, - "k": 5.0, - "n": 19.0 - }, - "graphrag": { - "recall_at_k": 0.8421052631578947, - "precision_at_k": 0.23157894736842108, - "ndcg_at_k": 0.7232825999888768, - "mrr": 0.7017543859649122, - "k": 5.0, - "n": 19.0 - } - } -} \ No newline at end of file diff --git a/experiments/20260723T221236Z/results.json b/experiments/20260723T221236Z/results.json deleted file mode 100644 index a4ae12c..0000000 --- a/experiments/20260723T221236Z/results.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "run_id": "20260723T221236Z", - "mode": "live", - "config_path": "configs/experiments/benchmark-v01.yaml", - "embedding": { - "model": "gemini-embedding-001", - "dimensions": 1536 - }, - "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", - "contextualizer_model": "gemini-3.1-flash-lite", - "summarizer_model": "gemini-3.1-flash-lite", - "graphrag_llm_model": "gemini-3.1-flash-lite", - "graphrag_mode": "local", - "generation_model": "gemini-3.1-flash-lite", - "judge_model": "gemini-3.1-flash-lite", - "k": 5, - "n_chunks": 735, - "metrics": { - "dense": { - "recall_at_k": 0.9557077625570777, - "precision_at_k": 0.25296803652968036, - "ndcg_at_k": 0.925337830224588, - "mrr": 0.9375951293759512, - "k": 5.0, - "n": 219.0, - "errors": 0.0, - "citation_accuracy": 0.6591157822809199, - "faithfulness": NaN, - "answer_relevancy": 0.8961740856796329, - "answer_n": 218.0, - "answer_errors": 1.0 - }, - "sparse_bm25": { - "recall_at_k": 0.910958904109589, - "precision_at_k": 0.22557077625570776, - "ndcg_at_k": 0.8458042240958054, - "mrr": 0.8533485540334855, - "k": 5.0, - "n": 219.0, - "errors": 0.0, - "citation_accuracy": 0.6397535313819718, - "faithfulness": NaN, - "answer_relevancy": 0.8616308094291161, - "answer_n": 218.0, - "answer_errors": 1.0 - }, - "hybrid_rrf": { - "recall_at_k": 0.9598173515981735, - "precision_at_k": 0.2484018264840183, - "ndcg_at_k": 0.9194175108157908, - "mrr": 0.9315068493150684, - "k": 5.0, - "n": 219.0, - "errors": 0.0, - "citation_accuracy": 0.6596276483373258, - "faithfulness": NaN, - "answer_relevancy": 0.8990548275588984, - "answer_n": 217.0, - "answer_errors": 2.0 - }, - "reranked": { - "recall_at_k": 0.8808219178082192, - "precision_at_k": 0.20547945205479454, - "ndcg_at_k": 0.8066878923032164, - "mrr": 0.8105022831050228, - "k": 5.0, - "n": 219.0, - "errors": 0.0, - "citation_accuracy": 0.6323496651837666, - "faithfulness": NaN, - "answer_relevancy": 0.8277884006811124, - "answer_n": 217.0, - "answer_errors": 2.0 - }, - "parent_child": { - "recall_at_k": 0.9557077625570777, - "precision_at_k": 0.28622526636225265, - "ndcg_at_k": 0.9265385800272361, - "mrr": 0.9406392694063926, - "k": 5.0, - "n": 219.0, - "errors": 0.0, - "citation_accuracy": 0.6481780451963938, - "faithfulness": NaN, - "answer_relevancy": 0.9001799851374453, - "answer_n": 218.0, - "answer_errors": 1.0 - } - } -} \ No newline at end of file diff --git a/experiments/20260724T014218Z/results.json b/experiments/20260724T014218Z/results.json deleted file mode 100644 index 561502b..0000000 --- a/experiments/20260724T014218Z/results.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "run_id": "20260724T014218Z", - "mode": "live", - "config_path": "configs/experiments/benchmark-v01.yaml", - "embedding": { - "model": "gemini-embedding-001", - "dimensions": 1536 - }, - "reranker_model": "cross-encoder/ms-marco-MiniLM-L-6-v2", - "contextualizer_model": "gemini-3.1-flash-lite", - "summarizer_model": "gemini-3.1-flash-lite", - "graphrag_llm_model": "gemini-3.1-flash-lite", - "graphrag_mode": "local", - "generation_model": "gemini-3.1-flash-lite", - "judge_model": "gemini-3.1-flash-lite", - "k": 5, - "n_chunks": 735, - "metrics": { - "dense": { - "recall_at_k": 0.9557077625570777, - "precision_at_k": 0.25296803652968036, - "ndcg_at_k": 0.925337830224588, - "mrr": 0.9375951293759512, - "k": 5.0, - "n": 219.0, - "errors": 0.0, - "citation_accuracy": 0.6591157822809199, - "faithfulness": NaN, - "answer_relevancy": 0.8961740856796329, - "answer_n": 218.0, - "answer_errors": 1.0 - }, - "sparse_bm25": { - "recall_at_k": 0.910958904109589, - "precision_at_k": 0.22557077625570776, - "ndcg_at_k": 0.8458042240958054, - "mrr": 0.8533485540334855, - "k": 5.0, - "n": 219.0, - "errors": 0.0, - "citation_accuracy": 0.6397535313819718, - "faithfulness": NaN, - "answer_relevancy": 0.8616308094291161, - "answer_n": 218.0, - "answer_errors": 1.0 - } - } -} \ No newline at end of file diff --git a/experiments/embeddings-ptbr/runs.jsonl b/experiments/embeddings-ptbr/runs.jsonl deleted file mode 100644 index d721890..0000000 --- a/experiments/embeddings-ptbr/runs.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"run_id": "20260722T181647Z", "model": "BAAI/bge-m3", "dimensions": 1024, "device": "cpu", "k": 5, "n_chunks": 465, "metrics": {"dense": {"recall_at_k": 0.7894736842105263, "precision_at_k": 0.2210526315789474, "ndcg_at_k": 0.745409029681905, "mrr": 0.5964912280701754, "k": 5.0, "n": 19.0}, "sparse": {"recall_at_k": 0.6578947368421053, "precision_at_k": 0.18947368421052632, "ndcg_at_k": 0.5960151369932086, "mrr": 0.43859649122807015, "k": 5.0, "n": 19.0}, "hybrid": {"recall_at_k": 0.7368421052631579, "precision_at_k": 0.2105263157894737, "ndcg_at_k": 0.696354006458703, "mrr": 0.5570175438596491, "k": 5.0, "n": 19.0}}} -{"run_id": "20260722T194201Z", "model": "gemini-embedding-001", "provider": "gemini", "dimensions": 1536, "device": null, "k": 5, "n_chunks": 465, "metrics": {"dense": {"recall_at_k": 0.9473684210526315, "precision_at_k": 0.26315789473684215, "ndcg_at_k": 0.9408723868115801, "mrr": 0.7894736842105263, "k": 5.0, "n": 19.0}, "sparse": {"recall_at_k": 0.6578947368421053, "precision_at_k": 0.18947368421052632, "ndcg_at_k": 0.5960151369932086, "mrr": 0.43859649122807015, "k": 5.0, "n": 19.0}, "hybrid": {"recall_at_k": 0.9473684210526315, "precision_at_k": 0.26315789473684215, "ndcg_at_k": 0.8704239107089652, "mrr": 0.6903508771929825, "k": 5.0, "n": 19.0}}} -{"run_id": "20260722T194822Z", "model": "gemini-embedding-2", "provider": "gemini", "dimensions": 1536, "device": null, "k": 5, "n_chunks": 465, "metrics": {"dense": {"recall_at_k": 0.7631578947368421, "precision_at_k": 0.2210526315789474, "ndcg_at_k": 0.75800197279428, "mrr": 0.6403508771929824, "k": 5.0, "n": 19.0}, "sparse": {"recall_at_k": 0.6578947368421053, "precision_at_k": 0.18947368421052632, "ndcg_at_k": 0.5960151369932086, "mrr": 0.43859649122807015, "k": 5.0, "n": 19.0}, "hybrid": {"recall_at_k": 0.8157894736842105, "precision_at_k": 0.23157894736842108, "ndcg_at_k": 0.7704690351607638, "mrr": 0.6289473684210526, "k": 5.0, "n": 19.0}}} diff --git a/scripts/build_split.py b/scripts/build_split.py new file mode 100644 index 0000000..2cc5fff --- /dev/null +++ b/scripts/build_split.py @@ -0,0 +1,40 @@ +"""Rebuild the versioned RegRAG-BR validation/test split deterministically.""" + +import json +from pathlib import Path + +from ragforge.evaluation.judgments import load_judgments +from ragforge.evaluation.split_builder import build_stratified_split + +ROOT = Path(__file__).resolve().parents[1] +JUDGMENTS_PATH = ROOT / "datasets" / "regrag-br" / "judgments.json" +SPLIT_PATH = ROOT / "datasets" / "regrag-br" / "split.json" +VALIDATION_RATIO = 0.15 +SEED = "regrag-br-v1" + + +def main() -> None: + """Write the deterministic stratified split to its canonical path.""" + judgments = load_judgments(JUDGMENTS_PATH) + dataset_version = json.loads(JUDGMENTS_PATH.read_text(encoding="utf-8"))["version"] + split = build_stratified_split( + judgments, + dataset_version=dataset_version, + validation_ratio=VALIDATION_RATIO, + seed=SEED, + ) + payload = { + "schema_version": split.schema_version, + "dataset_version": split.dataset_version, + "train": list(split.train), + "validation": list(split.validation), + "test": list(split.test), + } + SPLIT_PATH.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/src/ragforge/embeddings/caching.py b/src/ragforge/embeddings/caching.py new file mode 100644 index 0000000..1e10be7 --- /dev/null +++ b/src/ragforge/embeddings/caching.py @@ -0,0 +1,78 @@ +"""Persistent per-text embedding cache decorator.""" + +import dataclasses +import hashlib +import json + +from ragforge.adapters.llm_cache import LLMCache, cache_key +from ragforge.embeddings.identity import EmbeddingIdentity +from ragforge.embeddings.ports import EmbeddingModel + +_CACHE_SCHEMA_VERSION = 1 + + +class CachedEmbeddingModel: + """Cache embeddings by complete model identity and retrieval-text hash.""" + + def __init__( + self, + delegate: EmbeddingModel, + identity: EmbeddingIdentity, + cache: LLMCache, + ) -> None: + """Wrap ``delegate`` without changing its public embedding identity.""" + self._delegate = delegate + self._identity = identity + self._cache = cache + self.name = delegate.name + self.dimensions = delegate.dimensions + + def _key(self, text: str) -> str: + return cache_key( + kind="embedding", + schema_version=_CACHE_SCHEMA_VERSION, + identity=dataclasses.asdict(self._identity), + text_sha256=hashlib.sha256(text.encode()).hexdigest(), + ) + + def _deserialize(self, raw: str) -> list[float]: + payload: object = json.loads(raw) + if not isinstance(payload, list): + raise ValueError("cached embedding must be a JSON array") + if len(payload) != self.dimensions: + raise ValueError( + f"cached embedding has {len(payload)} dimensions; expected {self.dimensions}" + ) + if any(isinstance(value, bool) or not isinstance(value, (int, float)) for value in payload): + raise ValueError("cached embedding contains a non-numeric value") + return [float(value) for value in payload] + + def embed(self, texts: list[str]) -> list[list[float]]: + """Return cached vectors and batch only unique misses through the delegate.""" + if not texts: + return [] + + vectors_by_text: dict[str, list[float]] = {} + missing_texts: list[str] = [] + for text in dict.fromkeys(texts): + cached = self._cache.get(self._key(text)) + if cached is None: + missing_texts.append(text) + else: + vectors_by_text[text] = self._deserialize(cached) + + if missing_texts: + fresh_vectors = self._delegate.embed(missing_texts) + if len(fresh_vectors) != len(missing_texts): + raise ValueError( + f"embedder returned {len(fresh_vectors)} vectors for {len(missing_texts)} texts" + ) + for text, vector in zip(missing_texts, fresh_vectors, strict=True): + if len(vector) != self.dimensions: + raise ValueError( + f"embedder returned {len(vector)} dimensions; expected {self.dimensions}" + ) + vectors_by_text[text] = vector + self._cache.put(self._key(text), json.dumps(vector)) + + return [vectors_by_text[text] for text in texts] diff --git a/src/ragforge/embeddings/sentence_transformer_embedder.py b/src/ragforge/embeddings/sentence_transformer_embedder.py index 54823b1..892dab6 100644 --- a/src/ragforge/embeddings/sentence_transformer_embedder.py +++ b/src/ragforge/embeddings/sentence_transformer_embedder.py @@ -60,12 +60,18 @@ def embed(self, texts: list[str]) -> list[list[float]]: Embeddings are L2-normalized so a plain dot product is equivalent to cosine similarity, matching pgvector's cosine distance operator. + ``show_progress_bar=True`` renders a tqdm bar to stderr - the only + visibility into this call's progress while it runs, which matters + because CPU-bound encoding of a full corpus can take a long time + with no other output in between. Raises: EmbeddingError: If encoding fails. """ try: - vectors = self._model.encode(texts, convert_to_numpy=True, normalize_embeddings=True) + vectors = self._model.encode( + texts, convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=True + ) except Exception as exc: raise EmbeddingError(f"failed to encode {len(texts)} text(s): {exc}") from exc return cast(list[list[float]], vectors.tolist()) diff --git a/src/ragforge/evaluation/index_cache.py b/src/ragforge/evaluation/index_cache.py new file mode 100644 index 0000000..965d127 --- /dev/null +++ b/src/ragforge/evaluation/index_cache.py @@ -0,0 +1,138 @@ +"""Safe completion markers for reusable benchmark indexes.""" + +import json +from dataclasses import dataclass +from pathlib import Path + +from ragforge.domain.models import Chunk +from ragforge.evaluation.artifact_writer import write_atomic +from ragforge.evaluation.canonical_hash import canonical_json_hash + +_SCHEMA_VERSION = 1 + + +def index_fingerprint( + *, + stage: str, + index_namespace: str, + chunks: list[Chunk], + derivation_identity: str, +) -> str: + """Hash every input that determines one dense/sparse index's contents.""" + return canonical_json_hash( + { + "schema_version": _SCHEMA_VERSION, + "stage": stage, + "index_namespace": index_namespace, + "derivation_identity": derivation_identity, + "chunks": [ + { + "chunk_id": chunk.chunk_id, + "source_text": chunk.source_text, + "retrieval_text": chunk.retrieval_text, + "structural_ids": chunk.structural_ids, + "parent_id": chunk.parent_id, + "metadata": chunk.metadata, + } + for chunk in chunks + ], + } + ) + + +@dataclass(frozen=True, slots=True) +class IndexCompletion: + """One reusable index's validated completion identity.""" + + schema_version: int + stage: str + fingerprint: str + chunk_count: int + dense: bool + sparse: bool + + +class FileIndexRegistry: + """Atomically persist completion markers outside immutable run evidence.""" + + def __init__(self, root: Path) -> None: + """Create a registry rooted at a repository-local cache directory.""" + self._root = root + self._root.mkdir(parents=True, exist_ok=True) + + def _path(self, stage: str) -> Path: + return self._root / f"{stage}.json" + + def load(self, stage: str) -> IndexCompletion | None: + """Load a completion marker, or return None when none exists.""" + path = self._path(stage) + if not path.exists(): + return None + payload: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"index marker {path} must be a JSON object") + return IndexCompletion( + schema_version=int(payload["schema_version"]), + stage=str(payload["stage"]), + fingerprint=str(payload["fingerprint"]), + chunk_count=int(payload["chunk_count"]), + dense=bool(payload["dense"]), + sparse=bool(payload["sparse"]), + ) + + def matches( + self, + *, + stage: str, + fingerprint: str, + chunk_count: int, + dense: bool, + sparse: bool, + ) -> bool: + """Return whether the persisted marker exactly matches expected content.""" + marker = self.load(stage) + return marker == IndexCompletion( + schema_version=_SCHEMA_VERSION, + stage=stage, + fingerprint=fingerprint, + chunk_count=chunk_count, + dense=dense, + sparse=sparse, + ) + + def mark_complete( + self, + *, + stage: str, + fingerprint: str, + chunk_count: int, + dense: bool, + sparse: bool, + ) -> None: + """Atomically publish a marker only after every requested index is complete.""" + completion = IndexCompletion( + schema_version=_SCHEMA_VERSION, + stage=stage, + fingerprint=fingerprint, + chunk_count=chunk_count, + dense=dense, + sparse=sparse, + ) + write_atomic( + self._path(stage), + json.dumps( + { + "schema_version": completion.schema_version, + "stage": completion.stage, + "fingerprint": completion.fingerprint, + "chunk_count": completion.chunk_count, + "dense": completion.dense, + "sparse": completion.sparse, + }, + indent=2, + ), + ) + + def invalidate(self, stage: str) -> None: + """Remove a stale marker before rebuilding its external indexes.""" + self._path(stage).unlink(missing_ok=True) diff --git a/src/ragforge/evaluation/records.py b/src/ragforge/evaluation/records.py index 691642e..998164e 100644 --- a/src/ragforge/evaluation/records.py +++ b/src/ragforge/evaluation/records.py @@ -11,6 +11,7 @@ import json from dataclasses import dataclass from pathlib import Path +from typing import cast @dataclass(frozen=True, slots=True) @@ -112,8 +113,54 @@ def merge_question_records( def append_records_jsonl(path: Path, records: list[QuestionRecord]) -> None: - """Append each record as one JSON line to ``path`` (created if it does not exist).""" + """Append records not already present by strategy/question identity.""" + existing_keys = ( + {(record.strategy, record.question_id) for record in read_records_jsonl(path)} + if path.exists() + else set() + ) with path.open("a", encoding="utf-8") as handle: for record in records: + if (record.strategy, record.question_id) in existing_keys: + continue handle.write(json.dumps(record.to_json_dict(), ensure_ascii=False)) handle.write("\n") + + +def read_records_jsonl(path: Path) -> list[QuestionRecord]: + """Load stored records, keeping the latest unique strategy/question pair.""" + if not path.exists(): + return [] + records_by_key: dict[tuple[str, str], QuestionRecord] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + payload = cast(dict[str, object], json.loads(line)) + metrics_payload = cast(dict[str, object], payload["metrics"]) + metrics: dict[str, float] = {} + for key, value in metrics_payload.items(): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"record metric {key!r} must be numeric") + metrics[str(key)] = float(value) + record = QuestionRecord( + question_id=str(payload["question_id"]), + query_class=( + str(payload["query_class"]) if payload.get("query_class") is not None else None + ), + strategy=str(payload["strategy"]), + unanswerable=bool(payload["unanswerable"]), + retrieval_status=str(payload["retrieval_status"]), + generation_status=str(payload["generation_status"]), + judge_status=str(payload["judge_status"]), + retrieved_structural_ids=tuple( + str(value) for value in cast(list[object], payload["retrieved_structural_ids"]) + ), + answer_text=( + str(payload["answer_text"]) if payload.get("answer_text") is not None else None + ), + answer_citations=tuple( + str(value) for value in cast(list[object], payload["answer_citations"]) + ), + metrics=metrics, + errors=tuple(str(value) for value in cast(list[object], payload["errors"])), + ) + records_by_key[(record.strategy, record.question_id)] = record + return list(records_by_key.values()) diff --git a/src/ragforge/evaluation/run.py b/src/ragforge/evaluation/run.py index bf8f97b..97192c7 100644 --- a/src/ragforge/evaluation/run.py +++ b/src/ragforge/evaluation/run.py @@ -155,7 +155,7 @@ import dataclasses import json import shutil -import tempfile +import time from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path @@ -166,8 +166,9 @@ from opensearchpy import OpenSearch from ragforge.adapters.llm_cache import FileLLMCache -from ragforge.domain.models import Chunk +from ragforge.domain.models import Chunk, Judgment from ragforge.domain.protocols import RetrievalStrategy +from ragforge.embeddings.caching import CachedEmbeddingModel from ragforge.evaluation.artifact_writer import ( compute_checksums, write_atomic, @@ -177,6 +178,7 @@ from ragforge.evaluation.audit_ports import AuditResult from ragforge.evaluation.canonical_hash import canonical_json_hash from ragforge.evaluation.event_log import EventLog +from ragforge.evaluation.index_cache import FileIndexRegistry, index_fingerprint from ragforge.evaluation.index_namespace import derive_index_namespace from ragforge.evaluation.integrity import ( IntegrityError, @@ -187,21 +189,24 @@ from ragforge.evaluation.judgments import load_judgments from ragforge.evaluation.lineage_ports import GenerationLineage from ragforge.evaluation.manifest import load_corpus_manifest -from ragforge.evaluation.records import append_records_jsonl +from ragforge.evaluation.records import append_records_jsonl, read_records_jsonl from ragforge.evaluation.run_evidence import ( reject_if_evidence_dir_already_completed, write_question_artifacts, write_summaries, ) +from ragforge.evaluation.run_lock import BenchmarkAlreadyRunningError, BenchmarkRunLock from ragforge.evaluation.run_manifest import ( build_initial_manifest, finalize_manifest, resolve_git_sha, ) from ragforge.evaluation.run_reporting import ( + build_metric_breakdowns, build_run_record, format_answer_quality_table, format_results_table, + summarize_generation_usage, ) from ragforge.evaluation.run_strategies import ( _build_embedder, @@ -213,7 +218,7 @@ build_base_strategies, build_contextual_strategy, ) -from ragforge.evaluation.split import load_split +from ragforge.evaluation.split import Split, load_split from ragforge.generation.auditing_answer_generator import AuditingAnswerGenerator from ragforge.generation.gemini_answer_generator import GeminiAnswerGenerator from ragforge.generation.gemini_contextualizer import GeminiContextualizer @@ -273,6 +278,18 @@ _DEFAULT_ANSWER_QUALITY_WORKERS = 5 # Overridable via execution.gemini_max_in_flight (ADR-0014). _DEFAULT_GEMINI_MAX_IN_FLIGHT = 4 +_DEFAULT_EMBEDDING_CACHE_DIR = ".ragforge/cache/embeddings" +_DEFAULT_INDEX_CACHE_DIR = ".ragforge/cache/indexes" +_BASE_STRATEGY_LABELS = ( + "dense", + "sparse_bm25", + "hybrid_rrf", + "reranked", + "parent_child", +) +_SUPPORTED_STRATEGY_LABELS = frozenset( + (*_BASE_STRATEGY_LABELS, "contextual", "sac", "sac_contextual", "raptor", "graphrag") +) def parse_args() -> argparse.Namespace: @@ -306,6 +323,73 @@ def _reject_cache_mode(mode: str) -> None: ) +def _resolve_embedding_cache_dir(configured_path: str | None) -> Path: + """Resolve a repository-local persistent embedding cache directory. + + Configuration cannot redirect cached source-derived data outside the + repository. Absolute paths are accepted only when they still resolve + below ``ROOT``. + + Raises: + SystemExit: If the configured path escapes the repository. + """ + raw_path = Path(configured_path or _DEFAULT_EMBEDDING_CACHE_DIR) + resolved = (raw_path if raw_path.is_absolute() else ROOT / raw_path).resolve() + if not resolved.is_relative_to(ROOT): + raise SystemExit("execution.embedding_cache_dir must resolve inside the repository") + return resolved + + +def _resolve_index_cache_dir(configured_path: str | None) -> Path: + """Resolve the repository-local reusable-index marker directory.""" + raw_path = Path(configured_path or _DEFAULT_INDEX_CACHE_DIR) + resolved = (raw_path if raw_path.is_absolute() else ROOT / raw_path).resolve() + if not resolved.is_relative_to(ROOT): + raise SystemExit("execution.index_cache_dir must resolve inside the repository") + return resolved + + +def _validate_requested_strategies(raw_labels: object) -> tuple[str, ...]: + """Validate and preserve the configured benchmark strategy order. + + Raises: + SystemExit: If labels are absent, duplicated, non-string, or unsupported. + """ + if not isinstance(raw_labels, list) or not raw_labels: + raise SystemExit("strategies must be a non-empty list") + if any(not isinstance(label, str) for label in raw_labels): + raise SystemExit("every strategy label must be a string") + labels = tuple(raw_labels) + if len(labels) != len(set(labels)): + raise SystemExit("strategies must not contain duplicate labels") + unknown = sorted(set(labels) - _SUPPORTED_STRATEGY_LABELS) + if unknown: + raise SystemExit(f"unknown strategies: {', '.join(unknown)}") + return labels + + +def _select_split_judgments( + split: Split, + judgments: list[Judgment], + split_name: str, +) -> list[Judgment]: + """Return judgments in the declared split order. + + Raises: + SystemExit: If ``split_name`` is not a supported partition. + """ + split_ids_by_name = { + "train": split.train, + "validation": split.validation, + "test": split.test, + } + split_ids = split_ids_by_name.get(split_name) + if split_ids is None: + raise SystemExit("dataset.split must be one of train, validation, or test") + by_id = {judgment.question_id: judgment for judgment in judgments} + return [by_id[question_id] for question_id in split_ids] + + def _verify_resume_identity( previous: Mapping[str, object], index_namespace: str, @@ -349,18 +433,22 @@ def _verify_resume_identity( ) -def main() -> None: +def _run() -> None: """Index the real corpus with every strategy and score each against the golden set.""" args = parse_args() _reject_cache_mode(args.mode) args.config = args.config.resolve() config = yaml.safe_load(args.config.read_text(encoding="utf-8")) + requested_strategies = _validate_requested_strategies(config.get("strategies")) + requested_strategy_set = set(requested_strategies) + split_name = config["dataset"]["split"] top_k = config["retrieval"]["top_k"] rerank_pool = config["retrieval"]["rerank_pool"] embedding_provider = config["embedding"]["provider"] embedding_model = config["embedding"]["model"] embedding_dimensions = config["embedding"].get("dimensions") + embedding_device = config["embedding"].get("device") generation_model = config["generation"]["model"] judge_provider = config["judge"]["provider"] judge_model = config["judge"]["model"] @@ -371,6 +459,9 @@ def main() -> None: audit_provider = audit_config.get("provider", "openai") audit_model = audit_config.get("model") audit_reasoning_effort = audit_config.get("reasoning_effort", "medium") + pricing_config = config.get("pricing", {}).get("generation", {}) + generation_input_price = pricing_config.get("input_per_million_usd") + generation_output_price = pricing_config.get("output_per_million_usd") execution_config = config.get("execution", {}) answer_quality_workers = execution_config.get( "answer_quality_workers", _DEFAULT_ANSWER_QUALITY_WORKERS @@ -378,6 +469,8 @@ def main() -> None: gemini_max_in_flight = execution_config.get( "gemini_max_in_flight", _DEFAULT_GEMINI_MAX_IN_FLIGHT ) + embedding_cache_dir = _resolve_embedding_cache_dir(execution_config.get("embedding_cache_dir")) + index_cache_dir = _resolve_index_cache_dir(execution_config.get("index_cache_dir")) manifest = load_corpus_manifest(MANIFEST_PATH) split = load_split(SPLIT_PATH) @@ -389,6 +482,7 @@ def main() -> None: verify_split_integrity(split, judgments) except IntegrityError as exc: raise SystemExit(f"preflight integrity check failed:\n{exc}") from exc + judgments = _select_split_judgments(split, judgments, split_name) run_id = args.resume or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") run_dir = RESULTS_DIR / run_id @@ -416,9 +510,19 @@ def main() -> None: print(f"Loading embedding model {embedding_model} (provider={embedding_provider})...") embedder, embedding_identity = _build_embedder( - embedding_provider, embedding_model, embedding_dimensions, cache, gemini_max_in_flight + embedding_provider, + embedding_model, + embedding_dimensions, + cache, + gemini_max_in_flight, + device=embedding_device, ) embedding_identity_hash = canonical_json_hash(dataclasses.asdict(embedding_identity)) + embedder = CachedEmbeddingModel( + embedder, + embedding_identity, + FileLLMCache(embedding_cache_dir / embedding_identity_hash), + ) index_namespace = derive_index_namespace( manifest.content_hash, _CHUNKING_CONFIG_VERSION, @@ -430,6 +534,7 @@ def main() -> None: sac_table = f"bench_v01_sac_{index_namespace}" sac_contextual_table = f"bench_v01_sac_contextual_{index_namespace}" raptor_table = f"bench_v01_raptor_{index_namespace}" + index_registry = FileIndexRegistry(index_cache_dir / index_namespace) run_metrics: dict[str, dict[str, float]] = {} results_path = run_dir / "results.json" @@ -440,6 +545,7 @@ def main() -> None: ) run_metrics = previous["metrics"] print(f"Resuming {run_id}: {sorted(run_metrics)} already scored.") + pending_strategy_set = requested_strategy_set - set(run_metrics) print("Writing ADR-0017 evidence manifest and snapshots...") run_manifest = build_initial_manifest( @@ -455,7 +561,7 @@ def main() -> None: "judge": f"{judge_provider}/{judge_model}", "audit": f"{audit_provider}/{audit_model}" if audit_enabled else "disabled", }, - strategies=tuple(config["strategies"]), + strategies=requested_strategies, execution=dict(execution_config), ) write_atomic( @@ -513,13 +619,96 @@ def main() -> None: conn = psycopg.connect(DATABASE_URL) os_client = OpenSearch(hosts=["http://localhost:9200"], use_ssl=False, verify_certs=False) - graphrag_dir = Path(tempfile.mkdtemp(prefix="ragforge-bench-graphrag-")) - tables = [base_table, contextual_table, sac_table, sac_contextual_table, raptor_table] generation_lineage_by_strategy: dict[str, list[GenerationLineage]] = {} audit_results_by_strategy: dict[str, list[AuditResult]] = {} + stage_durations_seconds: dict[str, float] = {} + stage_started_at: dict[str, float] = {} + + def _stage_started(event_stage: str, stage: str) -> None: + """Record and emit the start of one benchmark stage.""" + key = f"{event_stage}:{stage}" + stage_started_at[key] = time.monotonic() + event_log.emit(event_stage, "started", {"stage": stage}) + + def _stage_completed(event_stage: str, stage: str) -> None: + """Record and emit successful completion with monotonic duration.""" + key = f"{event_stage}:{stage}" + duration = time.monotonic() - stage_started_at.pop(key) + stage_durations_seconds[key] = duration + event_log.emit( + event_stage, + "completed", + {"stage": stage, "duration_seconds": duration}, + ) + + def _ensure_reusable_index( + *, + stage: str, + chunks: list[Chunk], + dense_store: DenseChunkStore, + sparse_store: SparseChunkStore | None, + derivation_identity: str, + ) -> None: + """Reuse a complete exact index or rebuild and atomically mark it complete.""" + fingerprint = index_fingerprint( + stage=stage, + index_namespace=index_namespace, + chunks=chunks, + derivation_identity=derivation_identity, + ) + expected_ids = {chunk.chunk_id for chunk in chunks} + expects_sparse = sparse_store is not None + marker_matches = index_registry.matches( + stage=stage, + fingerprint=fingerprint, + chunk_count=len(chunks), + dense=True, + sparse=expects_sparse, + ) + stores_match = dense_store.has_exact_chunk_ids(expected_ids) and ( + sparse_store is None or sparse_store.has_exact_chunk_ids(expected_ids) + ) + if marker_matches and stores_match: + stage_durations_seconds[f"indexing:{stage}"] = 0.0 + event_log.emit( + "indexing", + "reused", + {"stage": stage, "fingerprint": fingerprint, "chunk_count": len(chunks)}, + ) + return + + index_registry.invalidate(stage) + _stage_started("indexing", stage) + dense_store.drop_schema() + if sparse_store is not None: + sparse_store.drop_index() + embeddings = embedder.embed([chunk.retrieval_text for chunk in chunks]) + dense_store.create_schema(dimensions=embedder.dimensions) + dense_store.upsert_chunks(chunks, embeddings) + dense_store.create_search_index() + if sparse_store is not None: + sparse_store.create_index() + sparse_store.index_chunks(chunks) + index_registry.mark_complete( + stage=stage, + fingerprint=fingerprint, + chunk_count=len(chunks), + dense=True, + sparse=expects_sparse, + ) + _stage_completed("indexing", stage) def _checkpoint() -> None: """Write the run record as computed so far - survives a later strategy crashing.""" + metric_breakdowns = build_metric_breakdowns( + read_records_jsonl(records_path), + judgments, + ) + generation_usage = summarize_generation_usage( + generation_lineage_by_strategy, + input_price_per_million_usd=generation_input_price, + output_price_per_million_usd=generation_output_price, + ) record = build_run_record( run_id=run_id, mode=args.mode, @@ -538,31 +727,41 @@ def _checkpoint() -> None: n_chunks=len(all_chunks), top_k=top_k, run_metrics=run_metrics, + metric_breakdowns=metric_breakdowns, + stage_durations_seconds=stage_durations_seconds, + generation_usage=generation_usage, ) results_path.write_text(json.dumps(record, ensure_ascii=False, indent=2)) def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: """Score ``strategy``, append its records.jsonl lines, then checkpoint results.json. - A no-op when ``label`` is already in ``run_metrics`` (--resume): the - stage's indexing above this call still runs regardless (contextual/ - RAPTOR construction isn't cache-wired in this increment), but the - expensive per-question generation+judge calls are skipped entirely - rather than merely cache-hit. + A no-op when ``label`` is already in ``run_metrics`` (--resume). + The outer orchestration also excludes completed labels from stage + construction, so neither indexing nor per-question calls are repeated. """ if label in run_metrics: print(f" skipping {label} (already scored, --resume)") return + started = time.monotonic() event_log.emit("strategy", "started", {"label": label}) - metrics, records, candidate_lineage = _evaluate( - strategy, - judgments, - generator, - judge_factory, - top_k, - answer_quality_workers, - embedding_identity_hash=embedding_identity_hash, - ) + try: + metrics, records, candidate_lineage = _evaluate( + strategy, + judgments, + generator, + judge_factory, + top_k, + answer_quality_workers, + embedding_identity_hash=embedding_identity_hash, + ) + except BaseException: + event_log.emit( + "strategy", + "failed", + {"label": label, "duration_seconds": time.monotonic() - started}, + ) + raise generation_lineage = base_generator.drain_generation_lineage() generation_lineage_by_strategy[label] = generation_lineage if auditing_generator is not None: @@ -572,152 +771,230 @@ def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: run_metrics[label] = metrics append_records_jsonl(records_path, records) write_question_artifacts(artifacts_dir, label, records, candidate_lineage) + write_summaries( + artifacts_dir, + run_metrics, + generation_lineage_by_strategy, + audit_results_by_strategy, + metric_breakdowns=build_metric_breakdowns( + read_records_jsonl(records_path), + judgments, + ), + generation_usage=summarize_generation_usage( + generation_lineage_by_strategy, + input_price_per_million_usd=generation_input_price, + output_price_per_million_usd=generation_output_price, + ), + ) _checkpoint() - event_log.emit("strategy", "completed", {"label": label, "n": metrics.get("n", 0.0)}) + duration = time.monotonic() - started + stage_durations_seconds[f"strategy:{label}"] = duration + event_log.emit( + "strategy", + "completed", + {"label": label, "n": metrics.get("n", 0.0), "duration_seconds": duration}, + ) try: - print("\n[1/6] Indexing the base chunks (dense + sparse)...") - event_log.emit("indexing", "started", {"stage": "base"}) - base_dense_store = DenseChunkStore(conn, table=base_table) - base_sparse_store = SparseChunkStore(os_client, index=base_table) - base_embeddings = embedder.embed([chunk.retrieval_text for chunk in all_chunks]) - base_dense_store.create_schema(dimensions=embedder.dimensions) - base_dense_store.upsert_chunks(all_chunks, base_embeddings) - base_sparse_store.create_index() - base_sparse_store.index_chunks(all_chunks) - - base_strategies = build_base_strategies( - base_dense_store, - base_sparse_store, - embedder, - CrossEncoderReranker(_RERANKER_MODEL), - rerank_pool, - ) - for label, strategy in base_strategies.items(): - print(f" evaluating {label}...") - _evaluate_and_checkpoint(label, strategy) - event_log.emit("indexing", "completed", {"stage": "base"}) - - print("\n[2/6] Building the Contextual Retrieval index (1 LLM call per chunk)...") - event_log.emit("indexing", "started", {"stage": "contextual"}) - contextualizer = GeminiContextualizer(_CONTEXTUALIZER_MODEL) - contextual_chunks_by_norm = { - norm_id: contextualize_chunks(full_text, chunks, contextualizer) - for norm_id, (full_text, chunks) in documents.items() - } - contextual_chunks = [ - chunk for chunks in contextual_chunks_by_norm.values() for chunk in chunks - ] - contextual_dense_store = DenseChunkStore(conn, table=contextual_table) - contextual_sparse_store = SparseChunkStore(os_client, index=contextual_table) - contextual_embeddings = embedder.embed( - [chunk.retrieval_text for chunk in contextual_chunks] - ) - contextual_dense_store.create_schema(dimensions=embedder.dimensions) - contextual_dense_store.upsert_chunks(contextual_chunks, contextual_embeddings) - contextual_sparse_store.create_index() - contextual_sparse_store.index_chunks(contextual_chunks) - contextual_strategy = build_contextual_strategy( - contextual_dense_store, contextual_sparse_store, embedder - ) - print(" evaluating contextual...") - _evaluate_and_checkpoint("contextual", contextual_strategy) - event_log.emit("indexing", "completed", {"stage": "contextual"}) - - print("\n[3/6] Building the SAC index (1 LLM call per document)...") - event_log.emit("indexing", "started", {"stage": "sac"}) - document_summarizer = GeminiDocumentSummarizer( - _DOCUMENT_SUMMARIZER_MODEL, cache=cache, max_in_flight=gemini_max_in_flight - ) - document_summaries = _summarize_documents( - documents, document_versions, document_summarizer, answer_quality_workers - ) - sac_chunks = [ - sac_chunk - for norm_id, (_, chunks) in documents.items() - for sac_chunk in apply_document_summary(document_summaries[norm_id], chunks) - ] - sac_dense_store = DenseChunkStore(conn, table=sac_table) - sac_embeddings = embedder.embed([chunk.retrieval_text for chunk in sac_chunks]) - sac_dense_store.create_schema(dimensions=embedder.dimensions) - sac_dense_store.upsert_chunks(sac_chunks, sac_embeddings) - sac_strategy = DenseRetrieval(sac_dense_store, embedder) - print(" evaluating sac...") - _evaluate_and_checkpoint("sac", sac_strategy) - event_log.emit("indexing", "completed", {"stage": "sac"}) - - print( - "\n[4/6] Building the SAC+Contextual index " - "(document summary + per-chunk context, no extra LLM calls)..." - ) - event_log.emit("indexing", "started", {"stage": "sac_contextual"}) - sac_contextual_chunks = [ - sac_chunk - for norm_id, chunks in contextual_chunks_by_norm.items() - for sac_chunk in apply_document_summary(document_summaries[norm_id], chunks) + requested_base_labels = [ + label for label in _BASE_STRATEGY_LABELS if label in pending_strategy_set ] - sac_contextual_dense_store = DenseChunkStore(conn, table=sac_contextual_table) - sac_contextual_embeddings = embedder.embed( - [chunk.retrieval_text for chunk in sac_contextual_chunks] - ) - sac_contextual_dense_store.create_schema(dimensions=embedder.dimensions) - sac_contextual_dense_store.upsert_chunks(sac_contextual_chunks, sac_contextual_embeddings) - sac_contextual_strategy = DenseRetrieval(sac_contextual_dense_store, embedder) - print(" evaluating sac_contextual...") - _evaluate_and_checkpoint("sac_contextual", sac_contextual_strategy) - event_log.emit("indexing", "completed", {"stage": "sac_contextual"}) - - print("\n[5/6] Building the RAPTOR tree (1 LLM call per group, per level, per document)...") - event_log.emit("indexing", "started", {"stage": "raptor"}) - summarizer = GeminiSummarizer(_SUMMARIZER_MODEL) - raptor_chunks: list[Chunk] = [] - for _, chunks in documents.values(): - raptor_chunks.extend(build_raptor_tree(chunks, summarizer)) - raptor_dense_store = DenseChunkStore(conn, table=raptor_table) - raptor_embeddings = embedder.embed([chunk.retrieval_text for chunk in raptor_chunks]) - raptor_dense_store.create_schema(dimensions=embedder.dimensions) - raptor_dense_store.upsert_chunks(raptor_chunks, raptor_embeddings) - raptor_strategy = DenseRetrieval(raptor_dense_store, embedder) - print(" evaluating raptor...") - _evaluate_and_checkpoint("raptor", raptor_strategy) - event_log.emit("indexing", "completed", {"stage": "raptor"}) - - print( - f"\n[6/6] Building the GraphRAG (LightRAG, mode={_GRAPHRAG_MODE}) index " - "(multiple LLM calls per chunk)..." - ) - event_log.emit("indexing", "started", {"stage": "graphrag"}) - rag = LightRAG( - working_dir=str(graphrag_dir), - embedding_func=build_gemini_embedding_func(embedder), - llm_model_func=build_gemini_llm_model_func(_GRAPHRAG_LLM_MODEL), - ) - asyncio.run(rag.initialize_storages()) - try: - for norm_id, (_, chunks) in documents.items(): - index_norm(rag, norm_id, chunks) - graphrag_strategy = GraphRagRetrieval( - rag, build_content_index(all_chunks), mode=_GRAPHRAG_MODE + if requested_base_labels: + print("\n[1/6] Indexing the base chunks (dense + sparse)...") + base_dense_store = DenseChunkStore(conn, table=base_table) + base_sparse_store = SparseChunkStore(os_client, index=base_table) + _ensure_reusable_index( + stage="base", + chunks=all_chunks, + dense_store=base_dense_store, + sparse_store=base_sparse_store, + derivation_identity=_RETRIEVAL_TEXT_SCHEMA_VERSION, + ) + + base_strategies = build_base_strategies( + base_dense_store, + base_sparse_store, + embedder, + CrossEncoderReranker(_RERANKER_MODEL), + rerank_pool, + ) + for label in requested_base_labels: + print(f" evaluating {label}...") + _evaluate_and_checkpoint(label, base_strategies[label]) + + contextual_chunks_by_norm: dict[str, list[Chunk]] = {} + needs_contextual_chunks = bool({"contextual", "sac_contextual"} & pending_strategy_set) + if needs_contextual_chunks: + print("\n[2/6] Building contextual retrieval text (1 LLM call per chunk)...") + _stage_started("contextualization", "contextual") + contextualizer = GeminiContextualizer(_CONTEXTUALIZER_MODEL) + contextual_chunks_by_norm = { + norm_id: contextualize_chunks(full_text, chunks, contextualizer) + for norm_id, (full_text, chunks) in documents.items() + } + _stage_completed("contextualization", "contextual") + + if "contextual" in pending_strategy_set: + contextual_chunks = [ + chunk for chunks in contextual_chunks_by_norm.values() for chunk in chunks + ] + contextual_dense_store = DenseChunkStore(conn, table=contextual_table) + contextual_sparse_store = SparseChunkStore(os_client, index=contextual_table) + _ensure_reusable_index( + stage="contextual", + chunks=contextual_chunks, + dense_store=contextual_dense_store, + sparse_store=contextual_sparse_store, + derivation_identity=_CONTEXTUALIZER_MODEL, + ) + contextual_strategy = build_contextual_strategy( + contextual_dense_store, contextual_sparse_store, embedder + ) + print(" evaluating contextual...") + _evaluate_and_checkpoint("contextual", contextual_strategy) + + document_summaries: dict[str, str] = {} + if {"sac", "sac_contextual"} & pending_strategy_set: + print("\n[3/6] Summarizing documents for SAC (1 LLM call per document)...") + _stage_started("summarization", "sac") + document_summarizer = GeminiDocumentSummarizer( + _DOCUMENT_SUMMARIZER_MODEL, cache=cache, max_in_flight=gemini_max_in_flight + ) + document_summaries = _summarize_documents( + documents, document_versions, document_summarizer, answer_quality_workers ) - print(" evaluating graphrag...") - _evaluate_and_checkpoint("graphrag", graphrag_strategy) - event_log.emit("indexing", "completed", {"stage": "graphrag"}) - finally: - asyncio.run(rag.finalize_storages()) + _stage_completed("summarization", "sac") + + if "sac" in pending_strategy_set: + sac_chunks = [ + sac_chunk + for norm_id, (_, chunks) in documents.items() + for sac_chunk in apply_document_summary(document_summaries[norm_id], chunks) + ] + sac_dense_store = DenseChunkStore(conn, table=sac_table) + _ensure_reusable_index( + stage="sac", + chunks=sac_chunks, + dense_store=sac_dense_store, + sparse_store=None, + derivation_identity=_DOCUMENT_SUMMARIZER_MODEL, + ) + sac_strategy = DenseRetrieval(sac_dense_store, embedder) + print(" evaluating sac...") + _evaluate_and_checkpoint("sac", sac_strategy) + + if "sac_contextual" in pending_strategy_set: + print("\n[4/6] Building the SAC+Contextual index...") + sac_contextual_chunks = [ + sac_chunk + for norm_id, chunks in contextual_chunks_by_norm.items() + for sac_chunk in apply_document_summary(document_summaries[norm_id], chunks) + ] + sac_contextual_dense_store = DenseChunkStore(conn, table=sac_contextual_table) + _ensure_reusable_index( + stage="sac_contextual", + chunks=sac_contextual_chunks, + dense_store=sac_contextual_dense_store, + sparse_store=None, + derivation_identity=f"{_DOCUMENT_SUMMARIZER_MODEL}+{_CONTEXTUALIZER_MODEL}", + ) + sac_contextual_strategy = DenseRetrieval(sac_contextual_dense_store, embedder) + print(" evaluating sac_contextual...") + _evaluate_and_checkpoint("sac_contextual", sac_contextual_strategy) + + if "raptor" in pending_strategy_set: + print( + "\n[5/6] Building the RAPTOR tree " + "(1 LLM call per group, per level, per document)..." + ) + summarizer = GeminiSummarizer(_SUMMARIZER_MODEL) + raptor_chunks: list[Chunk] = [] + for _, chunks in documents.values(): + raptor_chunks.extend(build_raptor_tree(chunks, summarizer)) + raptor_dense_store = DenseChunkStore(conn, table=raptor_table) + _ensure_reusable_index( + stage="raptor", + chunks=raptor_chunks, + dense_store=raptor_dense_store, + sparse_store=None, + derivation_identity=_SUMMARIZER_MODEL, + ) + raptor_strategy = DenseRetrieval(raptor_dense_store, embedder) + print(" evaluating raptor...") + _evaluate_and_checkpoint("raptor", raptor_strategy) + + if "graphrag" in pending_strategy_set: + print( + f"\n[6/6] Building the GraphRAG (LightRAG, mode={_GRAPHRAG_MODE}) index " + "(multiple LLM calls per chunk)..." + ) + graph_fingerprint = index_fingerprint( + stage="graphrag", + index_namespace=index_namespace, + chunks=all_chunks, + derivation_identity=f"{_GRAPHRAG_LLM_MODEL}:{_GRAPHRAG_MODE}", + ) + graphrag_dir = index_cache_dir / index_namespace / "graphrag-data" / graph_fingerprint + graph_reusable = ( + index_registry.matches( + stage="graphrag", + fingerprint=graph_fingerprint, + chunk_count=len(all_chunks), + dense=False, + sparse=False, + ) + and graphrag_dir.is_dir() + and any(graphrag_dir.iterdir()) + ) + if graph_reusable: + stage_durations_seconds["indexing:graphrag"] = 0.0 + event_log.emit( + "indexing", + "reused", + { + "stage": "graphrag", + "fingerprint": graph_fingerprint, + "chunk_count": len(all_chunks), + }, + ) + else: + index_registry.invalidate("graphrag") + shutil.rmtree(graphrag_dir, ignore_errors=True) + _stage_started("indexing", "graphrag") + graphrag_dir.mkdir(parents=True, exist_ok=True) + rag = LightRAG( + working_dir=str(graphrag_dir), + embedding_func=build_gemini_embedding_func(embedder), + llm_model_func=build_gemini_llm_model_func(_GRAPHRAG_LLM_MODEL), + ) + asyncio.run(rag.initialize_storages()) + try: + if not graph_reusable: + for norm_id, (_, chunks) in documents.items(): + index_norm(rag, norm_id, chunks) + index_registry.mark_complete( + stage="graphrag", + fingerprint=graph_fingerprint, + chunk_count=len(all_chunks), + dense=False, + sparse=False, + ) + _stage_completed("indexing", "graphrag") + graphrag_strategy = GraphRagRetrieval( + rag, build_content_index(all_chunks), mode=_GRAPHRAG_MODE + ) + print(" evaluating graphrag...") + _evaluate_and_checkpoint("graphrag", graphrag_strategy) + finally: + asyncio.run(rag.finalize_storages()) finally: - print("\nCleaning up disposable tables/indices...") + print("\nClosing reusable index connections...") conn.rollback() - with conn.cursor() as cur: - for table in tables: - cur.execute(f"DROP TABLE IF EXISTS {table}") - conn.commit() conn.close() - for table in tables: - os_client.indices.delete(index=table, ignore=[404]) - shutil.rmtree(graphrag_dir, ignore_errors=True) + os_client.close() - results_table = format_results_table(config["strategies"], run_metrics) - answer_quality_table = format_answer_quality_table(config["strategies"], run_metrics) + results_table = format_results_table(list(requested_strategies), run_metrics) + answer_quality_table = format_answer_quality_table(list(requested_strategies), run_metrics) print(f"\n{results_table}") print(f"\n{answer_quality_table}") @@ -725,8 +1002,20 @@ def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: print(f"\nRun record written to {run_dir.relative_to(ROOT)}/results.json") print("\nFinalizing ADR-0017 evidence directory...") - write_summaries( - artifacts_dir, run_metrics, generation_lineage_by_strategy, audit_results_by_strategy + merged_generation_usage = write_summaries( + artifacts_dir, + run_metrics, + generation_lineage_by_strategy, + audit_results_by_strategy, + metric_breakdowns=build_metric_breakdowns( + read_records_jsonl(records_path), + judgments, + ), + generation_usage=summarize_generation_usage( + generation_lineage_by_strategy, + input_price_per_million_usd=generation_input_price, + output_price_per_million_usd=generation_output_price, + ), ) report_record = build_run_record( run_id=run_id, @@ -746,6 +1035,16 @@ def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: n_chunks=len(all_chunks), top_k=top_k, run_metrics=run_metrics, + metric_breakdowns=build_metric_breakdowns( + read_records_jsonl(records_path), + judgments, + ), + stage_durations_seconds=stage_durations_seconds, + generation_usage=merged_generation_usage, + ) + write_atomic( + results_path, + json.dumps(report_record, ensure_ascii=False, indent=2), ) write_atomic( artifacts_dir / "report.json", json.dumps(report_record, ensure_ascii=False, indent=2) @@ -767,5 +1066,14 @@ def _evaluate_and_checkpoint(label: str, strategy: RetrievalStrategy) -> None: print(f"Evidence directory finalized at {artifacts_dir.relative_to(ROOT)}/") +def main() -> None: + """Run one benchmark at a time for the repository's shared indexes.""" + try: + with BenchmarkRunLock(ROOT / ".ragforge" / "benchmark.lock"): + _run() + except BenchmarkAlreadyRunningError as exc: + raise SystemExit(str(exc)) from exc + + if __name__ == "__main__": main() diff --git a/src/ragforge/evaluation/run_evidence.py b/src/ragforge/evaluation/run_evidence.py index 63ff0ba..be3f45c 100644 --- a/src/ragforge/evaluation/run_evidence.py +++ b/src/ragforge/evaluation/run_evidence.py @@ -16,6 +16,16 @@ from ragforge.evaluation.records import QuestionRecord +def _load_json_object(path: Path) -> dict[str, object]: + """Load an existing summary object, or return an empty mapping.""" + if not path.exists(): + return {} + payload: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"summary {path} must contain a JSON object") + return {str(key): value for key, value in payload.items()} + + def reject_if_evidence_dir_already_completed(artifacts_dir: Path) -> None: """Fail closed if ``artifacts_dir``'s manifest.json already says status="completed" (ADR-0017). @@ -75,7 +85,10 @@ def write_summaries( run_metrics: dict[str, dict[str, float]], generation_lineage_by_strategy: dict[str, list[GenerationLineage]], audit_results_by_strategy: dict[str, list[AuditResult]], -) -> None: + *, + metric_breakdowns: dict[str, object] | None = None, + generation_usage: dict[str, dict[str, float | int | None]] | None = None, +) -> dict[str, dict[str, float | int | None]]: """Write ``summaries/retrieval.json``, ``summaries/generation.json``, ``summaries/audit.json``. The same per-strategy aggregates already computed for @@ -86,25 +99,48 @@ def write_summaries( artifacts_dir / "summaries" / "retrieval.json", json.dumps(run_metrics, ensure_ascii=False, indent=2), ) + generation_path = artifacts_dir / "summaries" / "generation.json" + generation_payload = _load_json_object(generation_path) + generation_payload.update( + { + label: [dataclasses.asdict(entry) for entry in entries] + for label, entries in generation_lineage_by_strategy.items() + } + ) + write_atomic( + generation_path, + json.dumps(generation_payload, ensure_ascii=False, indent=2), + ) + audit_path = artifacts_dir / "summaries" / "audit.json" + audit_payload = _load_json_object(audit_path) + audit_payload.update( + { + label: compute_audit_report(results) + for label, results in audit_results_by_strategy.items() + } + ) write_atomic( - artifacts_dir / "summaries" / "generation.json", - json.dumps( - { - label: [dataclasses.asdict(entry) for entry in entries] - for label, entries in generation_lineage_by_strategy.items() - }, - ensure_ascii=False, - indent=2, - ), + audit_path, + json.dumps(audit_payload, ensure_ascii=False, indent=2), ) write_atomic( - artifacts_dir / "summaries" / "audit.json", - json.dumps( - { - label: compute_audit_report(results) - for label, results in audit_results_by_strategy.items() - }, - ensure_ascii=False, - indent=2, - ), + artifacts_dir / "summaries" / "breakdowns.json", + json.dumps(metric_breakdowns or {}, ensure_ascii=False, indent=2), ) + usage_path = artifacts_dir / "summaries" / "usage.json" + usage_payload = _load_json_object(usage_path) + usage_payload.update(generation_usage or {}) + write_atomic(usage_path, json.dumps(usage_payload, ensure_ascii=False, indent=2)) + merged_usage: dict[str, dict[str, float | int | None]] = {} + for strategy, raw_summary in usage_payload.items(): + if not isinstance(raw_summary, dict): + raise ValueError(f"usage summary for {strategy!r} must be an object") + normalized: dict[str, float | int | None] = {} + for key, value in raw_summary.items(): + if value is not None and ( + isinstance(value, bool) or not isinstance(value, (int, float)) + ): + raise ValueError(f"usage value {strategy}.{key} must be numeric or null") + normalized[str(key)] = value + merged_usage[strategy] = normalized + return merged_usage diff --git a/src/ragforge/evaluation/run_lock.py b/src/ragforge/evaluation/run_lock.py new file mode 100644 index 0000000..7dd4abe --- /dev/null +++ b/src/ragforge/evaluation/run_lock.py @@ -0,0 +1,63 @@ +"""Cross-process lock preventing concurrent mutation of shared benchmark indexes.""" + +import fcntl +import os +from pathlib import Path +from types import TracebackType +from typing import TextIO + + +class BenchmarkAlreadyRunningError(RuntimeError): + """Raised when another benchmark process owns the repository lock.""" + + +class BenchmarkRunLock: + """Hold an advisory repository lock for the complete benchmark process.""" + + def __init__(self, path: Path) -> None: + """Bind the lock to ``path`` without acquiring it yet.""" + self._path = path + self._handle: TextIO | None = None + + def acquire(self) -> None: + """Acquire the lock without waiting. + + Raises: + BenchmarkAlreadyRunningError: If another process owns the lock. + """ + self._path.parent.mkdir(parents=True, exist_ok=True) + handle = self._path.open("a+", encoding="utf-8") + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + handle.close() + raise BenchmarkAlreadyRunningError( + "another benchmark is already running for this repository" + ) from exc + handle.seek(0) + handle.truncate() + handle.write(f"{os.getpid()}\n") + handle.flush() + self._handle = handle + + def release(self) -> None: + """Release the lock when currently held.""" + if self._handle is None: + return + fcntl.flock(self._handle.fileno(), fcntl.LOCK_UN) + self._handle.close() + self._handle = None + + def __enter__(self) -> "BenchmarkRunLock": + """Acquire and return this lock.""" + self.acquire() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Always release the lock.""" + self.release() diff --git a/src/ragforge/evaluation/run_reporting.py b/src/ragforge/evaluation/run_reporting.py index eec45d4..89737f7 100644 --- a/src/ragforge/evaluation/run_reporting.py +++ b/src/ragforge/evaluation/run_reporting.py @@ -6,8 +6,14 @@ returns a string or a dict. """ +from collections import defaultdict +from statistics import mean + +from ragforge.domain.models import Judgment from ragforge.embeddings.identity import EmbeddingIdentity +from ragforge.evaluation.lineage_ports import GenerationLineage from ragforge.evaluation.ragas_judge import ABSTENTION_PROMPT_VERSION +from ragforge.evaluation.records import QuestionRecord # No reranker model has been chosen via a dedicated comparison (unlike the # embedding model, ADR-0005); a placeholder, not a data-driven winner - @@ -19,6 +25,85 @@ _GRAPHRAG_MODE = "local" +def _aggregate_record_group(records: list[QuestionRecord]) -> dict[str, object]: + """Aggregate every available numeric metric with explicit coverage.""" + values_by_metric: dict[str, list[float]] = defaultdict(list) + for record in records: + for metric, value in record.metrics.items(): + values_by_metric[metric].append(value) + return { + "selected": len(records), + "retrieval_succeeded": sum(record.retrieval_status == "succeeded" for record in records), + "answer_succeeded": sum(record.generation_status == "succeeded" for record in records), + "metrics": { + metric: {"mean": mean(values), "n": len(values)} + for metric, values in sorted(values_by_metric.items()) + }, + } + + +def build_metric_breakdowns( + records: list[QuestionRecord], + judgments: list[Judgment], +) -> dict[str, object]: + """Aggregate per-strategy metrics by query class and relevant document.""" + judgment_by_id = {judgment.question_id: judgment for judgment in judgments} + by_class: dict[tuple[str, str], list[QuestionRecord]] = defaultdict(list) + by_document: dict[tuple[str, str], list[QuestionRecord]] = defaultdict(list) + for record in records: + by_class[(record.strategy, record.query_class or "unknown")].append(record) + judgment = judgment_by_id[record.question_id] + documents = ( + {judged.ref.norm for judged in judgment.relevant_refs} + if judgment.relevant_refs + else {"__unanswerable__"} + ) + for document in documents: + by_document[(record.strategy, document)].append(record) + + def _render( + grouped: dict[tuple[str, str], list[QuestionRecord]], + ) -> dict[str, dict[str, object]]: + rendered: dict[str, dict[str, object]] = {} + for (strategy, dimension), grouped_records in sorted(grouped.items()): + rendered.setdefault(strategy, {})[dimension] = _aggregate_record_group(grouped_records) + return rendered + + return { + "by_query_class": _render(by_class), + "by_document": _render(by_document), + } + + +def summarize_generation_usage( + lineage_by_strategy: dict[str, list[GenerationLineage]], + *, + input_price_per_million_usd: float | None, + output_price_per_million_usd: float | None, +) -> dict[str, dict[str, float | int | None]]: + """Summarize generation-only latency, tokens, cache hits, and optional cost.""" + summaries: dict[str, dict[str, float | int | None]] = {} + for strategy, entries in sorted(lineage_by_strategy.items()): + prompt_tokens = sum(entry.prompt_tokens or 0 for entry in entries) + completion_tokens = sum(entry.completion_tokens or 0 for entry in entries) + estimated_cost: float | None = None + if input_price_per_million_usd is not None and output_price_per_million_usd is not None: + estimated_cost = ( + prompt_tokens * input_price_per_million_usd + + completion_tokens * output_price_per_million_usd + ) / 1_000_000 + summaries[strategy] = { + "calls": len(entries), + "latency_seconds": sum(entry.latency_seconds for entry in entries), + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": sum(entry.total_tokens or 0 for entry in entries), + "cache_hits": sum(entry.cache_hit for entry in entries), + "estimated_cost_usd": estimated_cost, + } + return summaries + + def format_results_table( strategy_labels: list[str], run_metrics: dict[str, dict[str, float]] ) -> str: @@ -82,6 +167,9 @@ def build_run_record( n_chunks: int, top_k: int, run_metrics: dict[str, dict[str, float]], + metric_breakdowns: dict[str, object] | None = None, + stage_durations_seconds: dict[str, float] | None = None, + generation_usage: dict[str, dict[str, float | int | None]] | None = None, ) -> dict[str, object]: """Assemble the JSON-serializable run record written to experiments//results.json. @@ -127,5 +215,8 @@ def build_run_record( "k": top_k, "n_chunks": n_chunks, "metrics": run_metrics, + "metric_breakdowns": metric_breakdowns or {}, + "stage_durations_seconds": stage_durations_seconds or {}, + "generation_usage": generation_usage or {}, "records_path": "records.jsonl", } diff --git a/src/ragforge/evaluation/run_strategies.py b/src/ragforge/evaluation/run_strategies.py index 941474a..57d859f 100644 --- a/src/ragforge/evaluation/run_strategies.py +++ b/src/ragforge/evaluation/run_strategies.py @@ -135,23 +135,28 @@ def _build_embedder( dimensions: int | None, cache: LLMCache | None, gemini_max_in_flight: int, + device: str | None = None, ) -> tuple[EmbeddingModel, EmbeddingIdentity]: """Construct the configured embedding provider and its identity (ADR-0013). ``provider: local`` is the operational default: no credentials needed, via SentenceTransformerEmbedder - ``dimensions`` is ignored since the model reports its own, and it has no hosted-call cache/limiter to wire - (there's no network call to skip). ``provider: gemini`` is the optional - hosted comparator, via GoogleGeminiEmbedder with ``dimensions`` - requesting a truncated (Matryoshka) size, ``cache`` (ADR-0004) consulted - per text, and ``gemini_max_in_flight`` bounding concurrent calls - (ADR-0014). Never a silent fallback between the two. + (there's no network call to skip). ``device`` (cpu/mps/cuda) is a pure + execution knob, not part of ``EmbeddingIdentity`` - it selects hardware, + not a different embedding space, and defaults to sentence-transformers' + own auto-detection (``None``) when omitted, matching prior behavior. + ``provider: gemini`` is the optional hosted comparator, via + GoogleGeminiEmbedder with ``dimensions`` requesting a truncated + (Matryoshka) size, ``cache`` (ADR-0004) consulted per text, and + ``gemini_max_in_flight`` bounding concurrent calls (ADR-0014). Never a + silent fallback between the two. Raises: SystemExit: If ``provider`` isn't one of "local"/"gemini". """ if provider == "local": - local_embedder = SentenceTransformerEmbedder(model) + local_embedder = SentenceTransformerEmbedder(model, device=device) identity = EmbeddingIdentity( provider="local", model=model, diff --git a/src/ragforge/evaluation/split_builder.py b/src/ragforge/evaluation/split_builder.py new file mode 100644 index 0000000..ccba752 --- /dev/null +++ b/src/ragforge/evaluation/split_builder.py @@ -0,0 +1,70 @@ +"""Deterministic stratified split construction for RegRAG-BR (ADR-0003).""" + +import hashlib +from collections import defaultdict + +from ragforge.domain.models import Judgment +from ragforge.evaluation.split import Split + + +def build_stratified_split( + judgments: list[Judgment], + *, + dataset_version: str, + validation_ratio: float = 0.15, + seed: str = "regrag-br-v1", +) -> Split: + """Partition judgments into deterministic validation and test sets. + + Selection is stratified by query class. Within each class, question IDs + are ordered by a SHA-256 score derived from the declared seed, avoiding + dependence on Python's randomized hash or a mutable PRNG implementation. + At least one question from every non-empty class is assigned to each + partition. + + Args: + judgments: Complete curated judgment collection. + dataset_version: Version copied into the resulting split artifact. + validation_ratio: Target fraction reserved for router development. + seed: Versioned selection seed. + + Raises: + ValueError: If inputs cannot produce a valid two-way split. + """ + if not judgments: + raise ValueError("judgments must not be empty") + if not 0.0 < validation_ratio < 1.0: + raise ValueError("validation_ratio must be between 0 and 1") + + by_class: dict[str, list[str]] = defaultdict(list) + original_order = [judgment.question_id for judgment in judgments] + for judgment in judgments: + if judgment.query.query_class is None: + raise ValueError(f"judgment {judgment.question_id!r} has no query class") + by_class[judgment.query.query_class.value].append(judgment.question_id) + + validation_ids: set[str] = set() + for query_class, question_ids in sorted(by_class.items()): + if len(question_ids) < 2: + raise ValueError(f"query class {query_class!r} needs at least two questions") + validation_count = max(1, round(len(question_ids) * validation_ratio)) + validation_count = min(validation_count, len(question_ids) - 1) + ranked_ids = sorted( + question_ids, + key=lambda question_id: hashlib.sha256( + f"{seed}:{query_class}:{question_id}".encode() + ).hexdigest(), + ) + validation_ids.update(ranked_ids[:validation_count]) + + validation = tuple( + question_id for question_id in original_order if question_id in validation_ids + ) + test = tuple(question_id for question_id in original_order if question_id not in validation_ids) + return Split( + schema_version=1, + dataset_version=dataset_version, + train=(), + validation=validation, + test=test, + ) diff --git a/src/ragforge/retrieval/dense/store.py b/src/ragforge/retrieval/dense/store.py index 48f0d5e..439cee2 100644 --- a/src/ragforge/retrieval/dense/store.py +++ b/src/ragforge/retrieval/dense/store.py @@ -27,9 +27,8 @@ def __init__(self, conn: psycopg.Connection, table: str = _DEFAULT_TABLE) -> Non self._table = table def create_schema(self, dimensions: int) -> None: - """Create the chunk table and its HNSW cosine index if they don't exist.""" + """Create the chunk table if it doesn't exist.""" table = sql.Identifier(self._table) - index = sql.Identifier(f"{self._table}_embedding_idx") with self._conn.cursor() as cur: cur.execute( sql.SQL( @@ -43,6 +42,13 @@ def create_schema(self, dimensions: int) -> None: ")" ).format(table=table, dimensions=sql.Literal(dimensions)) ) + self._conn.commit() + + def create_search_index(self) -> None: + """Create the HNSW index after bulk data loading.""" + table = sql.Identifier(self._table) + index = sql.Identifier(f"{self._table}_embedding_idx") + with self._conn.cursor() as cur: cur.execute( sql.SQL( "CREATE INDEX IF NOT EXISTS {index} ON {table} " @@ -51,6 +57,23 @@ def create_schema(self, dimensions: int) -> None: ) self._conn.commit() + def drop_schema(self) -> None: + """Drop the bound table and its dependent index.""" + with self._conn.cursor() as cur: + cur.execute(sql.SQL("DROP TABLE IF EXISTS {}").format(sql.Identifier(self._table))) + self._conn.commit() + + def has_exact_chunk_ids(self, expected_ids: set[str]) -> bool: + """Return whether the table exists and contains exactly ``expected_ids``.""" + with self._conn.cursor() as cur: + cur.execute("SELECT to_regclass(%s)", (self._table,)) + registration = cur.fetchone() + if registration is None or registration[0] is None: + return False + cur.execute(sql.SQL("SELECT chunk_id FROM {}").format(sql.Identifier(self._table))) + actual_ids = {row[0] for row in cur.fetchall()} + return actual_ids == expected_ids + def upsert_chunks(self, chunks: list[Chunk], embeddings: list[list[float]]) -> None: """Insert or update chunks and their embeddings, keyed by chunk_id. @@ -73,19 +96,19 @@ def upsert_chunks(self, chunks: list[Chunk], embeddings: list[list[float]]) -> N "metadata = EXCLUDED.metadata, " "embedding = EXCLUDED.embedding" ).format(table=table) + values = [ + ( + chunk.chunk_id, + chunk.source_text, + list(chunk.structural_ids), + chunk.parent_id, + Jsonb(chunk.metadata), + embedding, + ) + for chunk, embedding in zip(chunks, embeddings, strict=True) + ] with self._conn.cursor() as cur: - for chunk, embedding in zip(chunks, embeddings, strict=True): - cur.execute( - statement, - ( - chunk.chunk_id, - chunk.source_text, - list(chunk.structural_ids), - chunk.parent_id, - Jsonb(chunk.metadata), - embedding, - ), - ) + cur.executemany(statement, values) self._conn.commit() def get(self, chunk_id: str) -> Chunk | None: diff --git a/src/ragforge/retrieval/sparse/store.py b/src/ragforge/retrieval/sparse/store.py index 11f0bd3..eec077d 100644 --- a/src/ragforge/retrieval/sparse/store.py +++ b/src/ragforge/retrieval/sparse/store.py @@ -44,6 +44,22 @@ def create_index(self) -> None: }, ) + def drop_index(self) -> None: + """Delete the bound index if it exists.""" + self._client.indices.delete(index=self._index, ignore=[404]) + + def has_exact_chunk_ids(self, expected_ids: set[str]) -> bool: + """Return whether the index exists and contains exactly ``expected_ids``.""" + if not self._client.indices.exists(index=self._index): + return False + if self._client.count(index=self._index)["count"] != len(expected_ids): + return False + response = self._client.mget( + index=self._index, + body={"ids": sorted(expected_ids)}, + ) + return all(document.get("found", False) for document in response["docs"]) + def index_chunks(self, chunks: list[Chunk]) -> None: """Bulk-index chunks, keyed by chunk_id so re-indexing overwrites in place.""" actions = ( diff --git a/tests/unit/test_benchmark_run.py b/tests/unit/test_benchmark_run.py index e749dc0..1b7dc43 100644 --- a/tests/unit/test_benchmark_run.py +++ b/tests/unit/test_benchmark_run.py @@ -29,7 +29,13 @@ ModelIdentity, ) from ragforge.evaluation.manifest import load_corpus_manifest -from ragforge.evaluation.run import MANIFEST_PATH +from ragforge.evaluation.run import ( + MANIFEST_PATH, + _resolve_embedding_cache_dir, + _resolve_index_cache_dir, + _select_split_judgments, + _validate_requested_strategies, +) from ragforge.evaluation.run_reporting import ( build_run_record, format_answer_quality_table, @@ -41,6 +47,7 @@ build_base_strategies, build_contextual_strategy, ) +from ragforge.evaluation.split import Split from ragforge.retrieval.reranked.strategy import RerankedRetrieval @@ -61,6 +68,61 @@ def test_reject_cache_mode_allows_live() -> None: _reject_cache_mode("live") # must not raise +def test_resolve_embedding_cache_dir_rejects_a_path_outside_repository() -> None: + """Embedding cache data cannot be redirected outside the repository.""" + with pytest.raises(SystemExit, match="inside the repository"): + _resolve_embedding_cache_dir("../outside") + + +def test_resolve_index_cache_dir_rejects_a_path_outside_repository() -> None: + """Index completion metadata cannot be redirected outside the repository.""" + with pytest.raises(SystemExit, match="inside the repository"): + _resolve_index_cache_dir("../outside") + + +def test_validate_requested_strategies_preserves_valid_order() -> None: + """A valid subset remains in declarative configuration order.""" + assert _validate_requested_strategies(["graphrag", "dense"]) == ("graphrag", "dense") + + +@pytest.mark.parametrize( + "labels", + [ + [], + ["dense", "dense"], + ["unknown"], + ["dense", 1], + ], +) +def test_validate_requested_strategies_rejects_invalid_labels(labels: object) -> None: + """Invalid strategy declarations fail before infrastructure is touched.""" + with pytest.raises(SystemExit): + _validate_requested_strategies(labels) + + +def test_select_split_judgments_uses_declared_partition_order() -> None: + """The runner evaluates only the configured split, in artifact order.""" + judgments = [ + Judgment( + question_id=question_id, + query=Query(text=question_id), + relevant_refs=(), + ) + for question_id in ("q1", "q2", "q3") + ] + split = Split( + schema_version=1, + dataset_version="1", + train=(), + validation=("q2",), + test=("q3", "q1"), + ) + + selected = _select_split_judgments(split, judgments, "test") + + assert [judgment.question_id for judgment in selected] == ["q3", "q1"] + + class _FakeEmbedder: name = "fake-embedder" dimensions = 3 @@ -463,10 +525,11 @@ def test_evaluate_populates_candidate_lineage_when_embedding_identity_hash_is_gi class _FakeSentenceTransformerEmbedder: """Stands in for SentenceTransformerEmbedder - no real model load.""" - def __init__(self, model_name: str) -> None: + def __init__(self, model_name: str, device: str | None = None) -> None: self.name = model_name self.dimensions = 1024 self.revision = "main" + self.device = device def embed(self, texts: list[str]) -> list[list[float]]: return [[0.0] * self.dimensions for _ in texts] @@ -512,6 +575,26 @@ def test_build_embedder_constructs_a_local_embedder_without_credentials( assert identity.runtime == "local" +def test_build_embedder_forwards_device_override_to_the_local_embedder( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An explicit device (e.g. "cpu") reaches SentenceTransformerEmbedder, not silently dropped.""" + calls: list[tuple[str, str | None]] = [] + + class _RecordingEmbedder(_FakeSentenceTransformerEmbedder): + def __init__(self, model_name: str, device: str | None = None) -> None: + calls.append((model_name, device)) + super().__init__(model_name, device=device) + + monkeypatch.setattr(run_strategies, "SentenceTransformerEmbedder", _RecordingEmbedder) + + run_strategies._build_embedder( + "local", "Qwen/Qwen3-Embedding-0.6B", None, None, 4, device="cpu" + ) + + assert calls == [("Qwen/Qwen3-Embedding-0.6B", "cpu")] + + def test_build_embedder_constructs_a_gemini_embedder_with_output_dimensionality( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_embedding_cache.py b/tests/unit/test_embedding_cache.py new file mode 100644 index 0000000..ca49e8b --- /dev/null +++ b/tests/unit/test_embedding_cache.py @@ -0,0 +1,85 @@ +"""Tests for persistent per-text embedding caching.""" + +from pathlib import Path + +import pytest + +from ragforge.adapters.llm_cache import FileLLMCache +from ragforge.embeddings.caching import CachedEmbeddingModel +from ragforge.embeddings.identity import NO_QUERY_INSTRUCTION_HASH, EmbeddingIdentity + + +class _CountingEmbedder: + name = "counting" + dimensions = 2 + + def __init__(self) -> None: + self.calls: list[list[str]] = [] + + def embed(self, texts: list[str]) -> list[list[float]]: + self.calls.append(texts) + return [[float(len(text)), 1.0] for text in texts] + + +def _identity(model: str = "model-a") -> EmbeddingIdentity: + return EmbeddingIdentity( + provider="local", + model=model, + revision="revision", + dimensions=2, + normalize=True, + query_instruction_hash=NO_QUERY_INSTRUCTION_HASH, + runtime="local", + ) + + +def test_embedding_cache_batches_unique_misses_and_preserves_order(tmp_path: Path) -> None: + """Repeated texts are encoded once while output order and duplicates remain intact.""" + delegate = _CountingEmbedder() + embedder = CachedEmbeddingModel(delegate, _identity(), FileLLMCache(tmp_path)) + + vectors = embedder.embed(["aa", "bbb", "aa"]) + + assert delegate.calls == [["aa", "bbb"]] + assert vectors == [[2.0, 1.0], [3.0, 1.0], [2.0, 1.0]] + + +def test_embedding_cache_survives_a_new_wrapper_instance(tmp_path: Path) -> None: + """A later run with the same identity reuses the file-backed vector.""" + first_delegate = _CountingEmbedder() + CachedEmbeddingModel(first_delegate, _identity(), FileLLMCache(tmp_path)).embed(["text"]) + second_delegate = _CountingEmbedder() + + vectors = CachedEmbeddingModel(second_delegate, _identity(), FileLLMCache(tmp_path)).embed( + ["text"] + ) + + assert vectors == [[4.0, 1.0]] + assert second_delegate.calls == [] + + +def test_embedding_cache_isolated_by_embedding_identity(tmp_path: Path) -> None: + """Changing the model identity cannot reuse vectors from another embedding space.""" + first_delegate = _CountingEmbedder() + CachedEmbeddingModel(first_delegate, _identity("model-a"), FileLLMCache(tmp_path)).embed( + ["text"] + ) + second_delegate = _CountingEmbedder() + + CachedEmbeddingModel(second_delegate, _identity("model-b"), FileLLMCache(tmp_path)).embed( + ["text"] + ) + + assert second_delegate.calls == [["text"]] + + +def test_embedding_cache_rejects_a_corrupt_vector(tmp_path: Path) -> None: + """A cached vector with the wrong dimension fails closed.""" + delegate = _CountingEmbedder() + embedder = CachedEmbeddingModel(delegate, _identity(), FileLLMCache(tmp_path)) + embedder.embed(["text"]) + [cache_file] = tmp_path.iterdir() + cache_file.write_text('{"value": "[1.0]"}', encoding="utf-8") + + with pytest.raises(ValueError, match="dimensions"): + embedder.embed(["text"]) diff --git a/tests/unit/test_index_cache.py b/tests/unit/test_index_cache.py new file mode 100644 index 0000000..c8832b7 --- /dev/null +++ b/tests/unit/test_index_cache.py @@ -0,0 +1,82 @@ +"""Tests for safe reusable-index completion markers.""" + +from pathlib import Path + +from ragforge.domain.models import Chunk +from ragforge.evaluation.index_cache import FileIndexRegistry, index_fingerprint + + +def _chunk(text: str = "text") -> Chunk: + return Chunk( + chunk_id="c1", + source_text=text, + retrieval_text=text, + structural_ids=("N::art-1",), + ) + + +def test_index_fingerprint_changes_with_retrieval_text_or_derivation() -> None: + """Synthetic text and producing model both participate in cache identity.""" + base = index_fingerprint( + stage="contextual", + index_namespace="namespace", + chunks=[_chunk("one")], + derivation_identity="model-a", + ) + changed_text = index_fingerprint( + stage="contextual", + index_namespace="namespace", + chunks=[_chunk("two")], + derivation_identity="model-a", + ) + changed_model = index_fingerprint( + stage="contextual", + index_namespace="namespace", + chunks=[_chunk("one")], + derivation_identity="model-b", + ) + + assert len({base, changed_text, changed_model}) == 3 + + +def test_registry_matches_only_an_exact_completed_index(tmp_path: Path) -> None: + """A marker is reusable only with the same fingerprint, count, and store set.""" + registry = FileIndexRegistry(tmp_path) + registry.mark_complete( + stage="base", + fingerprint="fingerprint", + chunk_count=10, + dense=True, + sparse=True, + ) + + assert registry.matches( + stage="base", + fingerprint="fingerprint", + chunk_count=10, + dense=True, + sparse=True, + ) + assert not registry.matches( + stage="base", + fingerprint="different", + chunk_count=10, + dense=True, + sparse=True, + ) + + +def test_registry_invalidate_removes_completion(tmp_path: Path) -> None: + """A rebuild invalidates the old marker before touching external state.""" + registry = FileIndexRegistry(tmp_path) + registry.mark_complete( + stage="base", + fingerprint="fingerprint", + chunk_count=1, + dense=True, + sparse=False, + ) + + registry.invalidate("base") + + assert registry.load("base") is None diff --git a/tests/unit/test_records.py b/tests/unit/test_records.py index a8e2c85..4fed0bd 100644 --- a/tests/unit/test_records.py +++ b/tests/unit/test_records.py @@ -94,3 +94,14 @@ def test_append_records_jsonl_writes_one_json_line_per_record(tmp_path: Path) -> assert parsed[0]["strategy"] == "dense" assert parsed[1]["strategy"] == "sparse_bm25" assert parsed[0]["metrics"] == {"recall_at_k": 1.0, "citation_accuracy": 1.0} + + +def test_append_records_jsonl_is_idempotent_for_a_resumed_strategy(tmp_path: Path) -> None: + """Resume cannot duplicate an already persisted strategy/question record.""" + path = tmp_path / "records.jsonl" + records = merge_question_records("dense", [_retrieval("q1")], [_answer("q1")]) + + append_records_jsonl(path, records) + append_records_jsonl(path, records) + + assert len(path.read_text(encoding="utf-8").splitlines()) == 1 diff --git a/tests/unit/test_run_evidence_summaries.py b/tests/unit/test_run_evidence_summaries.py new file mode 100644 index 0000000..dce936c --- /dev/null +++ b/tests/unit/test_run_evidence_summaries.py @@ -0,0 +1,47 @@ +"""Tests for resumable benchmark summary persistence.""" + +from pathlib import Path + +from ragforge.evaluation.lineage_ports import GenerationLineage +from ragforge.evaluation.run_evidence import write_summaries + + +def _lineage(model: str) -> GenerationLineage: + return GenerationLineage( + provider="gemini", + model=model, + prompt_hash="prompt", + input_chunk_ids=(), + input_source_hashes=(), + answer_hash="answer", + parsed_citations=(), + prompt_tokens=1, + completion_tokens=2, + total_tokens=3, + latency_seconds=0.5, + cache_hit=False, + ) + + +def test_write_summaries_merges_prior_strategies_on_resume(tmp_path: Path) -> None: + """A resumed run preserves summaries checkpointed by earlier strategies.""" + write_summaries( + tmp_path, + {"dense": {"n": 1.0}}, + {"dense": [_lineage("model")]}, + {}, + generation_usage={"dense": {"calls": 1}}, + ) + + merged_usage = write_summaries( + tmp_path, + {"dense": {"n": 1.0}, "sparse": {"n": 1.0}}, + {"sparse": [_lineage("model")]}, + {}, + generation_usage={"sparse": {"calls": 1}}, + ) + + assert set(merged_usage) == {"dense", "sparse"} + generation = (tmp_path / "summaries" / "generation.json").read_text(encoding="utf-8") + assert '"dense"' in generation + assert '"sparse"' in generation diff --git a/tests/unit/test_run_lock.py b/tests/unit/test_run_lock.py new file mode 100644 index 0000000..5242f4c --- /dev/null +++ b/tests/unit/test_run_lock.py @@ -0,0 +1,27 @@ +"""Tests for the cross-process benchmark lock.""" + +from pathlib import Path + +import pytest + +from ragforge.evaluation.run_lock import BenchmarkAlreadyRunningError, BenchmarkRunLock + + +def test_benchmark_lock_rejects_a_second_owner(tmp_path: Path) -> None: + """Only one benchmark can mutate shared indexes in a repository.""" + first = BenchmarkRunLock(tmp_path / "benchmark.lock") + second = BenchmarkRunLock(tmp_path / "benchmark.lock") + + with first, pytest.raises(BenchmarkAlreadyRunningError): + second.acquire() + + +def test_benchmark_lock_can_be_reacquired_after_release(tmp_path: Path) -> None: + """A completed process leaves the lock reusable.""" + path = tmp_path / "benchmark.lock" + + with BenchmarkRunLock(path): + pass + + with BenchmarkRunLock(path): + pass diff --git a/tests/unit/test_run_reporting_breakdowns.py b/tests/unit/test_run_reporting_breakdowns.py new file mode 100644 index 0000000..c51bfa2 --- /dev/null +++ b/tests/unit/test_run_reporting_breakdowns.py @@ -0,0 +1,103 @@ +"""Tests for benchmark dimension and usage reporting.""" + +from ragforge.domain.models import ( + JudgedRef, + Judgment, + Query, + QueryClass, + RelevanceGrade, + StructuralRef, +) +from ragforge.evaluation.lineage_ports import GenerationLineage +from ragforge.evaluation.records import QuestionRecord +from ragforge.evaluation.run_reporting import ( + build_metric_breakdowns, + summarize_generation_usage, +) + + +def _record(question_id: str, query_class: str, score: float) -> QuestionRecord: + return QuestionRecord( + question_id=question_id, + query_class=query_class, + strategy="dense", + unanswerable=False, + retrieval_status="succeeded", + generation_status="succeeded", + judge_status="succeeded", + retrieved_structural_ids=(), + answer_text="answer", + answer_citations=(), + metrics={"recall_at_k": score}, + errors=(), + ) + + +def test_metric_breakdowns_report_class_document_mean_and_coverage() -> None: + """Breakdowns preserve denominators instead of publishing bare averages.""" + judgments = [ + Judgment( + question_id="q1", + query=Query("one", QueryClass.EXACT_FACTUAL), + relevant_refs=( + JudgedRef( + StructuralRef("NORM-1", "art-1"), + RelevanceGrade.RELEVANT, + ), + ), + ), + Judgment( + question_id="q2", + query=Query("two", QueryClass.EXACT_FACTUAL), + relevant_refs=( + JudgedRef( + StructuralRef("NORM-1", "art-2"), + RelevanceGrade.RELEVANT, + ), + ), + ), + ] + + breakdowns = build_metric_breakdowns( + [ + _record("q1", "exact_factual", 1.0), + _record("q2", "exact_factual", 0.0), + ], + judgments, + ) + + by_class = breakdowns["by_query_class"] + assert isinstance(by_class, dict) + exact = by_class["dense"]["exact_factual"] + assert exact["selected"] == 2 + assert exact["metrics"]["recall_at_k"] == {"mean": 0.5, "n": 2} + by_document = breakdowns["by_document"] + assert isinstance(by_document, dict) + assert by_document["dense"]["NORM-1"]["selected"] == 2 + + +def test_generation_usage_computes_configured_cost() -> None: + """Cost is calculated only from explicit per-million-token prices.""" + lineage = GenerationLineage( + provider="gemini", + model="model", + prompt_hash="hash", + input_chunk_ids=(), + input_source_hashes=(), + answer_hash="answer", + parsed_citations=(), + prompt_tokens=1_000_000, + completion_tokens=500_000, + total_tokens=1_500_000, + latency_seconds=2.5, + cache_hit=False, + ) + + usage = summarize_generation_usage( + {"dense": [lineage]}, + input_price_per_million_usd=1.0, + output_price_per_million_usd=2.0, + ) + + assert usage["dense"]["estimated_cost_usd"] == 2.0 + assert usage["dense"]["latency_seconds"] == 2.5 diff --git a/tests/unit/test_sentence_transformer_embedder.py b/tests/unit/test_sentence_transformer_embedder.py index c32a038..44685be 100644 --- a/tests/unit/test_sentence_transformer_embedder.py +++ b/tests/unit/test_sentence_transformer_embedder.py @@ -19,6 +19,7 @@ class _FakeSentenceTransformer: """Records the arguments it was constructed with; encodes deterministically.""" last_kwargs: ClassVar[dict[str, Any]] = {} + last_encode_kwargs: ClassVar[dict[str, Any]] = {} def __init__( self, model_name: str, device: str | None = None, revision: str | None = None @@ -33,7 +34,15 @@ def __init__( def get_embedding_dimension(self) -> int: return self._dimensions - def encode(self, texts: list[str], convert_to_numpy: bool, normalize_embeddings: bool) -> Any: + def encode( + self, + texts: list[str], + convert_to_numpy: bool, + normalize_embeddings: bool, + show_progress_bar: bool = False, + ) -> Any: + _FakeSentenceTransformer.last_encode_kwargs = {"show_progress_bar": show_progress_bar} + class _FakeArray: def __init__(self, rows: int, cols: int) -> None: self._rows, self._cols = rows, cols @@ -86,6 +95,15 @@ def test_embed_returns_one_vector_per_text_matching_the_model_dimension() -> Non assert all(len(vector) == 3 for vector in vectors) +def test_embed_requests_a_progress_bar_so_a_long_cpu_encode_stays_observable() -> None: + """embed() asks encode() to render a progress bar - the only visibility into a slow CPU run.""" + embedder = SentenceTransformerEmbedder("Qwen/Qwen3-Embedding-0.6B") + + embedder.embed(["a", "b"]) + + assert _FakeSentenceTransformer.last_encode_kwargs["show_progress_bar"] is True + + def test_load_failure_is_translated_to_embedding_error(monkeypatch: pytest.MonkeyPatch) -> None: """A failure inside SentenceTransformer's constructor becomes an EmbeddingError.""" diff --git a/tests/unit/test_split.py b/tests/unit/test_split.py index 5aaf536..4dbefef 100644 --- a/tests/unit/test_split.py +++ b/tests/unit/test_split.py @@ -45,11 +45,12 @@ def test_all_ids_concatenates_every_partition_in_order(tmp_path: Path) -> None: assert split.all_ids == ("q1", "q2", "q3", "q4") -def test_load_real_split_selects_all_230_questions_as_test() -> None: - """The real split puts every currently-curated question in the test partition.""" +def test_load_real_split_reserves_validation_without_losing_questions() -> None: + """The real split partitions all 230 curated questions without overlap.""" split = load_split(REAL_SPLIT_PATH) assert split.train == () - assert split.validation == () - assert len(split.test) == 230 - assert len(set(split.test)) == 230 + assert len(split.validation) == 36 + assert len(split.test) == 194 + assert len(set(split.all_ids)) == 230 + assert not set(split.validation) & set(split.test) diff --git a/tests/unit/test_split_builder.py b/tests/unit/test_split_builder.py new file mode 100644 index 0000000..8aa7086 --- /dev/null +++ b/tests/unit/test_split_builder.py @@ -0,0 +1,59 @@ +"""Tests for deterministic stratified RegRAG-BR split construction.""" + +from ragforge.domain.models import Judgment, Query, QueryClass +from ragforge.evaluation.split_builder import build_stratified_split + + +def _judgment(question_id: str, query_class: QueryClass) -> Judgment: + return Judgment( + question_id=question_id, + query=Query(text=question_id, query_class=query_class), + relevant_refs=(), + ) + + +def test_build_stratified_split_is_deterministic_and_disjoint() -> None: + """The same seed yields a stable partition with complete, disjoint coverage.""" + judgments = [_judgment(f"exact-{index}", QueryClass.EXACT_FACTUAL) for index in range(10)] + [ + _judgment(f"global-{index}", QueryClass.GLOBAL) for index in range(10) + ] + + first = build_stratified_split(judgments, dataset_version="1", seed="stable") + second = build_stratified_split(judgments, dataset_version="1", seed="stable") + + assert first == second + assert not set(first.validation) & set(first.test) + assert set(first.validation) | set(first.test) == { + judgment.question_id for judgment in judgments + } + + +def test_build_stratified_split_reserves_each_class_in_both_partitions() -> None: + """Every query class remains represented in validation and test.""" + judgments = [ + _judgment(f"{query_class.value}-{index}", query_class) + for query_class in QueryClass + for index in range(10) + ] + + split = build_stratified_split(judgments, dataset_version="1") + + for query_class in QueryClass: + prefix = f"{query_class.value}-" + assert any(question_id.startswith(prefix) for question_id in split.validation) + assert any(question_id.startswith(prefix) for question_id in split.test) + + +def test_build_stratified_split_rejects_invalid_ratio() -> None: + """A non-fractional validation ratio is rejected.""" + judgments = [ + _judgment("q1", QueryClass.EXACT_FACTUAL), + _judgment("q2", QueryClass.EXACT_FACTUAL), + ] + + try: + build_stratified_split(judgments, dataset_version="1", validation_ratio=1.0) + except ValueError as exc: + assert "validation_ratio" in str(exc) + else: + raise AssertionError("expected ValueError")