From 452aa64ef77ecf0e0ca699fe9564e6339cbe9407 Mon Sep 17 00:00:00 2001 From: zhangtao <982450544@qq.com> Date: Sun, 20 Sep 2026 11:56:13 +0800 Subject: [PATCH] feat(llm-wiki): add auditable raw-to-wiki compilation Co-Authored-By: Claude Sonnet 5 --- .env.example | 8 + api/manager/knowledge/v1/knowledge.proto | 171 ++++++ cmd/ongrid/main.go | 58 +- ...0260918100000_add_llm_wiki_tables.down.sql | 6 + .../20260918100000_add_llm_wiki_tables.up.sql | 100 ++++ deploy/.env.example | 5 + deploy/README.md | 3 +- deploy/docker-compose.yml | 7 + deploy/install/.env.example | 7 + deploy/install/README.md | 6 +- deploy/install/data-permissions.sh | 3 + deploy/install/docker-compose.yml | 7 + deploy/install/install.sh | 2 + docs/adr/ADR-033-llm-wiki-architecture.md | 118 ++++ docs/design/HLD-002-llm-wiki.md | 376 +++++++++++++ docs/rfc/RFC-004-llm-wiki.md | 305 ++++++++++ .../manager/biz/aiops/chatruntime/worker.go | 15 +- .../aiops/chatruntime/worker_prologue_test.go | 105 ++++ .../aiops/tools/query_knowledge_basetool.go | 66 ++- internal/manager/biz/knowledge/ingest_test.go | 63 +++ .../manager/biz/knowledge/list_docs_test.go | 103 ++++ .../biz/knowledge/llm_wiki/artifact.go | 116 ++++ .../manager/biz/knowledge/llm_wiki/build.go | 532 ++++++++++++++++++ .../biz/knowledge/llm_wiki/build_test.go | 120 ++++ .../biz/knowledge/llm_wiki/contract.go | 112 ++++ .../manager/biz/knowledge/llm_wiki/corpus.go | 218 +++++++ .../biz/knowledge/llm_wiki/corpus_test.go | 69 +++ .../manager/biz/knowledge/llm_wiki/delete.go | 77 +++ .../biz/knowledge/llm_wiki/evidence.go | 134 +++++ .../biz/knowledge/llm_wiki/evidence_test.go | 23 + .../biz/knowledge/llm_wiki/filestore.go | 215 +++++++ .../biz/knowledge/llm_wiki/filestore_test.go | 31 + .../manager/biz/knowledge/llm_wiki/job.go | 294 ++++++++++ .../biz/knowledge/llm_wiki/job_test.go | 477 ++++++++++++++++ .../biz/knowledge/llm_wiki/language.go | 8 + .../biz/knowledge/llm_wiki/language_test.go | 25 + .../manager/biz/knowledge/llm_wiki/llm.go | 109 ++++ .../biz/knowledge/llm_wiki/llm_test.go | 131 +++++ .../biz/knowledge/llm_wiki/markdown.go | 127 +++++ .../biz/knowledge/llm_wiki/markdown_test.go | 93 +++ .../manager/biz/knowledge/llm_wiki/paths.go | 188 +++++++ .../biz/knowledge/llm_wiki/paths_test.go | 38 ++ .../manager/biz/knowledge/llm_wiki/plan.go | 255 +++++++++ .../biz/knowledge/llm_wiki/plan_test.go | 57 ++ .../manager/biz/knowledge/llm_wiki/raw.go | 95 ++++ .../manager/biz/knowledge/llm_wiki/search.go | 33 ++ .../manager/biz/knowledge/llm_wiki/source.go | 146 +++++ .../biz/knowledge/llm_wiki/summarize.go | 178 ++++++ .../biz/knowledge/llm_wiki/summarize_test.go | 60 ++ .../manager/biz/knowledge/llm_wiki/tree.go | 509 +++++++++++++++++ .../manager/biz/knowledge/llm_wiki/usecase.go | 192 +++++++ .../manager/biz/knowledge/llm_wiki/wiki.go | 76 +++ internal/manager/biz/knowledge/usecase.go | 224 ++++---- .../data/knowledge/llm_wiki/index/index.go | 420 ++++++++++++++ .../knowledge/llm_wiki/index/index_test.go | 438 ++++++++++++++ .../llm_wiki/mysql_integration_test.go | 99 ++++ .../manager/data/knowledge/llm_wiki/store.go | 55 ++ .../knowledge/llm_wiki/store/build_repo.go | 227 ++++++++ .../data/knowledge/llm_wiki/store/job_repo.go | 220 ++++++++ .../data/knowledge/llm_wiki/store/repo.go | 92 +++ .../knowledge/llm_wiki/store/repo_test.go | 285 ++++++++++ .../knowledge/llm_wiki/store/source_repo.go | 173 ++++++ .../llm_wiki/store/tree_integration_test.go | 162 ++++++ .../llm_wiki/store/usage_integration_test.go | 98 ++++ .../data/knowledge/llm_wiki/store_test.go | 43 ++ .../manager/model/knowledge/llm_wiki/build.go | 52 ++ .../manager/model/knowledge/llm_wiki/index.go | 16 + .../manager/model/knowledge/llm_wiki/model.go | 84 +++ internal/manager/model/knowledge/model.go | 50 +- .../manager/server/knowledge/guards_test.go | 40 +- internal/manager/server/knowledge/http.go | 54 +- .../manager/server/knowledge/http_test.go | 51 +- .../manager/server/knowledge/llmwiki_http.go | 347 ++++++++++++ .../server/knowledge/llmwiki_http_test.go | 249 ++++++++ .../service/knowledge/hybrid_search.go | 129 +++++ .../service/knowledge/hybrid_search_test.go | 253 +++++++++ .../service/knowledge/llmwiki_usage.go | 76 +++ .../service/knowledge/llmwiki_usage_test.go | 55 ++ internal/pkg/config/config.go | 11 + internal/pkg/docextract/docx2md_test.go | 18 + internal/pkg/docextract/extract.go | 95 +++- internal/pkg/docextract/pdf2md.go | 36 +- internal/pkg/docextract/pdf2md_test.go | 53 +- internal/pkg/llm/client.go | 199 ++++++- internal/pkg/llm/client_test.go | 129 +++++ internal/pkg/prom/manager_metrics.go | 113 ++++ scripts/test-upgrade-data-permissions.sh | 7 +- web/src/api/knowledge.ts | 165 +++++- web/src/features/llm-wiki/LLMWikiJobs.tsx | 94 ++++ web/src/features/llm-wiki/LLMWikiPane.tsx | 265 +++++++++ .../features/llm-wiki/LLMWikiTree.test.tsx | 84 +++ web/src/features/llm-wiki/LLMWikiTree.tsx | 152 +++++ web/src/features/llm-wiki/LLMWikiViewer.tsx | 177 ++++++ web/src/features/llm-wiki/api.ts | 12 + web/src/features/llm-wiki/types.ts | 6 + web/src/pages/Knowledge.test.tsx | 289 +++++++++- web/src/pages/Knowledge.tsx | 73 ++- 97 files changed, 12012 insertions(+), 230 deletions(-) create mode 100644 db/migrations/20260918100000_add_llm_wiki_tables.down.sql create mode 100644 db/migrations/20260918100000_add_llm_wiki_tables.up.sql create mode 100644 docs/adr/ADR-033-llm-wiki-architecture.md create mode 100644 docs/design/HLD-002-llm-wiki.md create mode 100644 docs/rfc/RFC-004-llm-wiki.md create mode 100644 internal/manager/biz/aiops/chatruntime/worker_prologue_test.go create mode 100644 internal/manager/biz/knowledge/ingest_test.go create mode 100644 internal/manager/biz/knowledge/list_docs_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/artifact.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/build.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/build_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/contract.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/corpus.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/corpus_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/delete.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/evidence.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/evidence_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/filestore.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/filestore_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/job.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/job_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/language.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/language_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/llm.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/llm_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/markdown.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/markdown_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/paths.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/paths_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/plan.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/plan_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/raw.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/search.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/source.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/summarize.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/summarize_test.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/tree.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/usecase.go create mode 100644 internal/manager/biz/knowledge/llm_wiki/wiki.go create mode 100644 internal/manager/data/knowledge/llm_wiki/index/index.go create mode 100644 internal/manager/data/knowledge/llm_wiki/index/index_test.go create mode 100644 internal/manager/data/knowledge/llm_wiki/mysql_integration_test.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store/build_repo.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store/job_repo.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store/repo.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store/repo_test.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store/source_repo.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store/tree_integration_test.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store/usage_integration_test.go create mode 100644 internal/manager/data/knowledge/llm_wiki/store_test.go create mode 100644 internal/manager/model/knowledge/llm_wiki/build.go create mode 100644 internal/manager/model/knowledge/llm_wiki/index.go create mode 100644 internal/manager/model/knowledge/llm_wiki/model.go create mode 100644 internal/manager/server/knowledge/llmwiki_http.go create mode 100644 internal/manager/server/knowledge/llmwiki_http_test.go create mode 100644 internal/manager/service/knowledge/hybrid_search.go create mode 100644 internal/manager/service/knowledge/hybrid_search_test.go create mode 100644 internal/manager/service/knowledge/llmwiki_usage.go create mode 100644 internal/manager/service/knowledge/llmwiki_usage_test.go create mode 100644 web/src/features/llm-wiki/LLMWikiJobs.tsx create mode 100644 web/src/features/llm-wiki/LLMWikiPane.tsx create mode 100644 web/src/features/llm-wiki/LLMWikiTree.test.tsx create mode 100644 web/src/features/llm-wiki/LLMWikiTree.tsx create mode 100644 web/src/features/llm-wiki/LLMWikiViewer.tsx create mode 100644 web/src/features/llm-wiki/api.ts create mode 100644 web/src/features/llm-wiki/types.ts diff --git a/.env.example b/.env.example index a57767267..41f32a460 100644 --- a/.env.example +++ b/.env.example @@ -38,6 +38,14 @@ ONGRID_OPENAI_API_KEY= ONGRID_OPENAI_MODEL=gpt-4o ONGRID_OPENAI_BASE_URL= +# --- LLM Wiki (auditable Raw -> compiled Wiki knowledge layer) --- +# Vector search writes to the shared Qdrant (ONGRID_QDRANT_URL, default +# http://qdrant:6333) once ONGRID_EMBEDDING_* is configured; without an +# embedding key the Wiki runs lexical-only. +ONGRID_LLM_WIKI_ENABLED=true +ONGRID_LLM_WIKI_DIR=/var/lib/ongrid/llm-wiki +ONGRID_LLM_WIKI_TIMEOUT_SECONDS=600 + # --- Cloud-side Prometheus (ADR-009) --- # When ONGRID_PROM_ENABLED=true the manager forwards push_prom_samples to # remote_write, and the AI agent gains a query_promql tool. Leave false diff --git a/api/manager/knowledge/v1/knowledge.proto b/api/manager/knowledge/v1/knowledge.proto index 1467d06e7..f6e669f56 100644 --- a/api/manager/knowledge/v1/knowledge.proto +++ b/api/manager/knowledge/v1/knowledge.proto @@ -1,4 +1,5 @@ syntax = "proto3"; + package ongrid.manager.knowledge.v1; option go_package = "github.com/ongridio/ongrid/api/gen/manager/knowledge/v1;knowledgev1"; @@ -24,3 +25,173 @@ message UpdateRepositoryRequest { string branch = 2; string description = 3; } + +// --------------------------------------------------------------------------- +// LLM Wiki +// +// The compiled Wiki: mirrored raw sources, the generated pages of the active +// build and the compile jobs that produce them. Every route answers the shared +// `{code, message, data}` HTTP envelope, so only the `data` payload is modelled +// here. Node ids are opaque to clients: they encode the layer and the +// store-relative path and must be sent back unchanged. +// --------------------------------------------------------------------------- + +// One mirrored raw source. +message LLMWikiSource { + string id = 1; + string source_key = 2; + string source_type = 3; + string raw_path = 4; + string current_version_id = 5; + string status = 6; // pending | succeeded | stale | failed + string updated_at = 7; +} + +// One compile job. A tenant has at most one pending or running job. +message LLMWikiJob { + string id = 1; + string status = 2; // pending | running | succeeded | skipped | failed | cancelled + string stage = 3; + string error = 4; + string created_at = 5; + string updated_at = 6; +} + +// One node of the Wiki tree: a raw file, a generated page or a folder. +message LLMWikiTreeNode { + string id = 1; + string parent_id = 2; + string layer = 3; // raw | wiki + string kind = 4; // folder | file + string name = 5; + string relative_path = 6; + bool has_children = 7; + int64 child_count = 8; + int64 document_count = 9; + string source_id = 10; + string page_id = 11; + string page_type = 12; + string status = 13; + string updated_at = 14; +} + +// One published Wiki page. +message LLMWikiSearchHit { + string layer = 1; + string page_type = 2; + string page_id = 3; + string source_version_id = 4; + string title = 5; + string preview = 6; + double score = 7; + string matched_node = 8; +} + +// GET /v1/knowledge/llm-wiki/tree?layer=&parent_id= +message LLMWikiTreeRequest { + string layer = 1; // raw | wiki | empty for both layers + string parent_id = 2; // empty for the layer root +} +message LLMWikiTreeResponse { + repeated LLMWikiTreeNode items = 1; + int64 total = 2; + int64 document_count = 3; +} + +// GET /v1/knowledge/llm-wiki/nodes/{id} answers with the tree node flattened +// next to the body and its metadata, because the Go DTO embeds TreeNode. +// Metadata is a JSON object in `metadata_json`. +message LLMWikiNodeRequest { + string id = 1; +} +message LLMWikiNodeResponse { + string id = 1; + string parent_id = 2; + string layer = 3; + string kind = 4; + string name = 5; + string relative_path = 6; + bool has_children = 7; + int64 child_count = 8; + int64 document_count = 9; + string source_id = 10; + string page_id = 11; + string page_type = 12; + string status = 13; + string updated_at = 14; + string content = 15; + string metadata_json = 16; +} + +// GET /v1/knowledge/llm-wiki/nodes/{id}/preview streams the original bytes of a +// raw PDF or DOCX instead of the JSON envelope. +message LLMWikiPreviewRequest { + string id = 1; +} + +// DELETE /v1/knowledge/llm-wiki/nodes/{id} removes a raw source, the pages +// generated from it and its index entries. Generated pages cannot be deleted +// individually: they belong to an immutable build. +message LLMWikiDeleteRequest { + string id = 1; +} +message LLMWikiDeleteResponse { + bool deleted = 1; +} + +// GET /v1/knowledge/llm-wiki/sources?status=&limit= +message LLMWikiSourcesRequest { + string status = 1; + int32 limit = 2; +} +message LLMWikiSourcesResponse { + repeated LLMWikiSource items = 1; + int64 total = 2; +} + +// GET /v1/knowledge/llm-wiki/search?q=&limit= searches published Wiki pages +// only; knowledge-base path and tag filters are not supported here. +message LLMWikiSearchRequest { + string q = 1; + int32 limit = 2; +} +message LLMWikiSearchResponse { + repeated LLMWikiSearchHit items = 1; + int64 total = 2; +} + +// GET /v1/knowledge/llm-wiki/jobs?limit= +message LLMWikiJobsRequest { + int32 limit = 1; +} +message LLMWikiJobsResponse { + repeated LLMWikiJob items = 1; + int64 total = 2; +} + +// POST /v1/knowledge/llm-wiki/sync projects the organization knowledge base +// (manual and upload documents) into the Wiki raw tree. It is refused with a +// conflict while a compile job is pending or running. +message LLMWikiSyncResponse { + int64 created = 1; + int64 updated = 2; + int64 unchanged = 3; + int64 deleted = 4; + int64 total = 5; +} + +// POST /v1/knowledge/llm-wiki/compile queues one compilation and answers 202 +// with an LLMWikiJob as the payload. An empty source_ids compiles the whole +// corpus; otherwise only the listed sources are recompiled and the pages that do +// not depend on them are inherited from the active build. +message LLMWikiCompileRequest { + bool force = 1; + repeated string source_ids = 2; +} + +// POST /v1/knowledge/llm-wiki/jobs/{id}/retry re-queues a failed or cancelled +// job; POST /v1/knowledge/llm-wiki/jobs/{id}/cancel requests cancellation of a +// pending or running one. Both answer with the updated LLMWikiJob as the payload. +message LLMWikiJobRequest { + string id = 1; +} diff --git a/cmd/ongrid/main.go b/cmd/ongrid/main.go index 85ddd6faa..85f72fa51 100644 --- a/cmd/ongrid/main.go +++ b/cmd/ongrid/main.go @@ -119,6 +119,7 @@ import ( managerbizimbridgeslack "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/slack" managerbizimbridgetelegram "github.com/ongridio/ongrid/internal/manager/biz/imbridge/provider/telegram" managerbizknowledge "github.com/ongridio/ongrid/internal/manager/biz/knowledge" + managerbizllmwiki "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" managerbizmarketplace "github.com/ongridio/ongrid/internal/manager/biz/marketplace" managerbizmcp "github.com/ongridio/ongrid/internal/manager/biz/mcp" managerbizmonitor "github.com/ongridio/ongrid/internal/manager/biz/monitor" @@ -131,6 +132,8 @@ import ( manageraiopsdata "github.com/ongridio/ongrid/internal/manager/data/aiops/store" managerapprovaldata "github.com/ongridio/ongrid/internal/manager/data/approval/store" managerimbridgedata "github.com/ongridio/ongrid/internal/manager/data/imbridge/store" + managerllmwikidata "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki" + managerllmwikistore "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki/store" managerknowledgedata "github.com/ongridio/ongrid/internal/manager/data/knowledge/store" managermarketplacedata "github.com/ongridio/ongrid/internal/manager/data/marketplace/store" managermcpdata "github.com/ongridio/ongrid/internal/manager/data/mcp/store" @@ -194,6 +197,7 @@ import ( managersvcedge "github.com/ongridio/ongrid/internal/manager/service/edge" managersvcfb "github.com/ongridio/ongrid/internal/manager/service/frontierbound" managersvck8s "github.com/ongridio/ongrid/internal/manager/service/k8s" + managersvcknowledge "github.com/ongridio/ongrid/internal/manager/service/knowledge" managersvcmetric "github.com/ongridio/ongrid/internal/manager/service/metric" managersvcprom "github.com/ongridio/ongrid/internal/manager/service/prometheus" managersvcsystemhealth "github.com/ongridio/ongrid/internal/manager/service/systemhealth" @@ -301,6 +305,7 @@ func main() { manageraudtdata.Migrate, managerreportdata.Migrate, managerflowdata.Migrate, + managerllmwikistore.Migrate, managerpacketcapturedata.Migrate, ); err != nil { log.Error("run migrations", slog.Any("err", err)) @@ -1501,9 +1506,42 @@ func main() { if qdrantURL == "" { qdrantURL = "http://qdrant:6333" } + qdrantClient := qdrantx.New(qdrantURL, log.With(slog.String("comp", "qdrant"))) + var maybeEmbedder embedding.Embedder + if embErr != nil { + log.Warn("knowledge: embedder unavailable — reads enabled, writes disabled", + slog.Any("err", embErr)) + } else { + maybeEmbedder = embedder + } + var llmWikiUC *managerbizllmwiki.Usecase + if cfg.LLMWiki.Enabled { + wikiRoot := strings.TrimSpace(cfg.LLMWiki.Dir) + wikiFiles, err := managerbizllmwiki.NewFileStore(wikiRoot) + if err != nil { + log.Error("llm wiki: file store failed", slog.Any("err", err)) + } else if ensureErr := wikiFiles.Ensure(rootCtx); ensureErr != nil { + log.Error("llm wiki: initialize file tree failed", slog.Any("err", ensureErr)) + } else { + wikiStore, openErr := managerllmwikidata.Open(rootCtx, db, qdrantClient, maybeEmbedder, embDim, log.With(slog.String("comp", "llmwiki-db"))) + if openErr != nil { + log.Error("llm wiki: store open failed", slog.Any("err", openErr)) + } else { + wikiRepo := wikiStore.Repository() + wikiIndexer := wikiStore.SearchIndex + // Bind Wiki compilation to the configured LLM model. Provider/Model + // are left empty so the shared LLM router follows the user's default. + wikiSummarizer := managerbizllmwiki.NewLLMAdapter(llmClient, "", "", cfg.OpenAI.Model) + llmWikiUC, err = managerbizllmwiki.NewWithUsageRecorder(rootCtx, wikiRepo, wikiFiles, wikiSummarizer, wikiIndexer, log.With(slog.String("comp", "llmwiki")), managersvcknowledge.NewLLMWikiUsageRecorder(aiopsRepo), managerbizllmwiki.CompileTriggerOption{Owner: "ongrid-manager", Timeout: time.Duration(cfg.LLMWiki.TimeoutSeconds) * time.Second}) + if err != nil { + log.Error("llm wiki: usecase failed", slog.Any("err", err)) + } + } + } + } var knowledgeUC *managerbizknowledge.Usecase + var knowledgeSearcher *managersvcknowledge.HybridSearcher { - qdrantClient := qdrantx.New(qdrantURL, log.With(slog.String("comp", "qdrant"))) // Build with a nil embedder when one isn't configured — the // usecase exposes read paths (ListDocs/Repos/GetDoc/ListPaths) // and gates write paths (CreateManualDoc/Sync/Search) on @@ -1511,13 +1549,6 @@ func main() { // fresh install instead of 404'ing. Operator configures // ONGRID_EMBEDDING_API_KEY later → writes unblock without // restart-of-stack (only the manager needs the key on boot). - var maybeEmbedder embedding.Embedder - if embErr != nil { - log.Warn("knowledge: embedder unavailable — reads enabled, writes disabled", - slog.Any("err", embErr)) - } else { - maybeEmbedder = embedder - } uc, kErr := managerbizknowledge.New(rootCtx, knowledgeRepo, qdrantClient, maybeEmbedder, os.Getenv("ONGRID_KNOWLEDGE_REPO_DIR"), log.With(slog.String("comp", "knowledge"))) @@ -1526,7 +1557,8 @@ func main() { } else { knowledgeUC = uc go knowledgeUC.RunAutoSync(rootCtx) - toolsReg.SetKnowledgeSearcher(knowledgeUC) + knowledgeSearcher = managersvcknowledge.NewHybridSearcher(knowledgeUC, llmWikiUC) + toolsReg.SetKnowledgeSearcher(knowledgeSearcher) apmService.WithSourceRevisions(knowledgeUC) // GitHub-PAT-via-GIT_ASKPASS resolver wiring // removed. SSH-style repos use ssh_identities; HTTPS auth @@ -2044,6 +2076,14 @@ func main() { var knowledgeHandler *managerserverknowledge.Handler if knowledgeUC != nil { knowledgeHandler = managerserverknowledge.NewHandler(knowledgeUC) + knowledgeHandler.SetSearchService(knowledgeSearcher) + knowledgeHandler.SetAuthz(authzMW) + } + if llmWikiUC != nil { + if knowledgeHandler == nil { + knowledgeHandler = managerserverknowledge.NewHandler(nil) + } + knowledgeHandler.SetLLMWikiService(llmWikiUC) knowledgeHandler.SetAuthz(authzMW) } diff --git a/db/migrations/20260918100000_add_llm_wiki_tables.down.sql b/db/migrations/20260918100000_add_llm_wiki_tables.down.sql new file mode 100644 index 000000000..fecf53a79 --- /dev/null +++ b/db/migrations/20260918100000_add_llm_wiki_tables.down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS wiki_lexical; +DROP TABLE IF EXISTS wiki_build_pages; +DROP TABLE IF EXISTS wiki_builds; +DROP TABLE IF EXISTS wiki_compile_jobs; +DROP TABLE IF EXISTS wiki_source_versions; +DROP TABLE IF EXISTS wiki_sources; diff --git a/db/migrations/20260918100000_add_llm_wiki_tables.up.sql b/db/migrations/20260918100000_add_llm_wiki_tables.up.sql new file mode 100644 index 000000000..1080c92b3 --- /dev/null +++ b/db/migrations/20260918100000_add_llm_wiki_tables.up.sql @@ -0,0 +1,100 @@ +-- LLM Wiki catalog and derived search tables on the shared application +-- database. Mirrors the GORM models under +-- internal/manager/model/knowledge/llm_wiki; MySQL 8.0 InnoDB, utf8mb4. +CREATE TABLE IF NOT EXISTS wiki_sources ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + source_key VARCHAR(512) NOT NULL DEFAULT '', + source_type VARCHAR(32) NOT NULL DEFAULT '', + raw_path VARCHAR(1024) NOT NULL DEFAULT '', + current_version_id BIGINT UNSIGNED NULL, + content_sha256 VARCHAR(64) NOT NULL DEFAULT '', + status VARCHAR(24) NOT NULL DEFAULT 'pending', + created_at DATETIME(3) NULL, + updated_at DATETIME(3) NULL, + deleted_at DATETIME(3) NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_wiki_source (tenant_id, source_key), + KEY idx_wiki_source_status (tenant_id, status), + KEY idx_wiki_source_deleted (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS wiki_source_versions ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + source_id BIGINT UNSIGNED NOT NULL, + sha256 VARCHAR(64) NOT NULL DEFAULT '', + size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0, + snapshot_path VARCHAR(1024) NOT NULL DEFAULT '', + schema_version VARCHAR(32) NOT NULL DEFAULT 'v1', + created_at DATETIME(3) NULL, + updated_at DATETIME(3) NULL, + deleted_at DATETIME(3) NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_wiki_source_version (tenant_id, source_id, sha256), + KEY idx_wiki_version_source (tenant_id, source_id), + KEY idx_wiki_version_deleted (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS wiki_compile_jobs ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + active_key VARCHAR(64) NULL, + status VARCHAR(24) NOT NULL DEFAULT 'pending', + stage VARCHAR(32) NOT NULL DEFAULT 'queued', + force_compile TINYINT(1) NOT NULL DEFAULT 0, + source_ids VARCHAR(2048) NOT NULL DEFAULT '', + lease_owner VARCHAR(128) NOT NULL DEFAULT '', + lease_expires_at DATETIME(3) NULL, + attempt INT UNSIGNED NOT NULL DEFAULT 0, + cancel_requested TINYINT(1) NOT NULL DEFAULT 0, + error_message VARCHAR(2048) NOT NULL DEFAULT '', + created_at DATETIME(3) NULL, + updated_at DATETIME(3) NULL, + deleted_at DATETIME(3) NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_wiki_job_active (active_key), + KEY idx_wiki_job_claim (tenant_id, status, lease_expires_at), + KEY idx_wiki_job_deleted (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS wiki_builds ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + status VARCHAR(24) NOT NULL DEFAULT 'staging', + page_count INT NOT NULL DEFAULT 0, + error_msg TEXT NOT NULL, + created_at DATETIME(3) NULL, + activated_at DATETIME(3) NULL, + PRIMARY KEY (id), + KEY idx_wiki_build_tenant_status (tenant_id, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS wiki_build_pages ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + build_id BIGINT UNSIGNED NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + page_id VARCHAR(64) NOT NULL DEFAULT '', + page_type VARCHAR(24) NOT NULL DEFAULT 'generated', + title VARCHAR(512) NOT NULL DEFAULT '', + body_path VARCHAR(1024) NOT NULL DEFAULT '', + body_sha256 VARCHAR(64) NOT NULL DEFAULT '', + source_refs_json JSON NOT NULL, + created_at DATETIME(3) NULL, + PRIMARY KEY (id), + UNIQUE KEY uk_wiki_build_page (build_id, page_id), + KEY idx_wiki_build_page_tenant (tenant_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Derived search table: rebuildable from build pages and artifacts. +CREATE TABLE IF NOT EXISTS wiki_lexical ( + page_key VARCHAR(96) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + page_id VARCHAR(64) NOT NULL, + page_type VARCHAR(24) NOT NULL DEFAULT '', + title VARCHAR(512) NOT NULL DEFAULT '', + aliases VARCHAR(512) NOT NULL DEFAULT '', + content LONGTEXT NOT NULL, + PRIMARY KEY (page_key), + KEY idx_wiki_lexical_tenant (tenant_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/deploy/.env.example b/deploy/.env.example index c77279dc1..e8e6da0d1 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -44,6 +44,11 @@ ONGRID_EMBEDDING_DIM=512 ONGRID_EMBEDDING_API_KEY= ONGRID_EMBEDDING_BASE_URL= +# --- LLM Wiki persistent workspace --- +ONGRID_LLM_WIKI_ENABLED=true +ONGRID_LLM_WIKI_DIR=/var/lib/ongrid/llm-wiki +ONGRID_LLM_WIKI_TIMEOUT_SECONDS=600 + # --- Embedded observability retention (test environments can lower these) --- ONGRID_PROM_RETENTION_TIME=2160h ONGRID_PROM_RETENTION_SIZE=20GB diff --git a/deploy/README.md b/deploy/README.md index 04c41c733..f01714b28 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -93,7 +93,8 @@ In production, set both to strong values in `.env` **before** the first | `http://localhost:9090` | Prometheus UI (targets, graph) | | `localhost:3306` | MySQL (user `ongrid`, pw `ongrid`, db `ongrid`) | -Data lives in the `mysql_data` Docker volume. Back that volume up with +MySQL data lives in the `mysql_data` Docker volume. LLM Wiki files persist in +`../.cache/ongrid-llm-wiki` on the host. Back up MySQL with `sudo tar czf mysql.tgz -C "$(docker volume inspect -f '{{.Mountpoint}}' mysql_data)" .` or the equivalent in your ops tooling. diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 874c0defb..2b62d6969 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -104,6 +104,11 @@ services: ONGRID_EMBEDDING_BASE_URL: ${ONGRID_EMBEDDING_BASE_URL:-} ONGRID_EMBEDDING_API_KEY: ${ONGRID_EMBEDDING_API_KEY:-} ONGRID_EMBEDDING_CACHE_DIR: ${ONGRID_EMBEDDING_CACHE_DIR:-/var/lib/ongrid/embeddings} + # LLM Wiki source files, generated pages, version snapshots and staging data. + # The bind-mount target below follows ONGRID_LLM_WIKI_DIR. + ONGRID_LLM_WIKI_ENABLED: ${ONGRID_LLM_WIKI_ENABLED:-true} + ONGRID_LLM_WIKI_DIR: ${ONGRID_LLM_WIKI_DIR:-/var/lib/ongrid/llm-wiki} + ONGRID_LLM_WIKI_TIMEOUT_SECONDS: ${ONGRID_LLM_WIKI_TIMEOUT_SECONDS:-600} # Optional pip index mirror for cloud_bash runtime tool installs (empty = # pypi). Set to a regional mirror where pypi.org is slow. ONGRID_PIP_INDEX_URL: ${ONGRID_PIP_INDEX_URL:-} @@ -183,6 +188,8 @@ services: - ../.cache/ongrid-tools:/var/lib/ongrid/tools # Packet capture raw PCAP objects for local dev. - ../.cache/ongrid-packet-captures:/var/lib/ongrid/packet-captures + # LLM Wiki raw sources, generated Markdown, snapshots and staging data. + - ../.cache/ongrid-llm-wiki:${ONGRID_LLM_WIKI_DIR:-/var/lib/ongrid/llm-wiki} - ../.cache/ongrid-chat-attachments:/var/lib/ongrid/chat-attachments # ADR-008: HTTP API (8080) is no longer published to host in the # production-shape stack — nginx fronts it on 443/80. Metrics port diff --git a/deploy/install/.env.example b/deploy/install/.env.example index 7cfdfb8d1..3b4c267d8 100644 --- a/deploy/install/.env.example +++ b/deploy/install/.env.example @@ -124,6 +124,13 @@ ONGRID_EMBEDDING_MODEL=bge-small-zh-v1.5 ONGRID_EMBEDDING_BASE_URL= ONGRID_EMBEDDING_DIM=512 +# --- LLM Wiki persistent workspace --- +# The host side is /llm-wiki. docker-compose.yml mounts it at +# ONGRID_LLM_WIKI_DIR so raw sources, pages, snapshots and FTS survive upgrades. +ONGRID_LLM_WIKI_ENABLED=true +ONGRID_LLM_WIKI_DIR=/var/lib/ongrid/llm-wiki +ONGRID_LLM_WIKI_TIMEOUT_SECONDS=600 + # --- Advanced (usually leave alone) --- ONGRID_HTTP_ADDR=:8080 ONGRID_METRICS_ADDR=:9100 diff --git a/deploy/install/README.md b/deploy/install/README.md index b0fa30260..9584cffcb 100644 --- a/deploy/install/README.md +++ b/deploy/install/README.md @@ -161,7 +161,7 @@ sudo ./uninstall.sh --purge --yes ## 数据存储 -v0.7.45 起所有有状态服务（MySQL / Prometheus / Loki / Tempo / qdrant / Grafana）的数据卷**直接 bind-mount 到宿主机**，默认根路径 `/var/lib/ongrid`，可通过 `ONGRID_DATA_DIR` 覆盖： +v0.7.45 起所有有状态服务和 Manager 持久数据目录都**直接 bind-mount 到宿主机**，默认根路径 `/var/lib/ongrid`，可通过 `ONGRID_DATA_DIR` 覆盖： ```text /var/lib/ongrid/ @@ -170,7 +170,8 @@ v0.7.45 起所有有状态服务（MySQL / Prometheus / Loki / Tempo / qdrant / ├── loki/ # Loki chunks (uid 10001) ├── tempo/ # Tempo blocks (uid 10001) ├── qdrant/ # 向量 collection (root) -└── grafana/ # Grafana SQLite + plugins (uid 472) +├── grafana/ # Grafana SQLite + plugins (uid 472) +└── llm-wiki/ # 原始资料、不可变版本快照和 build 页面 artifact (uid 65532) /var/log/ongrid/ # manager slog 输出，可被宿主机 Collector / Vector / Fluent Bit 直接抓取 ``` @@ -198,6 +199,7 @@ sudo tar czf /backup/prom-$(date +%F).tar.gz -C /var/lib/ongrid/prometheus . sudo tar czf /backup/loki-$(date +%F).tar.gz -C /var/lib/ongrid/loki . sudo tar czf /backup/tempo-$(date +%F).tar.gz -C /var/lib/ongrid/tempo . sudo tar czf /backup/qdrant-$(date +%F).tar.gz -C /var/lib/ongrid/qdrant . +sudo tar czf /backup/llm-wiki-$(date +%F).tar.gz -C /var/lib/ongrid/llm-wiki . ``` ### 数据卷迁移（v0.7.45 前的安装升级到 v0.7.45+ 必读） diff --git a/deploy/install/data-permissions.sh b/deploy/install/data-permissions.sh index a6d531193..e43a544fc 100644 --- a/deploy/install/data-permissions.sh +++ b/deploy/install/data-permissions.sh @@ -102,6 +102,7 @@ ongrid_prepare_data_directories() { "$data_dir/skills" \ "$data_dir/pages" \ "$data_dir/packet-captures" \ + "$data_dir/llm-wiki" \ "$data_dir/chat-attachments" \ "$data_dir/workspace" \ "$data_dir/tools" \ @@ -123,6 +124,7 @@ ongrid_prepare_data_directories() { ongrid_ensure_path_owner 65532:65532 "$data_dir/skills" || failed=1 ongrid_ensure_path_owner 65532:65532 "$data_dir/pages" || failed=1 ongrid_ensure_path_owner 65532:65532 "$data_dir/packet-captures" || failed=1 + ongrid_ensure_path_owner 65532:65532 "$data_dir/llm-wiki" || failed=1 ongrid_ensure_path_owner 65532:65532 "$data_dir/chat-attachments" || failed=1 ongrid_ensure_path_owner 65532:65532 "$data_dir/workspace" || failed=1 ongrid_ensure_path_owner 65532:65532 "$data_dir/tools" || failed=1 @@ -149,6 +151,7 @@ ongrid_repair_data_permissions() { ongrid_chown_tree_required 65532:65532 "$data_dir/skills" || failed=1 ongrid_chown_tree_required 65532:65532 "$data_dir/pages" || failed=1 ongrid_chown_tree_required 65532:65532 "$data_dir/packet-captures" || failed=1 + ongrid_chown_tree_required 65532:65532 "$data_dir/llm-wiki" || failed=1 ongrid_chown_tree_required 65532:65532 "$data_dir/chat-attachments" || failed=1 ongrid_chown_tree_required 65532:65532 "$data_dir/workspace" || failed=1 ongrid_chown_tree_required 65532:65532 "$data_dir/tools" || failed=1 diff --git a/deploy/install/docker-compose.yml b/deploy/install/docker-compose.yml index 8226250f7..95383c1ca 100644 --- a/deploy/install/docker-compose.yml +++ b/deploy/install/docker-compose.yml @@ -143,6 +143,10 @@ services: # dir; the install bundle pre-stages it so air-gapped operators # don't need HF reach. ONGRID_EMBEDDING_CACHE_DIR: ${ONGRID_EMBEDDING_CACHE_DIR:-/var/lib/ongrid/embeddings} + # LLM Wiki source files, generated pages, version snapshots and staging data. + ONGRID_LLM_WIKI_ENABLED: ${ONGRID_LLM_WIKI_ENABLED:-true} + ONGRID_LLM_WIKI_DIR: ${ONGRID_LLM_WIKI_DIR:-/var/lib/ongrid/llm-wiki} + ONGRID_LLM_WIKI_TIMEOUT_SECONDS: ${ONGRID_LLM_WIKI_TIMEOUT_SECONDS:-600} ONNX_PATH: /usr/lib/libonnxruntime.so # OTel tracing — Tempo OTLP HTTP receiver (HLD-004 Phase-B trace # evaluators need real spans for traces_spanmetrics_* to populate). @@ -271,6 +275,9 @@ services: # reading them back from the Edge, then serves authenticated downloads and # private parser fetches from the same durable path. - ${ONGRID_DATA_DIR:-/var/lib/ongrid}/packet-captures:/var/lib/ongrid/packet-captures + # LLM Wiki raw sources, generated Markdown, version snapshots, staging + # files and staging data. It must survive container recreation and upgrades. + - ${ONGRID_DATA_DIR:-/var/lib/ongrid}/llm-wiki:${ONGRID_LLM_WIKI_DIR:-/var/lib/ongrid/llm-wiki} - ${ONGRID_DATA_DIR:-/var/lib/ongrid}/chat-attachments:/var/lib/ongrid/chat-attachments - ${ONGRID_DATA_DIR:-/var/lib/ongrid}/pcap-parser/manager:/var/lib/ongrid/pcap-parser:ro logging: diff --git a/deploy/install/install.sh b/deploy/install/install.sh index de663e74c..31b6d1ae3 100755 --- a/deploy/install/install.sh +++ b/deploy/install/install.sh @@ -585,6 +585,7 @@ mkdir -p \ "$ONGRID_DATA_DIR/skills" \ "$ONGRID_DATA_DIR/pages" \ "$ONGRID_DATA_DIR/packet-captures" \ + "$ONGRID_DATA_DIR/llm-wiki" \ "$ONGRID_DATA_DIR/workspace" \ "$ONGRID_DATA_DIR/tools" \ "$ONGRID_LOG_DIR" @@ -616,6 +617,7 @@ chown -R 65532:65532 "$ONGRID_DATA_DIR/skills" 2>/dev/null || true # cloud_bash fails "mkdir session" (workspace) + can't install tools. chown -R 65532:65532 "$ONGRID_DATA_DIR/pages" 2>/dev/null || true chown -R 65532:65532 "$ONGRID_DATA_DIR/packet-captures" 2>/dev/null || true +chown -R 65532:65532 "$ONGRID_DATA_DIR/llm-wiki" 2>/dev/null || true chown -R 65532:65532 "$ONGRID_DATA_DIR/workspace" 2>/dev/null || true chown -R 65532:65532 "$ONGRID_DATA_DIR/tools" 2>/dev/null || true diff --git a/docs/adr/ADR-033-llm-wiki-architecture.md b/docs/adr/ADR-033-llm-wiki-architecture.md new file mode 100644 index 000000000..506757d17 --- /dev/null +++ b/docs/adr/ADR-033-llm-wiki-architecture.md @@ -0,0 +1,118 @@ +# ADR-033：LLM Wiki 架构 + +- 状态：已接受 +- 日期：2026-09-11 +- 关联 RFC：[RFC-004：LLM Wiki 编译、存储与检索实现](../rfc/RFC-004-llm-wiki.md) +- 关联 HLD：[HLD-002：LLM Wiki 可审计知识编译层](../design/HLD-002-llm-wiki.md) + +## 背景 + +LLM Wiki 同时包含四条需要长期稳定的架构边界： + +1. 编译管线如何把 Source 转换为页面。 +2. Raw、版本、页面 artifact 与 catalog 分别由什么存储负责。 +3. MySQL/SQLite 双方言如何共享 schema 和词法检索语义。 +4. 页面向量放在哪里，以及与词法检索如何配合。 + +## 决策 + +### 1. 使用类型化、线性编译管线 + +`buildCompiler.Compile` 固定按以下顺序执行： + +1. 创建 `staging` Build。 +2. `loadCorpus` 读取 Source 和 Version。 +3. `prepareCorpusChunks` 读取快照并切块。 +4. `summarizer.Summarize` 生成 digest。 +5. `planner.Plan` 生成页面计划。 +6. `evidenceResolver.Resolve` 解析 section 来源。 +7. `pageWriter.WritePage` 和 `artifactStore.writePageBody` 写页面。 +8. `validateBuild` 校验 artifact。 +9. `indexBuildPages` 更新词法和向量索引。 +10. `Publish` 在事务中激活 build。 + +包内类型包括 `corpus`、`digestItem`、`plan`、`resolvedPage`、`WikiBuildPage` 和 `IndexDocument`。当前不存在 `sourceIR/parseIR/semanticIR/knowledgeIR`。 + +### 2. FileStore 与应用数据库分离 + +FileStore 保存： + +- `raw/` 当前 Raw 文件。 +- `.llm-wiki/versions//.md` 不可变版本快照。 +- `wiki/builds//pages/.md` 生成页面 artifact。 + +应用数据库保存： + +- Source、Version 和 CompileJob。 +- Build 和 BuildPage catalog。 +- `wiki_lexical`。 + +应用数据库是 job、build 和来源关系的权威 catalog；FileStore 是 Raw、版本和页面 artifact 的权威存储；Qdrant 是派生向量索引。 + +已删除的表（2026-09-20，从 `db/migrations/20260918100000_add_llm_wiki_tables` 与 `store.Migrate` 移除）： + +- `wiki_source_chunks`：从未有写入路径，chunk 只存在于编译期内存和 build 页面里。 +- `wiki_index_meta`：只写不读，索引重建实际以 Qdrant 是否已有向量为准。 + +### 3. 跟随 `ONGRID_DB_DIALECT` + +- `managerllmwikidata.Open` 接收共享 `*gorm.DB`。 +- Wiki 不创建独立 `.llm-wiki/wiki.db`，不拥有连接池。 +- `store.Migrate` 只接受 `mysql` 和 `sqlite`。 +- MySQL 当前 schema 位于 `db/migrations/20260918100000_add_llm_wiki_tables`。 +- SQLite 设置 `foreign_keys=ON`、`busy_timeout=5000`、`journal_mode=WAL`。 +- 词法索引使用普通表 `wiki_lexical` 和 `LIKE ... ESCAPE '!'`，不使用 SQLite FTS5 或 MySQL 专有全文索引。 + +### 4. 页面向量使用 Qdrant + +- collection：`ongrid_llm_wiki` +- point ID：`sha256("wiki::")` 前 8 字节 +- payload：tenant 字符串、page_id、page_type、title、body_hash +- 查询强制 tenant filter +- `body_hash` 未变化时跳过 embedding +- 配置 embedder 时 Qdrant 是硬依赖 +- 向量失败只影响派生索引，不被词法结果掩盖 +- 词法与向量结果使用 RRF 常数 60 融合 + +## 一致性边界 + +- active build 切换在应用数据库事务中完成。 +- 页面 artifact 通过 SHA-256 校验。 +- 新 build 激活后删除旧 build artifact；旧 Build/BuildPage 数据库行保留为 `superseded`。 +- 文件系统和应用数据库之间不存在跨存储事务。 +- 当前 `Reconcile` 只从 SourceVersion 快照恢复缺失 Raw 文件，不重放编译任务。 +- Qdrant 和 `wiki_lexical` 都可重建。 + +## 备选方案 + +### 单个无类型编译函数 + +降低文件数量，但错误边界、清理逻辑和增量继承无法由类型表达。 + +### 五层公共 IR + +隔离更强，但当前代码没有对应类型，也没有跨包复用需求。 + +### 独立 SQLite 文件 + +有利于单目录携带，但与共享 MySQL 的部署、迁移和备份边界冲突。 + +### 数据库内 BLOB 向量和进程内余弦 + +少一个外部依赖，但需要线性扫描，无法利用 Qdrant HNSW。 + +### 复用旧 RAG collection + +会混合页面级与 chunk 级 point，payload 和重建边界不同。 + +## 后果 + +- 编译流程固定且可审计，但当前不支持动态 DAG。 +- Wiki catalog 跟随应用数据库，FileStore 仍需独立持久化和备份。 +- 配置 embedder 后，Qdrant 成为 Wiki 的运行依赖。 +- 词法检索是全表 `LIKE`，规模上限需要和向量检索一起评估。 +- Source 的 `succeeded` 状态没有生产赋值路径。 +- `force_compile` 字段存在，但当前编译代码没有读取。 +- usage session 会把每个成功 compiler LLM 调用的 usage 写入现有 AIOps chat transcript；记录失败只告警，不阻断 Wiki 编译。 +- `manager_metrics.go` 中的 LLM Wiki collectors 当前没有被编译代码更新。 +- OpenAI-compatible embedding 用量和 Wiki 编译调用的每日 token budget 不在当前范围内。 diff --git a/docs/design/HLD-002-llm-wiki.md b/docs/design/HLD-002-llm-wiki.md new file mode 100644 index 000000000..030897fce --- /dev/null +++ b/docs/design/HLD-002-llm-wiki.md @@ -0,0 +1,376 @@ +# HLD-002：LLM Wiki 可审计知识编译层 + +- 状态：已批准 +- 日期：2026-09-10 +- 实现入口：`internal/manager/biz/knowledge/llm_wiki/` +- HTTP 入口：`internal/manager/server/knowledge/llmwiki_http.go` +- 数据入口：`internal/manager/data/knowledge/llm_wiki/` +- 关联 ADR：[ADR-033：LLM Wiki 架构](../adr/ADR-033-llm-wiki-architecture.md) +- 关联 RFC：[RFC-004：LLM Wiki 编译、存储与检索实现](../rfc/RFC-004-llm-wiki.md) + +## 1. 范围 + +LLM Wiki 是 Manager 进程内的知识编译模块，不是独立服务。当前代码提供： + +- 将组织知识库中的 `manual`、`upload`、`repo` 文档同步为 Wiki Raw 来源。 +- 将 Raw 内容切块、摘要、规划为生成页面并写入不可变 build。 +- 通过 active build 提供 Raw 与生成页文件树、详情读取和 PDF/DOCX Raw 预览。 +- 通过应用数据库词法索引和 Qdrant 页级向量索引提供 Wiki 检索。 +- 将 Wiki 检索与旧组织知识库组合为 `hybrid`、`wiki`、`rag` 三种模式。 +- 提供编译任务创建、查询、取消和重试 HTTP API。 + +当前代码不包含独立队列消费者、Topic/Evidence 表、Canonical Topic 页面、staging manifest、Schema HTTP API、单独上传 Wiki API 或单独 build 查询 API。 + +## 2. 系统边界 + +```mermaid +flowchart LR + UI["Knowledge UI"] --> HTTP["Knowledge HTTP Handler"] + AGENT["query_knowledge"] --> HYBRID["HybridSearcher"] + HTTP --> UC["llm_wiki.Usecase"] + HYBRID --> UC + HYBRID --> RAW["Organization Knowledge Usecase"] + UC --> FS["FileStore"] + UC --> REPO["Repository"] + UC --> LLM["CompilerLLM"] + UC --> IDX["SearchIndex"] + REPO --> DB["Application DB"] + IDX --> DB + IDX --> QD["Qdrant ongrid_llm_wiki"] +``` + +| 层 | 位置 | 当前职责 | +| --- | --- | --- | +| Server | `internal/manager/server/knowledge/llmwiki_http.go` | 注册 HTTP 路由、解析参数、组织知识库同步输入、统一响应和错误映射 | +| Service | `internal/manager/service/knowledge/hybrid_search.go`、`llmwiki_usage.go` | Raw/Wiki 混合检索、内部 Token usage session 适配 | +| Biz | `internal/manager/biz/knowledge/llm_wiki/` | Source 镜像、编译管线、页面写入、发布、文件树、删除、检索编排 | +| Data | `internal/manager/data/knowledge/llm_wiki/` | 应用数据库仓储、词法索引、Qdrant 向量索引 | +| Model | `internal/manager/model/knowledge/llm_wiki/` | Source、Version、CompileJob、Build、BuildPage、Lexical | +| Files | `ONGRID_LLM_WIKI_DIR` | Raw 文件、不可变版本快照、生成页面 artifact | + +## 3. 配置与启动 + +| 环境变量 | 代码默认值 | 当前行为 | +| --- | --- | --- | +| `ONGRID_LLM_WIKI_ENABLED` | `false` | 为 `false` 时不创建 Wiki Usecase，不注册 Wiki HTTP 路由，不启用混合检索 Wiki 分支 | +| `ONGRID_LLM_WIKI_DIR` | `/var/lib/ongrid/llm-wiki` | FileStore 根目录 | +| `ONGRID_LLM_WIKI_TIMEOUT_SECONDS` | `600` | 后台编译任务超时；任务租约为其加 1 分钟 | + +Compose 示例将 `ONGRID_LLM_WIKI_ENABLED` 默认设为 `true`，代码默认仍为 `false`。 + +启动流程： + +1. `NewFileStore` 解析根目录，`Ensure` 删除非当前布局目录/文件并创建 `raw/`、`wiki/builds/`、`.llm-wiki/versions/`。 +2. `llmwikidata.Open` 在应用数据库上创建 Repository 和 SearchIndex。 +3. embedder 已配置时，`index.New` 必须成功连接 Qdrant、确保 `ongrid_llm_wiki` collection 和 `tenant_id` keyword payload index，否则 Wiki 初始化失败。 +4. `Usecase.New` 执行 `Reconcile`，从版本快照恢复缺失 Raw 文件。单个 Source 的 `current_version_id` 指向的行不存在、或快照不可读时，只把该 Source 标记为 `failed` 并跳过：初始化失败会导致 `llmWikiUC` 为 nil，所有 `llm-wiki` 路由都不会注册，前端只能看到 404，也就无法通过重新同步修复它。 +5. 若索引支持 `HasVectors` 且当前租户无向量，尝试从 active build 回填索引。回填失败只记录 warning。 + +## 4. 来源同步与版本 + +### 4.1 来源创建 + +当前生产入口只有 `POST /v1/knowledge/llm-wiki/sync`： + +1. Handler 分别完整分页读取组织知识库的 `manual`、`upload` 和 `repo` 文档；任一类型读取失败时中止，不提交不完整的来源集合。 +2. 每个文档转换为 `OrganizationSource{ID, Title, Path, Content}`。 +3. `SyncOrganizationSources` 使用 `SourceKey = "organization:"`。 +4. Raw 相对路径为 `doc.Path/.md`；路径经过安全清理但保留 Unicode。 +5. 文件名缺失时使用 `document-`。 +6. 内容写入 Raw 文件，并写入内容寻址快照 `.llm-wiki/versions//.md`。 +7. 同步是非破坏性的增量刷新：只新增或更新本次快照中出现的 Source，不删除缺失项。代码仓库重新索引时会先清空再回填 Qdrant，缺失可能只是短暂状态；而删除任一 Source 会使整个 active build 失效，因此同步入口不能以“本次没看到”作为删除依据。 +8. 删除旧 Raw Source 继续使用显式的 `DELETE /v1/knowledge/llm-wiki/nodes/{id}`；同步响应为兼容保留 `deleted` 字段，但该入口返回 `0`。 + +`repo` 文档在 Wiki 中仍是组织知识库投影：SourceType 为 `organization`，SourceKey 使用其稳定的知识库文档 ID。Raw 目录沿用组织树的 `path/title` 布局，不按仓库额外分层；不同仓库出现相同 path/title 时可能指向同一 Raw 展示路径，但内容寻址版本快照仍按 SourceKey 隔离。仓库同步本身不会自动触发 Wiki 同步，仍由现有 Wiki 同步入口显式刷新；仓库同步期间触发 Wiki 同步也只会增量写入当前可见文件，不会清空旧 Raw 或 active Wiki。 + +`MirrorSource` 支持其他 `sourceType` 的通用路径生成，但当前生产代码没有除 `organization` 外的调用方。 + +### 4.2 身份与状态 + +- Source 唯一键：`(tenant_id, source_key)`。 +- SourceVersion 唯一键：`(tenant_id, source_id, sha256)`。 +- 首次写入 Source 状态为 `pending`。 +- 内容变化且已有版本时状态为 `stale`。 +- 内容未变化时复用当前版本；若 `raw_path` 或 `source_type` 变化，只更新 Source 元数据。 +- 当前编译成功路径不会把 Source 标记为 `succeeded`。`succeeded` 常量存在，但没有生产赋值路径。 +- `Reconcile` 无法从快照恢复 Raw 时，将 Source 标记为 `failed`。 + +## 5. 文件布局 + +```text +/ +├── raw/ # 当前 Raw 文件 +├── wiki/ +│ └── builds/ +│ └── / +│ └── pages/ +│ └── .md # 每个 build 的页面 artifact +└── .llm-wiki/ + └── versions/ + └── / + └── .md # 不可变 SourceVersion 快照 +``` + +`Ensure` 会删除以下非当前布局路径：`concepts`、`entities`、`wiki/concepts`、`wiki/entities`、`wiki/sources`、`wiki/topics`、`wiki/index.md`、`wiki/log.md`、`.llm-wiki/staging`、`schema.md`。 + +文件安全边界： + +- 绝对路径、`..` 路径逃逸和符号链接会被拒绝。 +- 写文件使用临时文件、`fsync` 和 rename。 +- `MirrorSource` 限制单来源不超过 `16 MiB`。 +- 读取 Raw 和生成页分别限制为 `16 MiB` 和 `1 MiB`。 +- 文件树锁 `LockArtifacts` 当前用于树查询、节点读取、预览、同步和删除；后台编译写 artifact 不走该锁。 +- 编译与来源变更的互斥由 job 表承担：存在 `pending/running` job 时，同步和删除返回 conflict；发布前还会校验页面引用的 Source 仍然存在，已删除来源的页面不会被重新发布。 + +## 6. 编译管线 + +当前实现是一条包内私有、线性执行的管线： + +```mermaid +flowchart TD + JOB["CompileJob"] --> BUILD["CreateBuild(staging)"] + BUILD --> CORPUS["loadCorpus"] + CORPUS --> CHUNK["prepareCorpusChunks"] + CHUNK --> DIGEST["Summarizer"] + DIGEST --> PLAN["Planner"] + PLAN --> RESOLVE["Evidence Resolver"] + RESOLVE --> WRITE["Page Writer"] + WRITE --> INHERIT["继承未受影响的旧页面"] + INHERIT --> VALIDATE["校验文件与 SHA-256"] + VALIDATE --> INDEX["Clear + IndexPage"] + INDEX --> PUBLISH["ActivateBuild"] +``` + +### 6.1 Corpus + +- `loadCorpus` 分页读取租户的全部 Source（`ListSourcesAfter`，每页 200）；仓储的单页查询会把 limit 归一为 200，直接依赖它会让语料静默漏掉第 200 条之后的来源。 +- `source_ids` 非空时只保留指定 Source。 +- `current_version_id` 为空的 Source 被跳过。 +- Version 小于 100 bytes 的文档被跳过。 +- 相同 SHA-256 的文档只保留第一个，按 Source ID 排序。 +- 当前不按 Source 状态过滤。 +- 切块后没有任何 chunk 时不调用 Planner：删除 staging build，job 以 `skipped` 结束，已发布的 build 保持不变。 + +### 6.2 Chunk + +- 只按 `\n\n` 分段。 +- 目标 chunk 大小为 `4000` 个估算 token，按 `4 bytes/token` 近似为 `16000 bytes`。 +- 单个超长段落不会被再次切分。 +- chunk 保存文档索引、文档内序号、字节起止和文本。 + +### 6.3 Digest + +- 估算总 token 小于 `32000` 时，直接把 chunk 文本作为 digest item，不调用 LLM。 +- 大于等于 `32000` 时，逐个 chunk 调用 LLM 生成纯文本摘要。 +- 当前没有 chunk cache、批量 LLM、层级摘要、repair 或多阶段摘要。 +- digest 只暴露从 0 开始的 `Digest source_id`，不暴露数据库 Source ID。 +- 每个条目还带一行 `Document: <标题>`（文档无标题时省略）。Planner 看不到 Corpus，标题是它判断这段话出自什么材料、以及给页面命名的唯一依据。标题中的换行/制表符折叠为空格，并按 rune 截断到 200 字符，避免破坏条目分帧。 + +### 6.4 Planner + +- Planner 请求严格 JSON Schema。 +- 第一次调用返回错误时才重试 `json_object` 模式。 +- 返回后做本地结构校验：pages 非空、`page_id/title` 非空、`page_id` 不重复、每页至少一个 section、section 的 heading/content 非空。 +- `page_id` 必须是 kebab-case（白名单正则）且不超过 64 字符（`wiki_build_pages.page_id` 列宽）；`page_id` 会作为 artifact 文件名，校验在写文件前完成。 +- 人类可读的标题、章节和正文被要求继承来源语言；代码通过统一 Prompt 约束，不做语言检测。 +- Prompt 同时约束不得虚构：标题和正文只能取自 digest 陈述的内容，不得自行补出 API、参数、数值、日期、人名、角色或步骤，也不得把猜测写成材料陈述的事实；留空优于编造。 + +### 6.5 Evidence 与 Writer + +- section 的 `source_ids` 是 digest 数组下标。 +- 非法下标会使编译失败。 +- digest index 可映射到一个或多个 Source；找不到对应 Corpus 文档时该来源被忽略。 +- page source refs 记录 Source、SourceVersion、首个 chunk ordinal、内容 hash 和 `source_path`。 +- Writer 输出页面标题和 Markdown section。 +- section 已包含 Markdown 特征或超过 3 行时，原样写入；否则调用 LLM 优化。 +- section Writer LLM 失败时回退为原始 section 文本。 +- section Writer Prompt 同样约束不得补入来源材料里没有的内容。 +- 页面末尾用 `---` 分隔后列出被引用来源的标题和 ID（按 Source 去重）。footer 由代码写出，不经过模型，因此无法像正文那样继承来源语言；为了不给每个页面套上一句固定语言的散文，footer 不含任何语言的文字，只有分隔线和来源文档自己的标题。 + +### 6.6 Build 页面写入 + +- 每个生成页写为 `wiki/builds//pages/.md`。 +- `page_type` 当前固定为 `generated`。 +- 页面来源目录取 SourceRefs 中最小 SourceID 的 `source_path` 所在目录。 +- 页面文件名使用 `page_id`；旧数据没有 `source_path` 时保持扁平路径。 +- 单 Source 增量编译时： + - 与所选 Source 有来源交集的旧页面不继承，由本次编译结果替换。 + - 与所选 Source 无交集且未被本次生成的 page ID 覆盖的旧页面，会校验 hash 后复制到新 build。 + +### 6.7 校验、索引与发布 + +- 所有页面文件必须存在且 SHA-256 匹配，否则 build 标记失败并清理。 +- 索引是 best-effort：`Clear` 后逐页写 `wiki_lexical` 和 Qdrant。 +- 索引失败不会回滚 build，也不会改变最终 job stage；日志记录 warning。 +- 发布要求 build 状态为 `validated`。 +- `ActivateBuild` 在数据库事务中把旧 active build 标记为 `superseded`，再把新 build 标记为 `active`。 +- 发布后 best-effort 删除旧 build 的 artifact 目录；旧 build 的数据库行和页面行保留。 +- 失败或取消的 staging build 会删除 artifact 目录、BuildPage 行和 Build 行。 + +## 7. 任务模型 + +状态常量： + +- `pending` +- `running` +- `succeeded` +- `skipped` +- `failed` +- `cancelled` + +当前行为： + +- `CreateCompileJob` 在租户存在任意 `pending/running` job 时返回 conflict。 +- `active_key` 是租户级常量（`model.ActiveJobKey`），带唯一索引 `uk_wiki_job_active`：在飞的 job 持有它，离开 `running` 时释放。因此“每租户同时只有一个在飞 job”由数据库约束保证，而不只依赖上面那条 conflict 预检查——并发创建可以同时通过预检查。 +- `force` 只写入 `force_compile`，当前编译逻辑没有读取该字段。 +- `CreateCompileJob` 和 `RetryJob` 在配置了 trigger owner 时启动一个带超时的 goroutine。 +- 当前没有生产 worker 循环调用 `RunOnce`。 +- 进程退出后，`pending` job 不会被当前代码自动拉起；过期 `running` job 可由 `ClaimJob` 恢复。 +- `CancelJob` 立即把 job 状态写为 `cancelled` 并设置 `cancel_requested=true`。 +- 编译失败（LLM、计划校验、写文件、发布任一阶段）会把 job 写成 `failed`，stage 为失败发生的 step；已进入终态的 job（例如 `cancelled`）不会被改写。 +- 编译器在每个主要阶段前、发布前检查取消标记。 +- 语料为空时 job 以 `skipped` 结束且不发布 build；成功编译但没有页面时同样是 `skipped`。 +- 其他成功 job 状态为 `succeeded`，stage 为 `completed`。 +- 失败 stage 使用失败发生处的 step 文本，错误截断到 2048 bytes。 +- `RetryJob` 重新使用原 `source_ids`，active key 与新建 job 一致（租户级 key）。 + +## 8. 数据模型 + +应用数据库表： + +| 表 | 当前用途 | +| --- | --- | +| `wiki_sources` | Raw Source 当前版本、路径和状态 | +| `wiki_source_versions` | 不可变内容版本与快照路径 | +| `wiki_compile_jobs` | 编译任务、active key、租约、取消和错误 | +| `wiki_builds` | build 状态、页面数和激活时间 | +| `wiki_build_pages` | build 内页面、artifact 路径、hash 和来源引用 JSON | +| `wiki_lexical` | 可重建词法索引；`aliases` 字段存在但 `IndexPage` 当前不写入别名 | + +MySQL 表由 `db/migrations/20260918100000_add_llm_wiki_tables` 创建；`wiki_source_chunks` 和 `wiki_index_meta` 已从该 migration 与 `store.Migrate` 中删除（前者从未被写入，后者的 schema version 无读取方）。SQLite 和 MySQL 的 GORM schema 由 `store.Migrate` 的 `AutoMigrate` 对齐。 + +## 9. 检索 + +### 9.1 Wiki 内部索引 + +`IndexPage`： + +1. 按 `(tenant_id, page_id)` 替换 `wiki_lexical` 行。 +2. 未配置 embedder 时结束。 +3. 已配置 embedder 时计算 Qdrant point ID，读取现有 point。 +4. 现有 `body_hash` 等于当前页面内容 hash 时跳过 embedding。 +5. 否则 embedding `title + "\n\n" + content` 并 upsert Qdrant。 + +`Clear` 先按 tenant 删除 Qdrant point，再删除该 tenant 的 `wiki_lexical` 行。 + +词法检索使用 `LIKE ... ESCAPE '!'`，查询中的 `%`、`_`、`!` 会被转义。排序优先 title、其次 aliases、最后 content。 + +向量检索使用 Qdrant： + +- collection：`ongrid_llm_wiki` +- point ID：`sha256("wiki::")` 前 8 字节 +- payload：tenant 字符串、page_id、page_type、title、body_hash +- 查询强制 tenant 过滤 +- Qdrant 错误不会被词法结果掩盖 + +词法和向量结果用 RRF 常数 60 融合，标题包含查询时加 `0.005`。融合值只决定排序：`score` 返回各层自己测得的相似度（Qdrant cosine，或词法腿的三档匹配权重），因此不跨层可比。曾把 `1.5/(60+rank)`、`1.0/(60+rank)` 当作 `score` 返回，这两个值在前十名内只差一个百分位，页面搜索整屏显示为 `0.02`，协调员 playbook 注入的 `0.6` 阈值也永远无法通过。 + +### 9.2 混合检索 + +`HybridSearcher`： + +- `rag`，或 Wiki 未启用：只查 Raw。 +- `wiki`：只返回 Wiki。 +- `hybrid`：分别查询 Wiki 和 Raw；两边都失败时返回组合错误，一边失败时返回另一边，否则按名次交错两层——名次 n 的 Wiki 命中后面跟名次 n 的 Raw 命中，谁先取完就把剩余名额让给另一层。此前用权重合并（Wiki `1.5`、Raw `1.0`）：任何大于 1 的权重都会让名次相同的 Wiki 命中全部排在 Raw 之前，于是 limit 个 Wiki 命中占满结果、Raw 一个都看不到。 +- 带 `Path`、`PathPrefix` 或 `Tags` 过滤时一律不返回 Wiki 命中：Wiki 页面是按来源生成的，既不落在知识库路径下也没有 tag，无法校验是否满足过滤条件。此时 `hybrid`/`rag` 只查 Raw，`wiki` 模式返回空结果，而不是返回无视过滤条件的命中。 + +## 10. HTTP API + +所有路径挂载在认证后的 `/api/v1`。业务响应统一为 `{code, message, data}`；预览接口直接返回文件内容。这些路由的请求 / 响应契约声明在 `api/manager/knowledge/v1/knowledge.proto`（`LLMWiki*` 消息）。 + +| 方法 | 路径 | 当前行为 | +| --- | --- | --- | +| `GET` | `/v1/knowledge/llm-wiki/tree` | 读取 `layer=raw|wiki`；不传 `parent_id` 返回完整扁平树，传入后过滤该目录子节点 | +| `GET` | `/v1/knowledge/llm-wiki/nodes/{id}` | 读取 Raw 或 active build 页面详情 | +| `GET` | `/v1/knowledge/llm-wiki/nodes/{id}/preview` | Raw PDF 原样返回；Raw DOCX 返回提取后的纯文本 | +| `DELETE` | `/v1/knowledge/llm-wiki/nodes/{id}` | 实际只允许删除 Raw Source | +| `GET` | `/v1/knowledge/llm-wiki/sources` | 查询 Source，支持 `status`、`limit` | +| `GET` | `/v1/knowledge/llm-wiki/search` | 查询 Wiki 索引 | +| `GET` | `/v1/knowledge/llm-wiki/jobs` | 查询最近编译任务 | +| `POST` | `/v1/knowledge/llm-wiki/sync` | 同步组织知识库 `manual/upload/repo` 文档 | +| `POST` | `/v1/knowledge/llm-wiki/compile` | 创建编译任务，返回 202 | +| `POST` | `/v1/knowledge/llm-wiki/jobs/{id}/retry` | 重试 failed/cancelled job | +| `POST` | `/v1/knowledge/llm-wiki/jobs/{id}/cancel` | 请求取消 job | + +`sync`、`compile`、`retry`、`cancel` 使用 `knowledge:doc/write` 中间件；`DELETE nodes/{id}` 使用 `knowledge:doc/delete` 中间件。 + +## 11. 前端 + +- Knowledge 页面加载 Raw 和 Wiki 两棵扁平树并在本地构建目录。 +- Raw 目录继续使用组织知识库的路径显示映射。 +- 生成页显示 active build 的标题；树路径由来源目录和 `page_id` 派生。 +- Raw 文件支持文本/Markdown 查看、PDF 内嵌预览、DOCX 纯文本预览。 +- Raw 文件可以触发单 Source 编译和删除。 +- 页面显示关联 Wiki/Source，可跳转对应节点。 +- Job 面板只显示 pending、running、failed job，并支持取消；failed job 支持重试。 +- 前端没有直接上传 LLM Wiki Raw 的入口；“同步组织知识库”先同步现有组织文档。 + +## 12. 删除语义 + +- 删除 Raw Source 会： + - 删除 Source 和其全部 SourceVersion 行。 + - 删除 Raw 文件。 + - 删除当前 active build 的 artifact 目录、BuildPage 行和 Build 行。 + - 清空该 tenant 的全部 Wiki 词法和向量索引。 +- 当前 HTTP 入口不允许单独删除生成页面。 +- 删除后需要重新同步并编译。 + +## 13. 可观测性与已知差距 + +- `LLMWikiUsageRecorder` 会为每个 job 创建内部 AIOps work session，并在 job 结束时关闭。 +- 每个成功的 compiler LLM 调用都会通过 `TokenUsageSink.Record` 写入一条 assistant token 记录，因此 `/v1/usage/today` 和周报会包含 Wiki 编译用量。 +- usage session 的启动、逐次记录和关闭均为 best-effort；失败只记录 error 日志，不改变编译 job 结果。 +- `manager_metrics.go` 中定义并注册了 `ongrid_llmwiki_*` collectors，但当前 LLM Wiki 编译代码没有引用或递增这些字段。 +- 共享 LLM 客户端自身仍可能产生通用 LLM 调用和 token 指标。 +- 当前没有 `index_failed` stage，也没有独立的编译阶段缓存指标。 +- OpenAI-compatible embedding 用量和 Wiki 编译调用的每日 token budget 不在当前统计范围内。 + +## 14. 安全与数据边界 + +- Raw 和版本快照位于受管 FileStore，路径进入 `resolve` 后统一校验。 +- `safeUploadFileName` 只移除路径分隔符和控制字符，保留 Unicode。 +- `safeRelativeDirectory` 同样保留来源目录语言。 +- 页面来源路径来自已持久化 Source，不接受客户端直接指定 artifact 路径。 +- 当前 HTTP tenant 固定为 `DefaultTenantID`，没有从登录上下文解析租户。 +- Qdrant、词法查询和 build 查询都使用 tenant 条件。 +- 删除 API 的权限边界见第 10 节，`DELETE` 路由使用 `knowledge:doc/delete` 中间件。 + +## 15. 部署、备份与回滚 + +- 文件目录必须持久化；Docker 安装将宿主机 `/llm-wiki` 挂载到 `ONGRID_LLM_WIKI_DIR`。 +- 备份至少覆盖整个 LLM Wiki 文件目录和共享应用数据库。 +- Qdrant 中向量可重建，但生产备份策略应决定是否同时备份 collection。 +- 关闭 `ONGRID_LLM_WIKI_ENABLED` 并重启可停止路由、编译和 Wiki 混合检索分支。 +- 回滚代码时不要删除 Raw、Version 或应用数据库中的 Wiki 表，否则只能重新同步和编译。 +- MySQL down migration 会删除当前 Wiki 表；只能在确认无需保留 catalog 时执行。 + +## 16. 当前验证覆盖 + +自动化测试覆盖： + +- 组织路径和生成页路径生成、遍历防护和无来源路径页面回退。 +- chunk 切分。 +- digest 不暴露数据库 Source ID，并携带文档标题（换行折叠、rune 截断）。 +- evidence resolver 拒绝数据库 Source ID 作为 digest index。 +- 增量编译继承无来源交集的旧页面。 +- 来源语言 Prompt 约束，以及 Planner/Writer Prompt 的禁止虚构约束。 +- 页面 footer 只含分隔线与来源标题，不含固定语言的散文。 +- 混合检索按名次交错、短层让出名额，以及 Wiki-only 检索不受影响。 +- concepts/entities 目录清理。 +- Build、Job、Source 仓储的 SQLite 行为，以及可选 MySQL integration。 +- 词法 tenant 隔离、LIKE 转义、向量失败不降级、point ID 稳定、Qdrant 必填和启动回填。 +- HTTP 编译响应、Raw 预览和删除响应。 +- Token usage recorder 的 session 适配，以及成功 LLM 调用的逐次 usage 记录。 +- 前端 LLM Wiki 树目录显示映射。 + +当前没有使用 fake LLM 跑完整 `Compile` → `Publish` 的 Go 测试，也没有自动化的真实 Qdrant 端到端编译测试。 diff --git a/docs/rfc/RFC-004-llm-wiki.md b/docs/rfc/RFC-004-llm-wiki.md new file mode 100644 index 000000000..8127c50be --- /dev/null +++ b/docs/rfc/RFC-004-llm-wiki.md @@ -0,0 +1,305 @@ +# RFC-004：LLM Wiki 编译、存储与检索实现 + +## 元信息 + +- 状态：已完成 +- 日期：2026-09-11 +- 关联 ADR：[ADR-033：LLM Wiki 架构](../adr/ADR-033-llm-wiki-architecture.md) +- 关联 HLD：[HLD-002：LLM Wiki 可审计知识编译层](../design/HLD-002-llm-wiki.md) + +## 背景 + +LLM Wiki 需要从组织知识库生成可重建的页面，并同时满足： + +- Source 和版本可审计。 +- LLM 输出不能直接决定数据库 ID、页面路径和 build 状态。 +- 单次编译可以只处理选定 Source，同时保留其他现有页面。 +- 页面 artifact、catalog 和派生索引在失败时保持可恢复边界。 +- MySQL/SQLite 共享词法检索语义。 +- 配置 embedder 时使用 Qdrant 做页级向量检索。 + +## 当前架构 + +```mermaid +flowchart LR + ORG["Organization Knowledge"] --> SYNC["SyncOrganizationSources"] + SYNC --> RAW["raw/ + versions/"] + RAW --> JOB["CompileJob"] + JOB --> COMPILER["buildCompiler.Compile"] + COMPILER --> ARTIFACT["wiki/builds//pages/"] + COMPILER --> DB["Application DB"] + COMPILER --> QD["Qdrant ongrid_llm_wiki"] + DB --> TREE["Raw/Wiki Tree"] + DB --> SEARCH["Lexical Search"] + QD --> SEARCH +``` + +## 一、来源同步 + +`POST /v1/knowledge/llm-wiki/sync` 完整分页读取组织知识库的 `manual`、`upload` 和 `repo` 文档。任一类型或任一分页读取失败时中止，不向 Wiki 提交不完整的来源集合。同步由该入口显式触发，代码仓库自身的同步完成后不会自动刷新 Wiki。 + +每个文档： + +- `SourceKey = organization:` +- Raw 路径：`doc.Path/.md` +- 内容写入 Raw 文件 +- 内容写入 `.llm-wiki/versions//.md` +- Source/Version 写入应用数据库 + +`repo` 文档同样作为组织知识库投影写入，Wiki SourceType 保持 `organization`，Raw 目录不增加仓库命名空间。不同仓库的相同 path/title 可能共享 Raw 展示路径，但各 Source 的内容寻址版本快照仍按 SourceKey 隔离。 + +同步是非破坏性的增量刷新：本次出现的 Source 会新增或更新，未出现的 Source、Raw 文件和 active Wiki build 均保留。代码仓库重新索引会短暂清空并回填 Qdrant，因此缺失不能作为删除依据；旧 Raw Source 只能通过显式删除接口移除。响应中的 `deleted` 字段为兼容保留，此同步入口返回 `0`。 + +## 二、版本和身份 + +- Source：`(tenant_id, source_key)` 唯一。 +- SourceVersion：`(tenant_id, source_id, sha256)` 唯一。 +- 新 Source 状态：`pending`。 +- 内容变化且已有版本：`stale`。 +- 内容相同：复用版本；路径变化只更新 Source 元数据。 +- 当前编译成功路径不写 `succeeded`。 +- `Reconcile` 无法恢复 Raw 时写 `failed`。 + +## 三、编译管线 + +### 1. staging build + +`Compile` 首先创建 `WikiBuild{status: staging}`。 + +### 2. Corpus + +- 全部 Source 或 `source_ids` 子集。 +- 跳过无当前版本、小于 100 bytes、内容 hash 重复的文档。 +- 当前实际最多加载 200 个 Source。 + +### 3. Chunk + +- 按 `\n\n` 分段。 +- 目标约 4000 token，使用 `4 bytes/token` 近似。 +- 超长单段不再细分。 + +### 4. digest + +- 小于 32000 token：直接使用 chunk 文本。 +- 大于等于 32000 token：逐 chunk 调用 LLM 生成纯文本摘要。 +- 每个条目渲染为 `--- Digest source_id: <零基下标> ---`，后跟一行 + `Document: <标题>`（文档无标题时省略），再跟正文。 +- Planner 看不到 Corpus 本身，标题是它唯一能判断这段话出自什么材料的依据， + 也是它给页面命名的依据。标题折行会破坏条目分帧，因此换行、制表符折叠为空格， + 并按 rune 截断到 200 字符。 +- 当前没有 cache、batch、层级摘要或 repair。 + +### 5. Planner + +- 首选严格 JSON Schema。 +- 第一次请求失败时回退 `json_object`。 +- 本地校验 pages/page_id/title/sections。 +- `page_id` Prompt 要求 kebab-case，但代码没有格式校验。 +- Prompt 要求标题和正文只能取自 digest 陈述的内容，禁止自行补出 API、参数、 + 数值、日期、人名、角色或步骤；缺信息留空，不用虚构填充。 + +### 6. Evidence + +- section `source_ids` 是 digest 下标。 +- 非法下标失败。 +- 无法映射到 Corpus 的来源被忽略。 +- BuildPage 的 `source_refs_json` 记录 Source、Version、首 chunk ordinal、内容 hash 和来源路径。 + +### 7. Writer + +- 已带 Markdown 结构或超过 3 行的 section 原样保留。 +- 其他 section 调用 LLM 优化，Prompt 要求不得补入来源材料里没有的内容。 +- LLM 失败时回退原始文本。 +- 页面末尾用 `---` 分隔后列出被引用的来源标题与 ID。footer 由代码写出， + 不走模型，因此不会继承来源语言，也不含任何语言的散文—— + 它只有一条分隔线和来源文档自己的标题。 + +### 8. 增量继承 + +当只编译选定 Source 时： + +- 引用所选 Source 的旧页面由新结果替换。 +- 不引用所选 Source 的旧页面复制到新 build。 +- 复制前校验旧 artifact hash。 + +### 9. 校验、索引和发布 + +- 页面文件必须存在且 SHA-256 匹配。 +- 索引失败为 warning，不阻止发布。 +- Build 状态改为 `validated`。 +- `ActivateBuild` 在事务中把旧 active 标记为 `superseded`，把新 build 标记为 `active`。 +- 旧 build artifact 在新 build 激活后删除；数据库行保留。 + +## 四、存储 + +### FileStore + +- `raw/` +- `.llm-wiki/versions/` +- `wiki/builds/` + +`Ensure` 删除 concepts/entities、topics/sources、index/log、staging、schema 等非当前布局路径。 + +### 应用数据库 + +MySQL/SQLite 双方言： + +- `wiki_sources` +- `wiki_source_versions` +- `wiki_compile_jobs` +- `wiki_builds` +- `wiki_build_pages` +- `wiki_lexical` + +`wiki_source_chunks` 和 `wiki_index_meta` 已删除：前者从来没有写入路径，后者只写不读。 + +### Qdrant + +- collection：`ongrid_llm_wiki` +- point ID：tenant + page ID 的 SHA-256 前 8 字节 +- payload 只保存 tenant、page ID、page type、title、body hash + +## 五、检索 + +词法检索： + +- `wiki_lexical` +- `LIKE ... ESCAPE '!'` +- title、aliases、content 匹配 +- aliases 当前没有写入数据 + +向量检索： + +- `title + "\n\n" + content` embedding +- tenant filter +- `body_hash` 未变化时跳过 embedding +- 向量错误直接返回 + +融合： + +RRF（常数 60）只决定顺序，不决定 `score`。 + +- `score` 返回各层自己测得的相似度。raw 层和 Wiki 向量腿是 Qdrant cosine； + Wiki 词法腿在向量腿没有召回该页时，返回三档匹配权重 `(matchRank+1)/3` + （标题 1.0 / aliases 0.67 / 正文 0.33）。两层都命中的页面返回 cosine。 +- 标题命中在原融合值上加 0.005，只影响顺序，不进入 `score`。 +- HybridSearcher 按名次交错两层，Wiki 先：名次 n 的 Wiki 命中后面跟名次 n 的 + raw 命中，谁先取完就把剩余名额让给另一层。 +- `score` 不跨层可比。调用方以列表顺序为准，`score` 只用于层内判断 + （例如协调员 playbook 注入的 0.6 阈值）。 + +历史：合并层曾把 `1.5/(60+rank)` 与 `1.0/(60+rank)` 写回 `score`。这两个值在前 +十名内只跨越一个百分位（0.0246 到 0.0143），因此页面搜索把整屏结果显示成 +`0.02`，协调员的 0.6 阈值也永远无法通过、playbook 注入静默失效。 + +合并顺序也曾用权重合并（Wiki 1.5、Raw 1.0）。任何大于 1.0 的权重都会让名次相同 +的 Wiki 命中全部排在 raw 之前（1.5/61 > 1.0/61），于是 limit 个 Wiki 命中就占满 +结果并挤掉全部 raw 命中——而 raw 是 Wiki 的编译来源，也是知识 prologue 查找的 +对象，不能被挤到看不见。加权名次合并做不到交错，因此改回按名次交错。 + +## 六、任务 + +- 状态：`pending/running/succeeded/skipped/failed/cancelled` +- 页面数为 0：`skipped` +- 其他成功：`succeeded/completed` +- 失败：stage 为失败步骤 +- 取消：立即写 `cancelled`，worker 在阶段边界检查 +- `force` 当前只持久化，不参与执行逻辑 +- 当前没有生产 worker 循环，创建/重试时直接在 goroutine 中执行 +- 进程重启不会自动拉起 pending job +- 过期 running job 可由 ClaimJob 恢复 + +## 七、失败与恢复 + +- 编译失败：Build 标为 `failed`，清理 staging artifact 和 BuildPage/Build 行。 +- 发布前取消：执行相同清理。 +- 索引失败：不回滚已发布页面，不改变 job stage。 +- `Reconcile`：只恢复缺失 Raw 文件，不处理 build manifest。 +- 没有 `index_failed` stage。 + +## 八、前端和 API + +保留的 API 为： + +- `GET tree` +- `GET nodes/{id}` +- `GET nodes/{id}/preview` +- `DELETE nodes/{id}` +- `GET sources` +- `GET search` +- `GET jobs` +- `POST sync` +- `POST compile` +- `POST jobs/{id}/retry` +- `POST jobs/{id}/cancel` + +前端支持 Raw/Wiki 树、详情、PDF/DOCX 预览、单 Source 编译、删除、任务取消/重试和来源跳转。 + +## 备选方案 + +### 五层公共 IR 和 Topic/Evidence pipeline + +可以提供更细的阶段类型，但当前代码没有这些模型和表。 + +### 单 SQLite 和 staging manifest + +提供单目录存储，但当前实现选择共享应用数据库，且没有 manifest reconcile。 + +### 数据库 BLOB 向量 + +无需 Qdrant，但会产生进程内线性扫描。 + +### 只保留词法检索 + +实现简单，但无法提供语义召回。 + +## 影响范围 + +- Biz：`internal/manager/biz/knowledge/llm_wiki/` +- Data：`internal/manager/data/knowledge/llm_wiki/` +- Server：`internal/manager/server/knowledge/llmwiki_http.go` +- Service：`internal/manager/service/knowledge/` +- Frontend：`web/src/features/llm-wiki/` +- Migration：`db/migrations/20260918100000_add_llm_wiki_tables` +- Files：`ONGRID_LLM_WIKI_DIR` + +## 验收标准 + +- Source 同步和版本复用可验证。 +- 编译可按全部或选定 Source 执行。 +- 页面 artifact hash 与数据库一致后才能发布。 +- active build 切换使用数据库事务。 +- 未受影响的旧页面在增量编译中保留。 +- 词法索引隔离 tenant，LIKE 通配符按字面匹配。 +- Qdrant point ID 稳定且 tenant-scoped。 +- 配置 embedder 时 Qdrant 不可用会使 Wiki 初始化失败。 +- Go 相关包 `-race` 测试和前端 typecheck/test 通过。 + +## 实施状态 + +- [x] 组织知识库同步。 +- [x] Source/Version 镜像。 +- [x] Corpus 加载与切块。 +- [x] 短 Corpus 直通与长 Corpus 摘要。 +- [x] Planner JSON Schema/json_object。 +- [x] Evidence resolver。 +- [x] Writer 和来源 footer。 +- [x] Build artifact 和 hash 校验。 +- [x] 增量继承。 +- [x] active build 事务切换。 +- [x] `wiki_lexical` 和 Qdrant 索引。 +- [x] HTTP API 和前端树。 +- [x] 编译阶段逐次调用 `TokenUsageSink.Record`，并复用 AIOps token 总量。 + +## 当前未实现 + +- chunk cache、batch 和层级摘要。 +- Topic/Evidence/Canonical 页面。 +- staging manifest。 +- 自动 worker 消费 pending job。 +- `force_compile` 执行逻辑。 +- Source `succeeded` 状态更新。 +- LLM Wiki 专用 metrics 更新。 +- fake LLM 完整编译测试和真实 Qdrant E2E。 +- OpenAI-compatible embedding token 用量。 +- Wiki 编译调用接入每日 token budget。 diff --git a/internal/manager/biz/aiops/chatruntime/worker.go b/internal/manager/biz/aiops/chatruntime/worker.go index cfcb248f3..98f8f9e03 100644 --- a/internal/manager/biz/aiops/chatruntime/worker.go +++ b/internal/manager/biz/aiops/chatruntime/worker.go @@ -996,11 +996,13 @@ func (rt *Runtime) prologueKBLookup(ctx context.Context, bag []basetool.BaseTool return "" } // Schema across versions has used both "query" and "q"; send - // "query" and let the tool ignore extras. + // "query" and let the tool ignore extras. The count argument is + // "max_results" — "top_k" and "min_score" are not in the tool schema and + // json.Unmarshal dropped them, so this call silently took the tool default + // of 5 and left the earlier "min_score" filtering to the check below. args, _ := json.Marshal(map[string]any{ - "query": userText, - "top_k": 3, - "min_score": 0.6, + "query": userText, + "max_results": 3, }) out, err := kb.InvokableRun(ctx, string(args)) if err != nil || out == "" { @@ -1022,6 +1024,11 @@ func (rt *Runtime) prologueKBLookup(ctx context.Context, bag []basetool.BaseTool if len(parsed.Items) == 0 { return "" } + // The score is the hit's own relevance, not the RRF rank value the hybrid + // searcher used to publish: a raw or Wiki-vector hit reports a Qdrant cosine + // on the same 0..1 scale this threshold was written for. A Wiki page matched + // only by the lexical leg reports its coarse match rank instead, so it clears + // the bar only on a title or alias hit. top := parsed.Items[0] if top.Score < 0.6 { return "" diff --git a/internal/manager/biz/aiops/chatruntime/worker_prologue_test.go b/internal/manager/biz/aiops/chatruntime/worker_prologue_test.go new file mode 100644 index 000000000..3233f439a --- /dev/null +++ b/internal/manager/biz/aiops/chatruntime/worker_prologue_test.go @@ -0,0 +1,105 @@ +package chatruntime + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/ongridio/ongrid/internal/manager/biz/aiops/tools/basetool" +) + +// kbStubTool stands in for query_knowledge: it records the arguments it was +// handed and answers with a scripted result list. +type kbStubTool struct { + name string + body string + argsJSON string + calls int +} + +func (t *kbStubTool) Info(context.Context) (*basetool.ToolInfo, error) { + return &basetool.ToolInfo{Name: t.name, Description: "fake", Class: "read"}, nil +} + +func (t *kbStubTool) InvokableRun(_ context.Context, argsJSON string, _ ...basetool.InvokeOption) (string, error) { + t.calls++ + t.argsJSON = argsJSON + return t.body, nil +} + +// kbResult renders the tool's wire shape for one hit at the given score. +func kbResult(score float64) string { + return fmt.Sprintf(`{"items":[{"title":"DNS 排障","preview":"先看解析器","score":%v}]}`, score) +} + +// TestPrologueKBLookup_SendsArgumentsTheToolDeclares — the prologue sent "top_k" +// and "min_score", neither of which is in the tool's schema, so json.Unmarshal +// dropped both and the call silently took the tool's own default count. +func TestPrologueKBLookup_SendsArgumentsTheToolDeclares(t *testing.T) { + tool := &kbStubTool{name: "query_knowledge", body: kbResult(0.9)} + + _ = (&Runtime{}).prologueKBLookup(context.Background(), []basetool.BaseTool{tool}, "dns 解析失败") + + if tool.calls != 1 { + t.Fatalf("tool calls = %d, want 1", tool.calls) + } + var args map[string]any + if err := json.Unmarshal([]byte(tool.argsJSON), &args); err != nil { + t.Fatalf("args %q: %v", tool.argsJSON, err) + } + if args["query"] != "dns 解析失败" { + t.Errorf("query arg = %v", args["query"]) + } + if args["max_results"] != float64(3) { + t.Errorf("max_results arg = %v, want 3", args["max_results"]) + } + for _, undeclared := range []string{"top_k", "min_score"} { + if _, ok := args[undeclared]; ok { + t.Errorf("arg %q is not in the tool schema and must not be sent", undeclared) + } + } +} + +// TestPrologueKBLookup_InjectsOnlyAboveTheRelevanceBar — the 0.6 bar reads a +// similarity. The hybrid searcher used to overwrite every score with an RRF rank +// value of ~0.02, which made this return "" for every query and silently +// disabled the playbook injection. +func TestPrologueKBLookup_InjectsOnlyAboveTheRelevanceBar(t *testing.T) { + for _, tc := range []struct { + name string + score float64 + want bool + }{ + {name: "cosine above the bar", score: 0.82, want: true}, + {name: "cosine below the bar", score: 0.41, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + tool := &kbStubTool{name: "query_knowledge", body: kbResult(tc.score)} + + got := (&Runtime{}).prologueKBLookup(context.Background(), []basetool.BaseTool{tool}, "dns 解析失败") + + if tc.want && got == "" { + t.Fatal("a relevant top hit must produce a KB block") + } + if !tc.want && got != "" { + t.Fatalf("an irrelevant top hit must produce nothing, got %q", got) + } + }) + } +} + +// TestPrologueKBLookup_IgnoresANonKnowledgeBag — the prologue must not fire +// unless the worker actually carries the tool. +func TestPrologueKBLookup_IgnoresANonKnowledgeBag(t *testing.T) { + tool := &kbStubTool{name: "query_promql", body: kbResult(0.9)} + + got := (&Runtime{}).prologueKBLookup(context.Background(), []basetool.BaseTool{tool}, "dns 解析失败") + + if got != "" { + t.Fatalf("got %q, want no KB block", got) + } + if tool.calls != 0 { + t.Fatalf("unrelated tool was called %d time(s)", tool.calls) + } +} diff --git a/internal/manager/biz/aiops/tools/query_knowledge_basetool.go b/internal/manager/biz/aiops/tools/query_knowledge_basetool.go index 51baec8ae..6dfa26b23 100644 --- a/internal/manager/biz/aiops/tools/query_knowledge_basetool.go +++ b/internal/manager/biz/aiops/tools/query_knowledge_basetool.go @@ -42,6 +42,12 @@ const queryKnowledgeSchema = `{ "type": "string", "description": "Natural language search query (full sentence preferred over keyword bag, e.g. 'DNS 解析失败怎么排查')." }, + "mode": { + "type": "string", + "enum": ["hybrid", "wiki", "rag"], + "default": "hybrid", + "description": "Retrieval layer. hybrid ranks Wiki above Raw while retaining evidence." + }, "path": { "type": "string", "description": "Optional exact path filter (e.g. '网络/DNS'). Empty = no filter. Mutually exclusive with path_prefix." @@ -99,6 +105,7 @@ func (t *QueryKnowledgeTool) Info(_ context.Context) (*basetool.ToolInfo, error) type queryKnowledgeArgs struct { Query string `json:"query"` + Mode string `json:"mode,omitempty"` Path string `json:"path,omitempty"` PathPrefix string `json:"path_prefix,omitempty"` Tags []string `json:"tags,omitempty"` @@ -106,14 +113,18 @@ type queryKnowledgeArgs struct { } type queryKnowledgeHit struct { - ID uint64 `json:"id"` - Title string `json:"title"` - SourceType string `json:"source_type"` - URL string `json:"url,omitempty"` - Path string `json:"path,omitempty"` - Tags []string `json:"tags,omitempty"` - Score float64 `json:"score"` - Preview string `json:"preview"` + ID uint64 `json:"id"` + Title string `json:"title"` + SourceType string `json:"source_type"` + URL string `json:"url,omitempty"` + Path string `json:"path,omitempty"` + Tags []string `json:"tags,omitempty"` + Score float64 `json:"score"` + Preview string `json:"preview"` + Layer string `json:"layer"` + PageType string `json:"page_type,omitempty"` + PageID string `json:"page_id,omitempty"` + SourceVersionID string `json:"source_version_id,omitempty"` } type queryKnowledgeResponse struct { @@ -142,7 +153,14 @@ func (t *QueryKnowledgeTool) InvokableRun(ctx context.Context, argsJSON string, if args.MaxResults > 20 { args.MaxResults = 20 } + if args.Mode == "" { + args.Mode = "hybrid" + } + if args.Mode != "hybrid" && args.Mode != "wiki" && args.Mode != "rag" { + return "", fmt.Errorf("%s: mode must be hybrid, wiki, or rag", ToolNameQueryKnowledge) + } hits, err := t.svc.Search(ctx, args.Query, knowledgebiz.SearchOptions{ + Mode: args.Mode, Path: args.Path, PathPrefix: args.PathPrefix, Tags: args.Tags, @@ -157,19 +175,24 @@ func (t *QueryKnowledgeTool) InvokableRun(ctx context.Context, argsJSON string, // Cap at ~800 chars per hit so a max_results=5 reply stays // under ~4k tokens. The LLM can re-ask for full content via // a follow-up if needed (future doc-fetch tool). - if len(preview) > 800 { - preview = preview[:800] + "…" + previewRunes := []rune(preview) + if len(previewRunes) > 800 { + preview = string(previewRunes[:800]) + "…" out.Truncated = true } out.Items = append(out.Items, queryKnowledgeHit{ - ID: h.Doc.ID, - Title: h.Doc.Title, - SourceType: h.Doc.SourceType, - URL: h.Doc.URL, - Path: h.Doc.Path, - Tags: h.Doc.Tags, - Score: h.Score, - Preview: preview, + ID: h.Doc.ID, + Title: h.Doc.Title, + SourceType: h.Doc.SourceType, + URL: h.Doc.URL, + Path: h.Doc.Path, + Tags: h.Doc.Tags, + Score: h.Score, + Preview: preview, + Layer: knowledgeLayer(h.Layer), + PageType: h.PageType, + PageID: h.PageID, + SourceVersionID: h.SourceVersionID, }) } out.Total = len(out.Items) @@ -179,3 +202,10 @@ func (t *QueryKnowledgeTool) InvokableRun(ctx context.Context, argsJSON string, } return string(body), nil } + +func knowledgeLayer(value string) string { + if value != "" { + return value + } + return "raw" +} diff --git a/internal/manager/biz/knowledge/ingest_test.go b/internal/manager/biz/knowledge/ingest_test.go new file mode 100644 index 000000000..6705f467b --- /dev/null +++ b/internal/manager/biz/knowledge/ingest_test.go @@ -0,0 +1,63 @@ +package knowledge + +import ( + "context" + "testing" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge" +) + +func TestEmbedScannedFiles_UsesOnePipelineForRepoAndVault(t *testing.T) { + for _, testCase := range []struct { + name string + sourceType string + repoID *uint64 + wantRepoID bool + }{ + {name: "repo", sourceType: model.SourceRepo, repoID: ptrU64(7), wantRepoID: true}, + {name: "vault", sourceType: model.SourceVault, wantRepoID: false}, + } { + t.Run(testCase.name, func(t *testing.T) { + vec := &fakeVec{} + u := &Usecase{embed: fakeEmbed{}, vec: vec} + files := []scannedFile{{URL: "docs/guide.md", Title: "Guide", Content: "body"}} + + if err := u.embedScannedFiles(context.Background(), files, testCase.sourceType, testCase.repoID, testTime()); err != nil { + t.Fatalf("embed scanned files: %v", err) + } + if len(vec.upserts) != 1 || len(vec.upserts[0]) != 1 { + t.Fatalf("upserts = %d batches/%d points, want one batch with one point", len(vec.upserts), len(vec.upserts[0])) + } + payload := vec.upserts[0][0].Payload + if payload["source_type"] != testCase.sourceType { + t.Fatalf("source_type = %v, want %q", payload["source_type"], testCase.sourceType) + } + _, hasRepoID := payload["repo_id"] + if hasRepoID != testCase.wantRepoID { + t.Fatalf("repo_id present = %v, want %v", hasRepoID, testCase.wantRepoID) + } + }) + } +} + +func TestEmbedScannedFiles_RejectsMismatchedVectorCount(t *testing.T) { + u := &Usecase{embed: shortEmbed{}, vec: &fakeVec{}} + files := []scannedFile{{URL: "guide.md", Title: "Guide", Content: "body"}} + + if err := u.embedScannedFiles(context.Background(), files, model.SourceVault, nil, testTime()); err == nil { + t.Fatal("accepted an embedding response with fewer vectors than inputs") + } +} + +type shortEmbed struct{} + +func (shortEmbed) Dim() int { return 4 } + +func (shortEmbed) Embed(context.Context, []string) ([][]float32, error) { + return nil, nil +} + +func testTime() time.Time { + return time.Unix(0, 0).UTC() +} diff --git a/internal/manager/biz/knowledge/list_docs_test.go b/internal/manager/biz/knowledge/list_docs_test.go new file mode 100644 index 000000000..4a66c70ed --- /dev/null +++ b/internal/manager/biz/knowledge/list_docs_test.go @@ -0,0 +1,103 @@ +package knowledge + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/ongridio/ongrid/internal/pkg/qdrantx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type scriptedScrollVec struct { + QdrantClient + calls []qdrantx.ScrollOpts + pages map[uint64]*qdrantx.ScrollResult + errAt *uint64 +} + +func (v *scriptedScrollVec) Scroll(_ context.Context, _ string, opts qdrantx.ScrollOpts) (*qdrantx.ScrollResult, error) { + v.calls = append(v.calls, opts) + key := uint64(0) + if opts.Offset != nil { + key = *opts.Offset + } + if v.errAt != nil && key == *v.errAt { + return nil, errors.New("qdrant unavailable") + } + if page, ok := v.pages[key]; ok { + return page, nil + } + return &qdrantx.ScrollResult{}, nil +} + +func listDocHit(id, alias uint64, chunkIndex int, title string) qdrantx.SearchHit { + return qdrantx.SearchHit{ID: id, Payload: map[string]any{ + "source_type": "repo", + "title": title, + "content": title, + "id_alias": alias, + "chunk_index": chunkIndex, + }} +} + +func TestListDocs_AllWalksEveryPageAndDeduplicatesAcrossPages(t *testing.T) { + first := make([]qdrantx.SearchHit, 0, 1000) + first = append(first, listDocHit(10, 1, 1, "tail chunk")) + for id := uint64(2); id <= 1000; id++ { + first = append(first, listDocHit(id, id, 0, fmt.Sprintf("doc-%d", id))) + } + next := uint64(1000) + vec := &scriptedScrollVec{pages: map[uint64]*qdrantx.ScrollResult{ + 0: {Points: first, NextOffset: &next}, + 1000: {Points: []qdrantx.SearchHit{listDocHit(1, 1, 0, "head chunk"), listDocHit(1001, 1001, 0, "doc-1001")}}, + }} + uc := &Usecase{vec: vec} + + docs, err := uc.ListDocs(context.Background(), ListDocsFilter{SourceType: "repo", All: true}) + + require.NoError(t, err) + require.Len(t, docs, 1001) + assert.Equal(t, "head chunk", docs[0].Title) + require.Len(t, vec.calls, 2) + assert.Nil(t, vec.calls[0].Offset) + require.NotNil(t, vec.calls[1].Offset) + assert.Equal(t, uint64(1000), *vec.calls[1].Offset) + assert.Equal(t, 1000, vec.calls[0].Limit) + assert.Equal(t, map[string]any{"source_type": "repo"}, vec.calls[0].MustMatch) +} + +func TestListDocs_BoundedQueryKeepsExistingLimitBehavior(t *testing.T) { + vec := &scriptedScrollVec{pages: map[uint64]*qdrantx.ScrollResult{ + 0: {Points: []qdrantx.SearchHit{ + listDocHit(1, 1, 0, "one"), + listDocHit(2, 2, 0, "two"), + listDocHit(3, 3, 0, "three"), + }}, + }} + uc := &Usecase{vec: vec} + + docs, err := uc.ListDocs(context.Background(), ListDocsFilter{SourceType: "repo", Limit: 2}) + + require.NoError(t, err) + require.Len(t, docs, 2) + require.Len(t, vec.calls, 1) + assert.Equal(t, 16, vec.calls[0].Limit) +} + +func TestListDocs_AllPropagatesLaterPageFailure(t *testing.T) { + next := uint64(7) + vec := &scriptedScrollVec{ + pages: map[uint64]*qdrantx.ScrollResult{0: {Points: []qdrantx.SearchHit{listDocHit(1, 1, 0, "one")}, NextOffset: &next}}, + errAt: &next, + } + uc := &Usecase{vec: vec} + + docs, err := uc.ListDocs(context.Background(), ListDocsFilter{SourceType: "repo", All: true}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "scroll all") + assert.Nil(t, docs) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/artifact.go b/internal/manager/biz/knowledge/llm_wiki/artifact.go new file mode 100644 index 000000000..47ffe3020 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/artifact.go @@ -0,0 +1,116 @@ +package llm_wiki + +import ( + "context" + "fmt" + "os" + "path/filepath" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" +) + +// artifactStore keeps the Markdown bodies of one build in its own directory +// tree, so a failed or superseded build never touches the published Wiki. +type artifactStore struct { + files *FileStore +} + +// newArtifactStore creates a build artifact store rooted at the Wiki file store. +func newArtifactStore(files *FileStore) *artifactStore { + return &artifactStore{files: files} +} + +// buildDir returns the base directory for a build's artifacts. +func (s *artifactStore) buildDir(buildID uint64) string { + return filepath.Join("builds", fmt.Sprintf("%d", buildID)) +} + +// pagePath returns the relative path of one page body inside a build. +func (s *artifactStore) pagePath(buildID uint64, pageID string) string { + return filepath.Join(s.buildDir(buildID), "pages", pageID+".md") +} + +// writePageBody writes a page body and returns its relative path and content hash. +func (s *artifactStore) writePageBody(ctx context.Context, buildID uint64, pageID, content string) (path, hash string, err error) { + if err := ctx.Err(); err != nil { + return "", "", err + } + relativePath := s.pagePath(buildID, pageID) + absolutePath, err := s.files.resolve("wiki", relativePath) + if err != nil { + return "", "", fmt.Errorf("build artifacts: resolve path: %w", err) + } + body := []byte(content) + if err := atomicWrite(absolutePath, body, 0o640); err != nil { + return "", "", fmt.Errorf("build artifacts: write page: %w", err) + } + return relativePath, contentSHA256(body), nil +} + +// readPageBody reads a page body back from the build artifact tree. +func (s *artifactStore) readPageBody(ctx context.Context, buildID uint64, pageID string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + absolutePath, err := s.files.resolve("wiki", s.pagePath(buildID, pageID)) + if err != nil { + return "", fmt.Errorf("build artifacts: resolve path: %w", err) + } + body, err := os.ReadFile(absolutePath) + if err != nil { + return "", fmt.Errorf("build artifacts: read page: %w", err) + } + return string(body), nil +} + +// deleteBuild removes every artifact of one build. +func (s *artifactStore) deleteBuild(ctx context.Context, buildID uint64) error { + if err := ctx.Err(); err != nil { + return err + } + dir, err := s.files.resolve("wiki", s.buildDir(buildID)) + if err != nil { + return fmt.Errorf("build artifacts: resolve dir: %w", err) + } + if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("build artifacts: delete build dir: %w", err) + } + return nil +} + +// validateBuild reports every page whose body is missing or whose hash no longer +// matches the recorded one. +func (s *artifactStore) validateBuild(ctx context.Context, buildID uint64, pages []*model.WikiBuildPage) []string { + var failures []string + + for _, page := range pages { + if page.BuildID != buildID { + failures = append(failures, fmt.Sprintf("page %s has mismatched build_id", page.PageID)) + continue + } + + content, err := s.readPageBody(ctx, buildID, page.PageID) + if err != nil { + failures = append(failures, fmt.Sprintf("page %s: %v", page.PageID, err)) + continue + } + + if actualHash := contentSHA256([]byte(content)); actualHash != page.BodySHA256 { + failures = append(failures, fmt.Sprintf("page %s: hash mismatch (expected %s, got %s)", + page.PageID, page.BodySHA256[:12], actualHash[:12])) + } + } + + return failures +} + +// newIndexDocument converts one build page into a search index document. +func newIndexDocument(page *model.WikiBuildPage, content string) IndexDocument { + return IndexDocument{ + TenantID: page.TenantID, + PageID: page.PageID, + PageType: page.PageType, + Title: page.Title, + Content: content, + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/build.go b/internal/manager/biz/knowledge/llm_wiki/build.go new file mode 100644 index 000000000..dd86b6dd9 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/build.go @@ -0,0 +1,532 @@ +package llm_wiki + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// BuildResult is the outcome of compiling one Wiki build. +type BuildResult struct { + BuildID uint64 + PageCount int + Chunks int + Success bool + // NoContent reports a corpus with nothing to compile. The build is dropped + // and the job ends as skipped rather than publishing an empty Wiki. + NoContent bool + // Stage names the pipeline step a failed build stopped at, so the compile + // job can record where it failed. + Stage string + Error error +} + +// ErrJobCancelled is returned when the owning compile job has a pending +// cancellation request. It stops the compilation before publication. +var ErrJobCancelled = errors.New("llmwiki: compile job cancelled") + +// buildCompiler orchestrates the Wiki build pipeline: it loads the corpus, +// chunks and summarizes it, plans the page structure, resolves evidence and +// writes every page into an isolated build that can then be published. +type buildCompiler struct { + repo BuildRepository + jobID *uint64 + cancelRepo BuildJobCancellationChecker + files *FileStore + artifacts *artifactStore + llm CompilerLLM + summarizer *summarizer + planner *planner + resolver *evidenceResolver + writer *pageWriter + indexer SearchIndex + log *slog.Logger +} + +// newBuildCompiler wires the compilation stages together. jobID and +// cancelRepo together opt a compiler into durable job cancellation checks. +func newBuildCompiler(repo BuildRepository, jobID *uint64, cancelRepo BuildJobCancellationChecker, files *FileStore, llm CompilerLLM, indexer SearchIndex, log *slog.Logger) *buildCompiler { + return &buildCompiler{ + repo: repo, + jobID: jobID, + cancelRepo: cancelRepo, + files: files, + artifacts: newArtifactStore(files), + llm: llm, + summarizer: newSummarizer(llm, log), + planner: newPlanner(llm, log), + resolver: newEvidenceResolver(), + writer: newPageWriter(llm, log), + indexer: indexer, + log: log, + } +} + +// bindJob points cancellation checks at the currently running job. +func (c *buildCompiler) bindJob(jobID uint64) { + c.jobID = &jobID +} + +// withUsageSink returns a per-job compiler whose LLM stages record usage into +// sink. The base compiler stays immutable so concurrent jobs cannot overwrite +// each other's accounting session or cancellation binding. +func (c *buildCompiler) withUsageSink(sink TokenUsageSink) *buildCompiler { + if c == nil || c.llm == nil || sink == nil { + return c + } + clone := *c + clone.jobID = nil + recordingLLM := newUsageRecordingLLM(c.llm, sink, c.log) + clone.summarizer = newSummarizer(recordingLLM, c.log) + clone.planner = newPlanner(recordingLLM, c.log) + clone.writer = newPageWriter(recordingLLM, c.log) + return &clone +} + +// Compile runs the whole pipeline and returns the staging build it produced: +// create build → load corpus → chunk → summarize → plan → resolve sources → +// write artifacts → validate → index. A failed or cancelled build is cleaned +// up before returning. sourceIDs is non-empty only those sources are compiled. +func (c *buildCompiler) Compile(ctx context.Context, tenantID uint64, sourceIDs ...uint64) *BuildResult { + result := &BuildResult{} + + build, err := c.repo.CreateBuild(ctx, tenantID) + if err != nil { + result.Stage = "create build" + result.Error = fmt.Errorf("create build: %w", err) + return result + } + result.BuildID = build.ID + c.log.InfoContext(ctx, "build: started staging build", slog.Uint64("build_id", build.ID)) + + defer func() { + if !result.Success { + if cleanupErr := c.cleanupFailedBuild(ctx, tenantID, build.ID); cleanupErr != nil { + c.log.ErrorContext(ctx, "build: cleanup failed", + slog.Uint64("build_id", build.ID), + slog.String("error", cleanupErr.Error())) + } + } + }() + + fail := func(step string, err error) *BuildResult { + result.Stage = step + result.Error = fmt.Errorf("%s: %w", step, err) + c.failBuild(ctx, tenantID, build.ID, result.Error) + return result + } + + if c.cancelled(ctx, tenantID) { + return fail("cancel build", ErrJobCancelled) + } + + corpus, err := loadCorpus(ctx, c.repo, tenantID, c.log, sourceIDs...) + if err != nil { + return fail("load corpus", err) + } + + loader := newCorpusContentLoader(c.repo, c.files, tenantID) + if err := prepareCorpusChunks(ctx, corpus, loader, c.log); err != nil { + return fail("prepare chunks", err) + } + result.Chunks = len(corpus.Chunks) + if len(corpus.Chunks) == 0 { + // Nothing survived chunking, so there is no evidence to plan from. + // Asking the planner anyway would either invent pages without sources or + // fail on an empty plan, so drop the staging build and let the job finish + // as skipped with the published build untouched. + if cleanupErr := c.cleanupFailedBuild(ctx, tenantID, build.ID); cleanupErr != nil { + c.log.ErrorContext(ctx, "build: failed to clean up empty build", + slog.Uint64("build_id", build.ID), + slog.String("error", cleanupErr.Error())) + } + result.NoContent = true + result.Success = true + c.log.InfoContext(ctx, "build: corpus has no chunks, skipping compilation", + slog.Uint64("build_id", build.ID)) + return result + } + + if c.cancelled(ctx, tenantID) { + return fail("summarize corpus", ErrJobCancelled) + } + digestItems, err := c.summarizer.Summarize(ctx, corpus) + if err != nil { + return fail("summarize corpus", err) + } + + if c.cancelled(ctx, tenantID) { + return fail("plan wiki", ErrJobCancelled) + } + plan, err := c.planner.Plan(ctx, buildDigest(digestItems)) + if err != nil { + return fail("plan wiki", err) + } + + if c.cancelled(ctx, tenantID) { + return fail("resolve evidence", ErrJobCancelled) + } + resolvedPages, err := c.resolver.Resolve(plan, corpus, digestItems) + if err != nil { + return fail("resolve evidence", err) + } + + if c.cancelled(ctx, tenantID) { + return fail("write pages", ErrJobCancelled) + } + pages, err := c.writePages(ctx, tenantID, build.ID, resolvedPages, sourceIDs) + if err != nil { + return fail("write pages", err) + } + result.PageCount = len(pages) + + if c.cancelled(ctx, tenantID) { + return fail("validation failed", ErrJobCancelled) + } + if validationErrors := c.artifacts.validateBuild(ctx, build.ID, pages); len(validationErrors) > 0 { + return fail("validation failed", fmt.Errorf("%v", validationErrors)) + } + + if c.cancelled(ctx, tenantID) { + return fail("mark validated", ErrJobCancelled) + } + if err := c.repo.UpdateBuildStatus(ctx, tenantID, build.ID, model.BuildValidated, ""); err != nil { + result.Stage = "mark validated" + result.Error = fmt.Errorf("mark validated: %w", err) + return result + } + + // Indexing is best effort: a stale search index must not fail a valid build. + if c.cancelled(ctx, tenantID) { + return fail("index build pages", ErrJobCancelled) + } + if err := c.indexBuildPages(ctx, tenantID, build.ID, pages); err != nil { + c.log.WarnContext(ctx, "build: indexing failed (non-fatal)", + slog.Uint64("build_id", build.ID), + slog.String("error", err.Error())) + } + + result.Success = true + c.log.InfoContext(ctx, "build: compilation complete", + slog.Uint64("build_id", build.ID), + slog.Int("pages", result.PageCount), + slog.Int("chunks", result.Chunks)) + + return result +} + +// Publish atomically activates a validated build and drops the artifacts of the +// build it replaces. It rechecks the job cancellation immediately before the +// activation boundary. +func (c *buildCompiler) Publish(ctx context.Context, tenantID, buildID uint64) error { + if c.cancelled(ctx, tenantID) { + if err := c.cleanupFailedBuild(ctx, tenantID, buildID); err != nil { + return fmt.Errorf("clean up cancelled build: %w", err) + } + return ErrJobCancelled + } + + build, err := c.repo.GetBuild(ctx, tenantID, buildID) + if err != nil { + return fmt.Errorf("get build: %w", err) + } + if build.Status != model.BuildValidated { + return fmt.Errorf("cannot publish build in status %q", build.Status) + } + + // A source deleted while this build was compiling must not come back: the + // source-set check that guards deletes races the job creation, so the + // citation check at the activation boundary is what makes it safe. + if err := c.validateCitedSources(ctx, tenantID, buildID); err != nil { + if cleanupErr := c.cleanupFailedBuild(ctx, tenantID, buildID); cleanupErr != nil { + return errors.Join(err, fmt.Errorf("clean up stale build: %w", cleanupErr)) + } + return err + } + + previousBuild, _ := c.repo.GetActiveBuild(ctx, tenantID) + + if err := c.repo.ActivateBuild(ctx, tenantID, buildID); err != nil { + return fmt.Errorf("activate build: %w", err) + } + + if previousBuild != nil { + if err := c.artifacts.deleteBuild(ctx, previousBuild.ID); err != nil { + c.log.WarnContext(ctx, "build: failed to cleanup old build artifacts", + slog.Uint64("old_build_id", previousBuild.ID), + slog.String("error", err.Error())) + } + } + + c.log.InfoContext(ctx, "build: published", slog.Uint64("build_id", buildID)) + return nil +} + +// cancelled checks the owning compile job's durable cancellation flag. It is a +// best-effort gate so cancellation can outpace the caller's context. +func (c *buildCompiler) cancelled(ctx context.Context, tenantID uint64) bool { + if c.jobID == nil || c.cancelRepo == nil { + return false + } + cancelled, err := c.cancelRepo.IsCancelRequested(ctx, tenantID, *c.jobID) + if err != nil { + c.log.ErrorContext(ctx, "build: failed to check job cancellation", + slog.Uint64("job_id", *c.jobID), + slog.String("error", err.Error())) + return false + } + return cancelled +} + +// writePages renders every resolved page to Markdown, stores its body in the +// build artifact tree and records the page rows for the build. Incremental +// builds copy forward pages that do not depend on any selected source. +func (c *buildCompiler) writePages(ctx context.Context, tenantID, buildID uint64, pages []*resolvedPage, selectedSourceIDs []uint64) ([]*model.WikiBuildPage, error) { + buildPages := make([]*model.WikiBuildPage, 0, len(pages)) + + for _, page := range pages { + if err := ctx.Err(); err != nil { + return nil, err + } + + content, err := c.writer.WritePage(ctx, page) + if err != nil { + return nil, fmt.Errorf("write page %s: %w", page.PageID, err) + } + + bodyPath, bodyHash, err := c.artifacts.writePageBody(ctx, buildID, page.PageID, content) + if err != nil { + return nil, fmt.Errorf("write artifact %s: %w", page.PageID, err) + } + + sourceRefs := make([]model.SourceRef, len(page.Sources)) + for i, source := range page.Sources { + sourceRefs[i] = model.SourceRef{ + SourceID: source.SourceID, + SourceVersionID: source.SourceVersionID, + ChunkOrdinal: source.ChunkOrdinal, + ContentHash: source.ContentHash, + SourcePath: source.DocumentTitle, + } + } + sourceRefsJSON, err := json.Marshal(sourceRefs) + if err != nil { + return nil, fmt.Errorf("encode source refs for %s: %w", page.PageID, err) + } + + buildPages = append(buildPages, &model.WikiBuildPage{ + BuildID: buildID, + TenantID: tenantID, + PageID: page.PageID, + PageType: "generated", + Title: page.Title, + BodyPath: bodyPath, + BodySHA256: bodyHash, + SourceRefsJSON: string(sourceRefsJSON), + }) + } + + if len(selectedSourceIDs) > 0 { + inherited, err := inheritUnselectedPages(ctx, c.repo, c.artifacts, tenantID, buildID, selectedSourceIDs, buildPages) + if err != nil { + return nil, fmt.Errorf("inherit unselected pages: %w", err) + } + buildPages = append(buildPages, inherited...) + } + + if err := c.repo.CreatePagesBatch(ctx, buildPages); err != nil { + return nil, fmt.Errorf("create pages: %w", err) + } + if err := c.repo.UpdateBuildPageCount(ctx, tenantID, buildID, len(buildPages)); err != nil { + return nil, fmt.Errorf("update page count: %w", err) + } + + return buildPages, nil +} + +// pageInheritanceRepository is the persistence needed to carry unaffected pages +// from the active build into a new incremental build. +type pageInheritanceRepository interface { + GetActiveBuild(ctx context.Context, tenantID uint64) (*model.WikiBuild, error) + ListPagesByBuild(ctx context.Context, tenantID, buildID uint64) ([]*model.WikiBuildPage, error) +} + +// inheritUnselectedPages copies active pages that do not cite any selected +// source into the new build. Pages whose evidence intersects the selected +// sources are allowed to be replaced by the incremental compile result. +func inheritUnselectedPages( + ctx context.Context, + repo pageInheritanceRepository, + artifacts *artifactStore, + tenantID, buildID uint64, + selectedSourceIDs []uint64, + generated []*model.WikiBuildPage, +) ([]*model.WikiBuildPage, error) { + active, err := repo.GetActiveBuild(ctx, tenantID) + if errors.Is(err, errs.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get active build: %w", err) + } + + oldPages, err := repo.ListPagesByBuild(ctx, tenantID, active.ID) + if err != nil { + return nil, fmt.Errorf("list active build pages: %w", err) + } + + selected := make(map[uint64]struct{}, len(selectedSourceIDs)) + for _, sourceID := range selectedSourceIDs { + selected[sourceID] = struct{}{} + } + generatedIDs := make(map[string]struct{}, len(generated)) + for _, page := range generated { + if page != nil { + generatedIDs[page.PageID] = struct{}{} + } + } + + inherited := make([]*model.WikiBuildPage, 0, len(oldPages)) + for _, page := range oldPages { + if page == nil || page.PageID == "" { + continue + } + if _, replaced := generatedIDs[page.PageID]; replaced { + continue + } + + dependsOnSelected, err := pageDependsOnSources(page.SourceRefsJSON, selected) + if err != nil { + return nil, fmt.Errorf("inspect page %s: %w", page.PageID, err) + } + if dependsOnSelected { + continue + } + + body, err := artifacts.readPageBody(ctx, active.ID, page.PageID) + if err != nil { + return nil, fmt.Errorf("read page %s from active build: %w", page.PageID, err) + } + if actualHash := contentSHA256([]byte(body)); actualHash != page.BodySHA256 { + return nil, fmt.Errorf("page %s hash mismatch in active build", page.PageID) + } + + bodyPath, bodyHash, err := artifacts.writePageBody(ctx, buildID, page.PageID, body) + if err != nil { + return nil, fmt.Errorf("copy page %s: %w", page.PageID, err) + } + + cloned := *page + cloned.ID = 0 + cloned.BuildID = buildID + cloned.BodyPath = bodyPath + cloned.BodySHA256 = bodyHash + inherited = append(inherited, &cloned) + } + + return inherited, nil +} + +// pageDependsOnSources reports whether a serialized source reference list +// intersects the selected source set. +func pageDependsOnSources(raw string, selected map[uint64]struct{}) (bool, error) { + if raw == "" { + return false, nil + } + + var refs []model.SourceRef + if err := json.Unmarshal([]byte(raw), &refs); err != nil { + return false, fmt.Errorf("decode source refs: %w", err) + } + for _, ref := range refs { + if _, ok := selected[ref.SourceID]; ok { + return true, nil + } + } + return false, nil +} + +// indexBuildPages feeds every written page into the search index. +func (c *buildCompiler) indexBuildPages(ctx context.Context, tenantID, buildID uint64, pages []*model.WikiBuildPage) error { + if c.indexer == nil { + return nil + } + if err := c.indexer.Clear(ctx, tenantID); err != nil { + return fmt.Errorf("clear previous build index: %w", err) + } + for _, page := range pages { + content, err := c.artifacts.readPageBody(ctx, buildID, page.PageID) + if err != nil { + return fmt.Errorf("read page %s for indexing: %w", page.PageID, err) + } + + if err := c.indexer.IndexPage(ctx, newIndexDocument(page, content)); err != nil { + return fmt.Errorf("index page %s: %w", page.PageID, err) + } + } + + return nil +} + +// validateCitedSources rejects a build whose pages cite a source that no longer +// exists, so pages generated from a deleted source are never activated. +func (c *buildCompiler) validateCitedSources(ctx context.Context, tenantID, buildID uint64) error { + pages, err := c.repo.ListPagesByBuild(ctx, tenantID, buildID) + if err != nil { + return fmt.Errorf("list build pages: %w", err) + } + + checked := make(map[uint64]struct{}) + for _, page := range pages { + if page == nil || page.SourceRefsJSON == "" { + continue + } + var refs []model.SourceRef + if err := json.Unmarshal([]byte(page.SourceRefsJSON), &refs); err != nil { + return fmt.Errorf("decode source refs of page %s: %w", page.PageID, err) + } + for _, ref := range refs { + if _, seen := checked[ref.SourceID]; seen { + continue + } + checked[ref.SourceID] = struct{}{} + if _, err := c.repo.GetSource(ctx, tenantID, ref.SourceID); err != nil { + if errors.Is(err, errs.ErrNotFound) { + return errors.Join(errs.ErrConflict, fmt.Errorf("source %d was deleted while the build was compiling", ref.SourceID)) + } + return fmt.Errorf("get source %d: %w", ref.SourceID, err) + } + } + } + return nil +} + +// failBuild marks a build as failed with an error message. The write runs on a +// detached context so a compile that ran out of time or was cancelled can still +// record why it stopped. +func (c *buildCompiler) failBuild(ctx context.Context, tenantID, buildID uint64, err error) { + stateCtx, cancel := newStateWriteContext(ctx) + defer cancel() + if updateErr := c.repo.UpdateBuildStatus(stateCtx, tenantID, buildID, model.BuildFailed, err.Error()); updateErr != nil { + c.log.ErrorContext(stateCtx, "build: failed to mark build as failed", + slog.Uint64("build_id", buildID), + slog.String("error", updateErr.Error())) + } +} + +// cleanupFailedBuild removes the artifacts and rows of a failed build. Like +// failBuild it stays writable after cancellation, otherwise a timed-out build +// would leave its staging directory and rows behind forever. +func (c *buildCompiler) cleanupFailedBuild(ctx context.Context, tenantID, buildID uint64) error { + stateCtx, cancel := newStateWriteContext(ctx) + defer cancel() + if err := c.artifacts.deleteBuild(stateCtx, buildID); err != nil { + return err + } + return c.repo.DeleteBuild(stateCtx, tenantID, buildID) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/build_test.go b/internal/manager/biz/knowledge/llm_wiki/build_test.go new file mode 100644 index 000000000..19eab17da --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/build_test.go @@ -0,0 +1,120 @@ +package llm_wiki + +import ( + "context" + "testing" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakePageInheritanceRepo struct { + active *model.WikiBuild + pages []*model.WikiBuildPage +} + +func (r *fakePageInheritanceRepo) GetActiveBuild(context.Context, uint64) (*model.WikiBuild, error) { + if r.active == nil { + return nil, errs.ErrNotFound + } + return r.active, nil +} + +func (r *fakePageInheritanceRepo) ListPagesByBuild(context.Context, uint64, uint64) ([]*model.WikiBuildPage, error) { + return r.pages, nil +} + +func TestInheritUnselectedPages_PreservesUnaffectedHistoricalPages(t *testing.T) { + ctx := context.Background() + files, err := NewFileStore(t.TempDir()) + require.NoError(t, err) + require.NoError(t, files.Ensure(ctx)) + artifacts := newArtifactStore(files) + + const ( + tenantID = uint64(3) + activeID = uint64(7) + newBuildID = uint64(8) + ) + + oldBodies := map[string]string{ + "selected-page": "# Selected\n\nold content\n", + "unaffected-page": "# Unaffected\n\nhistorical content\n", + "legacy-page": "# Legacy\n\nuntracked content\n", + } + oldPages := make([]*model.WikiBuildPage, 0, len(oldBodies)) + for pageID, body := range oldBodies { + _, bodyHash, writeErr := artifacts.writePageBody(ctx, activeID, pageID, body) + require.NoError(t, writeErr) + oldPages = append(oldPages, &model.WikiBuildPage{ + BuildID: activeID, + TenantID: tenantID, + PageID: pageID, + PageType: "generated", + Title: pageID, + BodySHA256: bodyHash, + }) + } + for _, page := range oldPages { + switch page.PageID { + case "selected-page": + page.SourceRefsJSON = `[{"source_id":10,"source_version_id":100,"chunk_ordinal":0,"content_hash":"a"}]` + case "unaffected-page": + page.SourceRefsJSON = `[{"source_id":20,"source_version_id":200,"chunk_ordinal":0,"content_hash":"b"}]` + default: + page.SourceRefsJSON = `[]` + } + } + + repo := &fakePageInheritanceRepo{ + active: &model.WikiBuild{ID: activeID, TenantID: tenantID, Status: model.BuildActive}, + pages: oldPages, + } + generated := []*model.WikiBuildPage{{BuildID: newBuildID, TenantID: tenantID, PageID: "fresh-page"}} + + inherited, err := inheritUnselectedPages(ctx, repo, artifacts, tenantID, newBuildID, []uint64{10}, generated) + require.NoError(t, err) + require.Len(t, inherited, 2) + + byID := make(map[string]*model.WikiBuildPage, len(inherited)) + for _, page := range inherited { + byID[page.PageID] = page + assert.Equal(t, newBuildID, page.BuildID) + body, readErr := artifacts.readPageBody(ctx, newBuildID, page.PageID) + require.NoError(t, readErr) + assert.Equal(t, oldBodies[page.PageID], body) + } + assert.NotContains(t, byID, "selected-page") + assert.Contains(t, byID, "unaffected-page") + assert.Contains(t, byID, "legacy-page") +} + +func TestPageDependsOnSources(t *testing.T) { + selected := map[uint64]struct{}{10: {}} + tests := []struct { + name string + raw string + want bool + wantErr bool + }{ + {name: "empty refs", raw: "", want: false}, + {name: "unselected source", raw: `[{"source_id":20}]`, want: false}, + {name: "selected source", raw: `[{"source_id":10}]`, want: true}, + {name: "mixed sources", raw: `[{"source_id":20},{"source_id":10}]`, want: true}, + {name: "invalid json", raw: `{`, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := pageDependsOnSources(tt.raw, selected) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/contract.go b/internal/manager/biz/knowledge/llm_wiki/contract.go new file mode 100644 index 000000000..7e46e8f16 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/contract.go @@ -0,0 +1,112 @@ +package llm_wiki + +import ( + "context" + "errors" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// Repository is the persistence the Wiki workflow needs from the data layer. +type Repository interface { + UpsertSourceVersion(ctx context.Context, source *model.Source, version *model.SourceVersion) (*model.Source, *model.SourceVersion, bool, error) + ListSources(ctx context.Context, tenantID uint64, status string, limit int) ([]*model.Source, int64, error) + ListSourcesAfter(ctx context.Context, tenantID uint64, status string, afterID uint64, limit int) ([]*model.Source, error) + GetSource(ctx context.Context, tenantID, id uint64) (*model.Source, error) + DeleteSource(ctx context.Context, tenantID, id uint64) error + GetVersion(ctx context.Context, tenantID, id uint64) (*model.SourceVersion, error) + CreateJob(ctx context.Context, job *model.CompileJob) error + ListJobs(ctx context.Context, tenantID uint64, limit int) ([]*model.CompileJob, int64, error) + HasActiveJob(ctx context.Context, tenantID uint64) (bool, error) + GetJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) + RetryJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) + CancelJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) + ClaimJob(ctx context.Context, tenantID uint64, owner string, lease time.Duration) (*model.CompileJob, error) + ClaimJobByID(ctx context.Context, tenantID, id uint64, owner string, lease time.Duration) (*model.CompileJob, error) + UpdateJob(ctx context.Context, tenantID, id uint64, status, stage, errMessage string) error + IsCancelRequested(ctx context.Context, tenantID, id uint64) (bool, error) + MarkSourceStatus(ctx context.Context, tenantID, id uint64, status string) error + CreateBuild(ctx context.Context, tenantID uint64) (*model.WikiBuild, error) + GetBuild(ctx context.Context, tenantID, buildID uint64) (*model.WikiBuild, error) + GetActiveBuild(ctx context.Context, tenantID uint64) (*model.WikiBuild, error) + UpdateBuildStatus(ctx context.Context, tenantID, buildID uint64, status, errorMsg string) error + UpdateBuildPageCount(ctx context.Context, tenantID, buildID uint64, count int) error + ActivateBuild(ctx context.Context, tenantID, buildID uint64) error + DeleteBuild(ctx context.Context, tenantID, buildID uint64) error + + CreatePage(ctx context.Context, page *model.WikiBuildPage) error + CreatePagesBatch(ctx context.Context, pages []*model.WikiBuildPage) error + GetPage(ctx context.Context, tenantID, buildID uint64, pageID string) (*model.WikiBuildPage, error) + ListPagesByBuild(ctx context.Context, tenantID, buildID uint64) ([]*model.WikiBuildPage, error) + DeleteBuildPages(ctx context.Context, tenantID, buildID uint64) error +} + +// BuildRepository is the narrower persistence surface the build compiler needs. +type BuildRepository interface { + ListSources(ctx context.Context, tenantID uint64, status string, limit int) ([]*model.Source, int64, error) + ListSourcesAfter(ctx context.Context, tenantID uint64, status string, afterID uint64, limit int) ([]*model.Source, error) + GetSource(ctx context.Context, tenantID, id uint64) (*model.Source, error) + GetVersion(ctx context.Context, tenantID, id uint64) (*model.SourceVersion, error) + CreateBuild(ctx context.Context, tenantID uint64) (*model.WikiBuild, error) + GetBuild(ctx context.Context, tenantID, buildID uint64) (*model.WikiBuild, error) + GetActiveBuild(ctx context.Context, tenantID uint64) (*model.WikiBuild, error) + UpdateBuildStatus(ctx context.Context, tenantID, buildID uint64, status, errorMsg string) error + UpdateBuildPageCount(ctx context.Context, tenantID, buildID uint64, count int) error + ActivateBuild(ctx context.Context, tenantID, buildID uint64) error + DeleteBuild(ctx context.Context, tenantID, buildID uint64) error + CreatePage(ctx context.Context, page *model.WikiBuildPage) error + CreatePagesBatch(ctx context.Context, pages []*model.WikiBuildPage) error + GetPage(ctx context.Context, tenantID, buildID uint64, pageID string) (*model.WikiBuildPage, error) + ListPagesByBuild(ctx context.Context, tenantID, buildID uint64) ([]*model.WikiBuildPage, error) + DeleteBuildPages(ctx context.Context, tenantID, buildID uint64) error +} + +// BuildJobCancellationChecker exposes the one job cancellation flag the build +// pipeline needs so cancellation stays a Biz-side concern. +type BuildJobCancellationChecker interface { + IsCancelRequested(ctx context.Context, tenantID, jobID uint64) (bool, error) +} + +// CompilerLLM is the single LLM entry point every compilation stage shares. The +// implementation pins provider and model, so the Wiki contract never silently +// follows the interactive chat default. +type CompilerLLM interface { + Complete(ctx context.Context, req llm.ChatReq) (*llm.ChatResp, error) + ModelVersion() string +} + +// TokenUsageRecorder opens the token accounting session of one compile job. +type TokenUsageRecorder interface { + Start(ctx context.Context, jobID uint64) (TokenUsageSink, error) +} + +// TokenUsageSink records the tokens one compile job spends. +type TokenUsageSink interface { + Record(ctx context.Context, usage llm.Usage) error + Close(ctx context.Context) error +} + +// SearchIndex is the Wiki search index. IndexPage and Clear keep the active +// published snapshot in sync; Search answers queries for the API. +type SearchIndex interface { + IndexPage(ctx context.Context, document IndexDocument) error + Clear(ctx context.Context, tenantID uint64) error + Search(ctx context.Context, tenantID uint64, query string, limit int) ([]SearchHit, error) +} + +// ErrNoPendingJob is returned when no compile job is waiting to be claimed. +var ErrNoPendingJob = errors.New("no pending wiki compile job") + +// CompileTriggerOption configures request-triggered compilation. +type CompileTriggerOption struct { + Owner string + Timeout time.Duration +} + +// noopTokenUsageSink is used when no usage recorder is configured. +type noopTokenUsageSink struct{} + +func (noopTokenUsageSink) Record(context.Context, llm.Usage) error { return nil } +func (noopTokenUsageSink) Close(context.Context) error { return nil } diff --git a/internal/manager/biz/knowledge/llm_wiki/corpus.go b/internal/manager/biz/knowledge/llm_wiki/corpus.go new file mode 100644 index 000000000..553b5904a --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/corpus.go @@ -0,0 +1,218 @@ +package llm_wiki + +import ( + "context" + "fmt" + "log/slog" + "sort" + "strings" +) + +// paragraphSeparator is the only boundary the chunker splits on. Keeping it in +// one place keeps paragraph splitting and chunk length accounting in sync. +const paragraphSeparator = "\n\n" + +// minCorpusDocumentBytes skips trivially small snapshots: they carry no +// structure worth turning into Wiki pages. +const minCorpusDocumentBytes = 100 + +// corpusDocument is one deduplicated document loaded from Raw Knowledge. +type corpusDocument struct { + SourceID uint64 + SourceVersionID uint64 + Title string + Content string + ContentHash string +} + +// corpusChunk is a paragraph-aligned piece of a corpusDocument. +type corpusChunk struct { + DocumentIndex int // index into corpus.Documents + Ordinal int // sequential within document + Start int // byte offset in original content + End int // byte offset in original content + Text string // the chunk text +} + +// corpus is the full set of documents compiled into one Wiki build. +type corpus struct { + Documents []corpusDocument + Chunks []corpusChunk +} + +// loadCorpus loads documents from Raw Knowledge and deduplicates them by +// content hash. When sourceIDs is non-empty only those sources are loaded. +// Documents are returned in a deterministic order so repeated builds over the +// same sources produce the same Wiki. +func loadCorpus(ctx context.Context, repo BuildRepository, tenantID uint64, log *slog.Logger, sourceIDs ...uint64) (*corpus, error) { + sources, err := listAllSources(ctx, repo, tenantID) + if err != nil { + return nil, fmt.Errorf("corpus: list sources: %w", err) + } + + var allowSet map[uint64]struct{} + if len(sourceIDs) > 0 { + allowSet = make(map[uint64]struct{}, len(sourceIDs)) + for _, id := range sourceIDs { + allowSet[id] = struct{}{} + } + } + + seenHashes := make(map[string]struct{}, len(sources)) + var documents []corpusDocument + + for _, source := range sources { + if allowSet != nil { + if _, ok := allowSet[source.ID]; !ok { + continue + } + } + + if source.CurrentVersionID == nil { + continue + } + + version, err := repo.GetVersion(ctx, tenantID, *source.CurrentVersionID) + if err != nil { + log.WarnContext(ctx, "corpus: skip source with missing version", + slog.Uint64("source_id", source.ID), + slog.String("error", err.Error())) + continue + } + + if version.SizeBytes < minCorpusDocumentBytes { + continue + } + + if _, duplicate := seenHashes[version.SHA256]; duplicate { + log.DebugContext(ctx, "corpus: skip duplicate document", + slog.Uint64("source_id", source.ID), + slog.String("hash", version.SHA256[:12])) + continue + } + seenHashes[version.SHA256] = struct{}{} + + documents = append(documents, corpusDocument{ + SourceID: source.ID, + SourceVersionID: version.ID, + Title: source.RawPath, + ContentHash: version.SHA256, + }) + } + + sort.Slice(documents, func(i, j int) bool { + return documents[i].SourceID < documents[j].SourceID + }) + + log.InfoContext(ctx, "corpus: loaded documents", + slog.Int("total_sources", len(sources)), + slog.Int("unique_documents", len(documents)), + slog.Any("filter_source_ids", sourceIDs)) + + return &corpus{Documents: documents}, nil +} + +// corpusContentLoader reads document content from the version snapshot on +// demand, so a large corpus never has to be held in memory at once. +type corpusContentLoader struct { + repo BuildRepository + files *FileStore + tenantID uint64 +} + +// newCorpusContentLoader creates a loader for reading document content. +func newCorpusContentLoader(repo BuildRepository, files *FileStore, tenantID uint64) *corpusContentLoader { + return &corpusContentLoader{ + repo: repo, + files: files, + tenantID: tenantID, + } +} + +// loadContent reads the full text of a document from its version snapshot. +func (l *corpusContentLoader) loadContent(ctx context.Context, doc corpusDocument) (string, error) { + version, err := l.repo.GetVersion(ctx, l.tenantID, doc.SourceVersionID) + if err != nil { + return "", fmt.Errorf("corpus: load version %d: %w", doc.SourceVersionID, err) + } + + body, err := l.files.Read(ctx, version.SnapshotPath, MaxSourceBytes) + if err != nil { + return "", fmt.Errorf("corpus: read snapshot %s: %w", version.SnapshotPath, err) + } + + return string(body), nil +} + +// prepareCorpusChunks loads every document and splits it into Wiki-sized chunks. +func prepareCorpusChunks(ctx context.Context, c *corpus, loader *corpusContentLoader, log *slog.Logger) error { + var totalChunks int + + for i := range c.Documents { + content, err := loader.loadContent(ctx, c.Documents[i]) + if err != nil { + return fmt.Errorf("corpus: load document %d: %w", c.Documents[i].SourceID, err) + } + c.Documents[i].Content = content + + chunks := splitDocumentIntoChunks(content, i) + c.Chunks = append(c.Chunks, chunks...) + totalChunks += len(chunks) + } + + log.InfoContext(ctx, "corpus: prepared chunks", + slog.Int("documents", len(c.Documents)), + slog.Int("total_chunks", totalChunks)) + + return nil +} + +// splitDocumentIntoChunks divides content into token-bounded chunks. Chunks end +// on a paragraph boundary whenever one is available inside the size limit. +func splitDocumentIntoChunks(content string, docIndex int) []corpusChunk { + if len(content) == 0 { + return nil + } + + var chunks []corpusChunk + ordinal := 0 + paragraphs := strings.Split(content, paragraphSeparator) + + var currentChunk strings.Builder + chunkStart := 0 + currentOffset := 0 + maxChunkBytes := TargetChunkTokens * charsPerToken + + for _, paragraph := range paragraphs { + paragraphLen := len(paragraph) + len(paragraphSeparator) + + if currentChunk.Len() > 0 && currentChunk.Len()+paragraphLen > maxChunkBytes { + chunks = append(chunks, newCorpusChunk(docIndex, ordinal, chunkStart, currentOffset, currentChunk.String())) + ordinal++ + chunkStart = currentOffset + currentChunk.Reset() + } + + if currentChunk.Len() > 0 { + currentChunk.WriteString(paragraphSeparator) + } + currentChunk.WriteString(paragraph) + currentOffset += paragraphLen + } + + if currentChunk.Len() > 0 { + chunks = append(chunks, newCorpusChunk(docIndex, ordinal, chunkStart, currentOffset, currentChunk.String())) + } + + return chunks +} + +func newCorpusChunk(docIndex, ordinal, start, end int, text string) corpusChunk { + return corpusChunk{ + DocumentIndex: docIndex, + Ordinal: ordinal, + Start: start, + End: end, + Text: strings.TrimSpace(text), + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/corpus_test.go b/internal/manager/biz/knowledge/llm_wiki/corpus_test.go new file mode 100644 index 000000000..5e99e652e --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/corpus_test.go @@ -0,0 +1,69 @@ +package llm_wiki + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSplitDocumentIntoChunks(t *testing.T) { + tests := []struct { + name string + content string + docIndex int + want int // expected number of chunks + }{ + { + name: "empty content", + content: "", + docIndex: 0, + want: 0, + }, + { + name: "short content single chunk", + content: "This is a short document.", + docIndex: 0, + want: 1, + }, + { + name: "multiple paragraphs", + content: `First paragraph with some content. + +Second paragraph with more content. + +Third paragraph with additional content.`, + docIndex: 0, + want: 1, // Should fit in one chunk + }, + { + name: "long content multiple chunks", + content: func() string { + // Generate content that exceeds chunk size + var s string + for i := 0; i < 100; i++ { + s += "This is a paragraph with enough content to simulate real documentation. " + s += "It contains multiple sentences that discuss various technical topics. " + s += "The purpose is to test the chunking algorithm with realistic content.\n\n" + } + return s + }(), + docIndex: 1, + want: 2, // Should split into multiple chunks + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + chunks := splitDocumentIntoChunks(tt.content, tt.docIndex) + assert.LessOrEqual(t, len(chunks), tt.want+1, "chunk count should be close to expected") + + // Verify chunk properties + for i, chunk := range chunks { + assert.Equal(t, tt.docIndex, chunk.DocumentIndex) + assert.Equal(t, i, chunk.Ordinal) + assert.NotEmpty(t, chunk.Text) + assert.Less(t, chunk.Start, chunk.End) + } + }) + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/delete.go b/internal/manager/biz/knowledge/llm_wiki/delete.go new file mode 100644 index 000000000..2f5b9a49c --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/delete.go @@ -0,0 +1,77 @@ +package llm_wiki + +import ( + "context" + "errors" + "fmt" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// DeleteNode deletes one mirrored raw source. Generated Wiki pages belong to an +// immutable build and cannot be deleted individually. +func (u *Usecase) DeleteNode(ctx context.Context, id string) error { + layer, relative, err := decodeNodeID(id) + if err != nil { + return errors.Join(errs.ErrInvalid, errors.New("invalid wiki file")) + } + if layer != "raw" { + return errors.Join(errs.ErrInvalid, errors.New("only raw source files can be deleted")) + } + if err := u.ensureNoActiveCompile(ctx, DefaultTenantID); err != nil { + return err + } + + unlock := u.files.LockArtifacts() + defer unlock() + + return u.deleteSource(ctx, relative) +} + +// deleteSource removes the mirrored source stored at one relative path. +func (u *Usecase) deleteSource(ctx context.Context, relative string) error { + sources, err := listAllSources(ctx, u.repo, DefaultTenantID) + if err != nil { + return fmt.Errorf("llmwiki: list sources for delete: %w", err) + } + for _, candidate := range sources { + if candidate.RawPath == relative { + return u.removeSource(ctx, candidate) + } + } + return errs.ErrNotFound +} + +// removeSource removes one known source, the pages generated from it and its +// index entries. +func (u *Usecase) removeSource(ctx context.Context, source *model.Source) error { + activeBuild, activeErr := u.repo.GetActiveBuild(ctx, DefaultTenantID) + if activeErr != nil && !errors.Is(activeErr, errs.ErrNotFound) { + return activeErr + } + if err := u.repo.DeleteSource(ctx, DefaultTenantID, source.ID); err != nil { + return err + } + var cleanupErr error + if err := u.files.RemoveSourceFile(ctx, source.RawPath); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove raw file: %w", err)) + } + if activeBuild != nil { + if err := newArtifactStore(u.files).deleteBuild(ctx, activeBuild.ID); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } + if err := u.repo.DeleteBuild(ctx, DefaultTenantID, activeBuild.ID); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } + } + if u.indexer != nil { + if err := u.indexer.Clear(ctx, DefaultTenantID); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } + } + if cleanupErr != nil { + return fmt.Errorf("llmwiki: source %q deleted but derived cleanup failed: %w", source.RawPath, cleanupErr) + } + return nil +} diff --git a/internal/manager/biz/knowledge/llm_wiki/evidence.go b/internal/manager/biz/knowledge/llm_wiki/evidence.go new file mode 100644 index 000000000..75c3f17d6 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/evidence.go @@ -0,0 +1,134 @@ +package llm_wiki + +import "fmt" + +// resolvedPage is a planned page with its source references resolved. +type resolvedPage struct { + PageID string + Title string + Sections []resolvedSection + Sources []resolvedSource +} + +// resolvedSection is one section of a resolved page. +type resolvedSection struct { + Heading string + Content string + Sources []resolvedSource +} + +// resolvedSource is one source document a page cites, with the metadata the +// page footer and the build records need. +type resolvedSource struct { + SourceID uint64 + SourceVersionID uint64 + ChunkOrdinal int + ContentHash string + DocumentTitle string +} + +// evidenceResolver turns the digest indices a plan references back into the +// documents and chunks they came from. +type evidenceResolver struct{} + +// newEvidenceResolver creates an evidence resolver. +func newEvidenceResolver() *evidenceResolver { + return &evidenceResolver{} +} + +// Resolve resolves source references for every page in the plan. +func (r *evidenceResolver) Resolve(planned *plan, c *corpus, items []digestItem) ([]*resolvedPage, error) { + index := newCorpusIndex(c) + pages := make([]*resolvedPage, 0, len(planned.Pages)) + + for _, page := range planned.Pages { + resolved, err := resolvePlanPage(page, index, items) + if err != nil { + return nil, fmt.Errorf("resolve page %s: %w", page.PageID, err) + } + pages = append(pages, resolved) + } + + return pages, nil +} + +// resolvePlanPage resolves one planned page: section source indices point at +// digest items, digest items point at source documents. +func resolvePlanPage(planned planPage, index *corpusIndex, items []digestItem) (*resolvedPage, error) { + resolved := &resolvedPage{PageID: planned.PageID, Title: planned.Title} + seenSources := make(map[uint64]struct{}) + + for _, section := range planned.Sections { + resolvedSection := resolvedSection{Heading: section.Heading, Content: section.Content} + + for _, itemIndex := range section.SourceIDs { + sourceIDs, found := resolveDigestReference(itemIndex, items) + if !found { + return nil, fmt.Errorf("invalid source_id %d for section %s", itemIndex, section.Heading) + } + for _, sourceID := range sourceIDs { + if _, duplicate := seenSources[sourceID]; duplicate { + continue + } + seenSources[sourceID] = struct{}{} + + source, found := index.resolve(sourceID) + if !found { + continue + } + resolvedSection.Sources = append(resolvedSection.Sources, source) + resolved.Sources = append(resolved.Sources, source) + } + } + + resolved.Sections = append(resolved.Sections, resolvedSection) + } + + return resolved, nil +} + +// resolveDigestReference accepts only the documented zero-based digest index. +func resolveDigestReference(reference int, items []digestItem) ([]uint64, bool) { + if reference >= 0 && reference < len(items) { + return items[reference].SourceIDs, true + } + return nil, false +} + +// corpusIndex answers "which document and chunk produced this source id?" for a +// whole corpus, so resolving a plan does not rescan every document per section. +type corpusIndex struct { + documents map[uint64]*corpusDocument + firstChunk map[uint64]int +} + +func newCorpusIndex(c *corpus) *corpusIndex { + index := &corpusIndex{ + documents: make(map[uint64]*corpusDocument, len(c.Documents)), + firstChunk: make(map[uint64]int, len(c.Documents)), + } + for i := range c.Documents { + index.documents[c.Documents[i].SourceID] = &c.Documents[i] + } + for _, chunk := range c.Chunks { + sourceID := c.Documents[chunk.DocumentIndex].SourceID + if _, exists := index.firstChunk[sourceID]; !exists { + index.firstChunk[sourceID] = chunk.Ordinal + } + } + return index +} + +func (i *corpusIndex) resolve(sourceID uint64) (resolvedSource, bool) { + doc, found := i.documents[sourceID] + if !found { + return resolvedSource{}, false + } + return resolvedSource{ + SourceID: doc.SourceID, + SourceVersionID: doc.SourceVersionID, + ChunkOrdinal: i.firstChunk[sourceID], + ContentHash: doc.ContentHash, + DocumentTitle: doc.Title, + }, true +} diff --git a/internal/manager/biz/knowledge/llm_wiki/evidence_test.go b/internal/manager/biz/knowledge/llm_wiki/evidence_test.go new file mode 100644 index 000000000..e775bebeb --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/evidence_test.go @@ -0,0 +1,23 @@ +package llm_wiki + +import "testing" + +func TestResolvePlanPageRejectsDatabaseSourceID(t *testing.T) { + c := &corpus{ + Documents: []corpusDocument{{SourceID: 6, SourceVersionID: 9, Title: "alerts.md"}}, + Chunks: []corpusChunk{{DocumentIndex: 0, Ordinal: 0}}, + } + items := []digestItem{{ChunkIndex: 0, SourceIDs: []uint64{6}}} + planned := planPage{PageID: "alerts", Title: "Alerts", Sections: []planSection{{Heading: "Introduction", Content: "content", SourceIDs: []int{6}}}} + + if _, err := resolvePlanPage(planned, newCorpusIndex(c), items); err == nil { + t.Fatal("database source id was accepted as a digest index") + } +} + +func TestBuildDigestExposesOnlyDigestIndices(t *testing.T) { + got := buildDigest([]digestItem{{Text: "alert overview", SourceIDs: []uint64{6}}}) + if got != "--- Digest source_id: 0 ---\nalert overview\n\n" { + t.Fatalf("buildDigest() = %q", got) + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/filestore.go b/internal/manager/biz/knowledge/llm_wiki/filestore.go new file mode 100644 index 000000000..7f5a1b7c1 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/filestore.go @@ -0,0 +1,215 @@ +package llm_wiki + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" +) + +// FileStore owns the on-disk Wiki tree: +// +// raw/ mirrored source files, as uploaded +// wiki/builds/ immutable generated Wiki builds +// .llm-wiki/versions/ content-addressed source snapshots +// +// Every path that enters or leaves the store goes through resolve, which keeps +// reads and writes inside the root and rejects symlinks. +type FileStore struct { + root string + artifactMu sync.Mutex +} + +// NewFileStore opens (but does not create) a Wiki tree at root. +func NewFileStore(root string) (*FileStore, error) { + root = strings.TrimSpace(root) + if root == "" { + return nil, errors.New("llmwiki: root is required") + } + abs, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("llmwiki: resolve root: %w", err) + } + return &FileStore{root: abs}, nil +} + +// Root returns the absolute root of the Wiki tree. +func (s *FileStore) Root() string { return s.root } + +// LockArtifacts serialises access to the Wiki tree and returns the unlock func. +func (s *FileStore) LockArtifacts() func() { + s.artifactMu.Lock() + return s.artifactMu.Unlock +} + +// Ensure creates the current Wiki layout and removes obsolete catalog/manifest +// artifacts. Generated pages exist only below wiki/builds. +func (s *FileStore) Ensure(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + for _, legacy := range []string{ + "concepts", + "entities", + "wiki/concepts", + "wiki/entities", + "wiki/sources", + "wiki/topics", + "wiki/index.md", + "wiki/log.md", + ".llm-wiki/staging", + "schema.md", + } { + path, err := s.resolve("", legacy) + if err != nil { + return err + } + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("llmwiki: remove legacy artifact %s: %w", legacy, err) + } + } + for _, dir := range []string{"raw", "wiki/builds", ".llm-wiki/versions"} { + path, err := s.resolve("", dir) + if err != nil { + return err + } + if err := os.MkdirAll(path, 0o750); err != nil { + return fmt.Errorf("llmwiki: create %s: %w", dir, err) + } + } + return nil +} + +// Read reads one file from the Wiki tree and rejects anything larger than max +// bytes, so a hostile or accidental huge file cannot exhaust memory. +func (s *FileStore) Read(ctx context.Context, relative string, max int64) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + path, err := s.resolve("", relative) + if err != nil { + return nil, err + } + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("llmwiki: open %s: %w", relative, err) + } + defer func() { _ = f.Close() /* read-only cleanup; the read result remains authoritative */ }() + body, err := io.ReadAll(io.LimitReader(f, max+1)) + if err != nil { + return nil, fmt.Errorf("llmwiki: read %s: %w", relative, err) + } + if int64(len(body)) > max { + return nil, fmt.Errorf("llmwiki: file exceeds %d bytes", max) + } + return body, nil +} + +// resolve maps a store-relative path to an absolute path inside the root. An +// empty layer addresses the root itself; otherwise layer is the first path +// element (raw, wiki, .llm-wiki). +func (s *FileStore) resolve(layer, relative string) (string, error) { + if filepath.IsAbs(relative) { + return "", errors.New("llmwiki: absolute path rejected") + } + clean := filepath.Clean(filepath.FromSlash(relative)) + if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", errors.New("llmwiki: path traversal rejected") + } + base := s.root + if layer != "" { + base = filepath.Join(base, layer) + } + target := filepath.Join(base, clean) + if target != base && !strings.HasPrefix(target, base+string(filepath.Separator)) { + return "", errors.New("llmwiki: path escapes root") + } + if info, err := os.Lstat(target); err == nil && info.Mode()&os.ModeSymlink != 0 { + return "", errors.New("llmwiki: symlink target rejected") + } else if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("llmwiki: inspect target: %w", err) + } + for current := filepath.Dir(target); strings.HasPrefix(current, s.root); current = filepath.Dir(current) { + info, err := os.Lstat(current) + if err == nil && info.Mode()&os.ModeSymlink != 0 { + return "", errors.New("llmwiki: symlink path rejected") + } + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return "", fmt.Errorf("llmwiki: inspect path: %w", err) + } + if current == s.root { + break + } + } + return target, nil +} + +// atomicWrite writes body to path through a temporary file and renames it into +// place, so readers never observe a partially written Wiki file. Writing the +// same content again is a no-op. +func atomicWrite(path string, body []byte, mode fs.FileMode) error { + if existing, err := os.ReadFile(path); err == nil && string(existing) == string(body) { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".llmwiki-*") + if err != nil { + return err + } + tmpName := tmp.Name() + keep := false + defer func() { + if !keep { + _ = os.Remove(tmpName) /* best-effort cleanup after the primary write error */ + } + }() + if err := tmp.Chmod(mode); err != nil { + _ = tmp.Close() /* cleanup after the primary chmod error */ + return err + } + if _, err := tmp.Write(body); err != nil { + _ = tmp.Close() /* cleanup after the primary write error */ + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() /* cleanup after the primary sync error */ + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpName, path); err != nil { + return err + } + keep = true + dir, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + defer func() { _ = dir.Close() /* Sync below reports the durability error */ }() + return dir.Sync() +} + +// contentSHA256 is the content hash used for sources, pages and build artifacts. +func contentSHA256(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) +} + +// shortHash derives a short stable directory name from arbitrary text. +func shortHash(text string) string { + sum := sha256.Sum256([]byte(text)) + return hex.EncodeToString(sum[:12]) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/filestore_test.go b/internal/manager/biz/knowledge/llm_wiki/filestore_test.go new file mode 100644 index 000000000..3db9d21c5 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/filestore_test.go @@ -0,0 +1,31 @@ +package llm_wiki + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEnsureRemovesLegacyConceptAndEntityDirectories(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + files, err := NewFileStore(root) + require.NoError(t, err) + require.NoError(t, files.Ensure(ctx)) + + for _, relative := range []string{"concepts", "entities", "wiki/concepts", "wiki/entities"} { + dir := filepath.Join(root, filepath.FromSlash(relative)) + require.NoError(t, os.MkdirAll(dir, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "legacy.md"), []byte("legacy"), 0o640)) + } + + require.NoError(t, files.Ensure(ctx)) + + for _, relative := range []string{"concepts", "entities", "wiki/concepts", "wiki/entities"} { + _, err := os.Stat(filepath.Join(root, filepath.FromSlash(relative))) + require.ErrorIs(t, err, os.ErrNotExist) + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/job.go b/internal/manager/biz/knowledge/llm_wiki/job.go new file mode 100644 index 000000000..bd3ee915c --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/job.go @@ -0,0 +1,294 @@ +package llm_wiki + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// stateWriteTimeout bounds the detached context used to persist a job's final +// state after the request or worker context is already gone. +const stateWriteTimeout = 10 * time.Second + +// jobOutcome is what one compilation produced, as far as the job record cares. +type jobOutcome struct { + noPagesProduced bool +} + +// CreateCompileJob queues one compilation and starts it in the background. +// When sourceIDs is non-empty only those sources are compiled; otherwise the +// full corpus is compiled. A tenant can have at most one active Wiki build job. +func (u *Usecase) CreateCompileJob(ctx context.Context, tenantID uint64, force bool, sourceIDs []uint64) (*model.CompileJob, error) { + sources, _, err := u.repo.ListSources(ctx, tenantID, "", 1) + if err != nil { + return nil, err + } + if len(sources) == 0 { + return nil, errors.Join(errs.ErrInvalid, errors.New("no wiki sources to compile")) + } + + activeKey := model.ActiveJobKey(tenantID) + job := &model.CompileJob{ + TenantID: tenantID, + ActiveKey: &activeKey, + Status: model.JobPending, + Stage: "queued", + ForceCompile: force || len(sourceIDs) > 0, + SourceIDs: formatSourceIDs(sourceIDs), + } + if err := u.repo.CreateJob(ctx, job); err != nil { + return nil, err + } + u.triggerCompile(job) + return job, nil +} + +// RunOnce claims and runs the next pending job. +func (u *Usecase) RunOnce(ctx context.Context, tenantID uint64, owner string, lease time.Duration) error { + job, err := u.repo.ClaimJob(ctx, tenantID, owner, lease) + if err != nil { + return err + } + return u.runJob(ctx, job) +} + +// ListJobs lists the compile jobs of a tenant, newest first. +func (u *Usecase) ListJobs(ctx context.Context, tenantID uint64, limit int) ([]*model.CompileJob, int64, error) { + return u.repo.ListJobs(ctx, tenantID, limit) +} + +// RetryJob re-queues a finished job and starts it again. +func (u *Usecase) RetryJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) { + job, err := u.repo.RetryJob(ctx, tenantID, id) + if err != nil { + return nil, err + } + u.triggerCompile(job) + return job, nil +} + +// CancelJob requests cancellation of a queued or running job. The worker picks +// the request up at its next cancellation check. +func (u *Usecase) CancelJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) { + return u.repo.CancelJob(ctx, tenantID, id) +} + +// RunJob claims and runs one specific job. +func (u *Usecase) RunJob(ctx context.Context, tenantID, jobID uint64, owner string, lease time.Duration) error { + job, err := u.repo.ClaimJobByID(ctx, tenantID, jobID, owner, lease) + if err != nil { + return err + } + return u.runJob(ctx, job) +} + +// triggerCompile runs a freshly created job in one bounded goroutine. The +// lifecycle context is used on purpose instead of the HTTP request context: the +// API returns 202 before compilation finishes, so the request context may +// already be canceled when this goroutine starts. +func (u *Usecase) triggerCompile(job *model.CompileJob) { + if job == nil || u.trigger.Owner == "" { + return + } + go func() { + defer func() { + if recovered := recover(); recovered != nil { + u.log.ErrorContext(u.runCtx, "wiki compile worker panicked", + slog.Any("panic", recovered), + slog.Uint64("job_id", job.ID)) + } + }() + ctx, cancel := context.WithTimeout(u.runCtx, u.trigger.Timeout) + defer cancel() + if err := u.RunJob(ctx, job.TenantID, job.ID, u.trigger.Owner, u.trigger.Timeout+time.Minute); err != nil && + !errors.Is(err, context.Canceled) && !errors.Is(err, ErrNoPendingJob) { + u.log.ErrorContext(ctx, "wiki compile job failed", slog.Uint64("job_id", job.ID), slog.Any("err", err)) + } + }() +} + +// runJob runs one claimed job and always closes its usage session, even when the +// compilation fails. +func (u *Usecase) runJob(ctx context.Context, job *model.CompileJob) error { + if job == nil { + return errors.New("llmwiki: compile job is required") + } + if u.summarizer == nil { + return u.failJob(ctx, job, "summarize", errors.New("llmwiki: summarizer is not configured")) + } + usageSink := u.startUsageSession(ctx, job.ID) + defer func() { + persistCtx, cancel := newStateWriteContext(ctx) + defer cancel() + u.closeUsageSession(persistCtx, usageSink) + }() + + outcome, err := u.runCompilation(ctx, job, usageSink) + if err != nil { + return err + } + return u.finishJob(ctx, job, outcome) +} + +// startUsageSession opens the token accounting session of a job, falling back to +// a no-op sink when no recorder is configured or session creation fails. +func (u *Usecase) startUsageSession(ctx context.Context, jobID uint64) TokenUsageSink { + if u.usage == nil { + return noopTokenUsageSink{} + } + sink, err := u.usage.Start(ctx, jobID) + if err != nil { + u.log.ErrorContext(ctx, "llmwiki: start token usage session failed", + slog.Uint64("job_id", jobID), + slog.Any("err", err)) + return noopTokenUsageSink{} + } + if sink == nil { + u.log.ErrorContext(ctx, "llmwiki: token usage recorder returned a nil sink", + slog.Uint64("job_id", jobID)) + return noopTokenUsageSink{} + } + return sink +} + +// closeUsageSession closes the usage session without letting accounting +// failures change the compile job's outcome. +func (u *Usecase) closeUsageSession(ctx context.Context, sink TokenUsageSink) { + if sink == nil { + return + } + if err := sink.Close(ctx); err != nil { + u.log.ErrorContext(ctx, "llmwiki: close token usage session failed", slog.Any("err", err)) + } +} + +// runCompilation compiles the tenant's full corpus into a build and publishes it. +func (u *Usecase) runCompilation(ctx context.Context, job *model.CompileJob, usageSink TokenUsageSink) (jobOutcome, error) { + outcome := jobOutcome{noPagesProduced: true} + + if u.compiler == nil { + u.log.WarnContext(ctx, "llmwiki: BuildCompiler not available, skipping compilation") + return outcome, nil + } + + compiler := u.compiler.withUsageSink(usageSink) + compiler.bindJob(job.ID) + sourceIDs := parseSourceIDs(job.SourceIDs) + buildResult := compiler.Compile(ctx, job.TenantID, sourceIDs...) + if !buildResult.Success { + return outcome, u.failJob(ctx, job, buildResult.Stage, buildResult.Error) + } + if buildResult.NoContent { + // Nothing to compile: the outcome already reports no pages, so the job + // finishes as skipped instead of publishing an empty build. + return outcome, nil + } + if err := compiler.Publish(ctx, job.TenantID, buildResult.BuildID); err != nil { + return outcome, u.failJob(ctx, job, "publish", err) + } + + outcome.noPagesProduced = buildResult.PageCount == 0 + return outcome, nil +} + +// parseSourceIDs parses a JSON array of source IDs from the job's SourceIDs field. +func parseSourceIDs(raw *string) []uint64 { + if raw == nil || *raw == "" { + return nil + } + var ids []uint64 + if err := json.Unmarshal([]byte(*raw), &ids); err != nil { + return nil + } + return ids +} + +// formatSourceIDs serialises source IDs to a JSON array string. +func formatSourceIDs(ids []uint64) *string { + if len(ids) == 0 { + return nil + } + b, err := json.Marshal(ids) + if err != nil { + return nil + } + s := string(b) + return &s +} + +// finishJob records the terminal status of a successful job. +func (u *Usecase) finishJob(ctx context.Context, job *model.CompileJob, outcome jobOutcome) error { + status, stage := model.JobSucceeded, "completed" + if outcome.noPagesProduced { + status, stage = model.JobSkipped, "skipped" + } + stateCtx, cancel := newStateWriteContext(ctx) + defer cancel() + return u.repo.UpdateJob(stateCtx, job.TenantID, job.ID, status, stage, "") +} + +// failJob records the terminal status of a failed job and returns the cause. A +// job that already reached a terminal state keeps it: CancelJob persists +// `cancelled` as soon as the request arrives, and a cancelled job must not be +// rewritten as failed when the cancellation reaches the pipeline. +func (u *Usecase) failJob(ctx context.Context, job *model.CompileJob, stage string, cause error) error { + stateCtx, cancel := newStateWriteContext(ctx) + defer cancel() + current, err := u.repo.GetJob(stateCtx, job.TenantID, job.ID) + if err == nil && isJobTerminal(current.Status) { + return cause + } + if updateErr := u.repo.UpdateJob(stateCtx, job.TenantID, job.ID, model.JobFailed, stage, truncateErrorMessage(cause)); updateErr != nil { + return errors.Join(cause, updateErr) + } + return cause +} + +// ensureNoActiveCompile refuses an operation that would change the source set +// while a compile job is queued or running. The compiler reads the source set +// once and publishes what it read, so a source removed mid-build would either +// be republished or fail the build halfway. +func (u *Usecase) ensureNoActiveCompile(ctx context.Context, tenantID uint64) error { + active, err := u.repo.HasActiveJob(ctx, tenantID) + if err != nil { + return fmt.Errorf("llmwiki: check active compile job: %w", err) + } + if active { + return errors.Join(errs.ErrConflict, errors.New("a wiki compile job is running")) + } + return nil +} + +// isJobTerminal reports whether a job already reached a state it cannot leave. +func isJobTerminal(status string) bool { + switch status { + case model.JobSucceeded, model.JobSkipped, model.JobFailed, model.JobCancelled: + return true + default: + return false + } +} + +// newStateWriteContext detaches from the caller's cancellation and bounds how +// long writing a job's final state may take. +func newStateWriteContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + return context.WithTimeout(context.WithoutCancel(ctx), stateWriteTimeout) +} + +// truncateErrorMessage fits an error into the job table's error column. +func truncateErrorMessage(err error) string { + value := err.Error() + if len(value) > 2048 { + return value[:2048] + } + return value +} diff --git a/internal/manager/biz/knowledge/llm_wiki/job_test.go b/internal/manager/biz/knowledge/llm_wiki/job_test.go new file mode 100644 index 000000000..145bd8072 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/job_test.go @@ -0,0 +1,477 @@ +package llm_wiki + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeWikiRepo overrides only the calls a test exercises. The embedded +// Repository is nil, so reaching an unexpected dependency panics instead of +// silently answering with zero values. +type fakeWikiRepo struct { + Repository + + job *model.CompileJob + updates []jobUpdate + hasActive bool + cancelRequested bool + createBuildErr error + build *model.WikiBuild + activeBuild *model.WikiBuild + pages []*model.WikiBuildPage + sources []*model.Source + version *model.SourceVersion + versions map[uint64]*model.SourceVersion + failedSources []uint64 + sourceErr error + deletedSources []uint64 + deletedBuild uint64 + createBuildID uint64 + buildStatus string + buildStatusMsg string + activated bool + nextVersionID uint64 +} + +type jobUpdate struct { + status string + stage string + message string +} + +func (r *fakeWikiRepo) GetJob(context.Context, uint64, uint64) (*model.CompileJob, error) { + if r.job == nil { + return nil, errs.ErrNotFound + } + return r.job, nil +} + +func (r *fakeWikiRepo) UpdateJob(_ context.Context, _, _ uint64, status, stage, message string) error { + r.updates = append(r.updates, jobUpdate{status: status, stage: stage, message: message}) + if r.job != nil { + r.job.Status = status + } + return nil +} + +func (r *fakeWikiRepo) HasActiveJob(context.Context, uint64) (bool, error) { + return r.hasActive, nil +} + +func (r *fakeWikiRepo) IsCancelRequested(context.Context, uint64, uint64) (bool, error) { + return r.cancelRequested, nil +} + +func (r *fakeWikiRepo) CreateBuild(context.Context, uint64) (*model.WikiBuild, error) { + if r.createBuildErr != nil { + return nil, r.createBuildErr + } + buildID := r.createBuildID + if buildID == 0 { + buildID = 1 + } + return &model.WikiBuild{ID: buildID}, nil +} + +func (r *fakeWikiRepo) UpdateBuildStatus(_ context.Context, _ uint64, _ uint64, status, message string) error { + r.buildStatus = status + r.buildStatusMsg = message + return nil +} + +func (r *fakeWikiRepo) GetBuild(context.Context, uint64, uint64) (*model.WikiBuild, error) { + return r.build, nil +} + +func (r *fakeWikiRepo) GetActiveBuild(context.Context, uint64) (*model.WikiBuild, error) { + if r.activeBuild != nil { + return r.activeBuild, nil + } + return nil, errs.ErrNotFound +} + +func (r *fakeWikiRepo) ActivateBuild(context.Context, uint64, uint64) error { + r.activated = true + return nil +} + +func (r *fakeWikiRepo) DeleteBuild(_ context.Context, _ uint64, buildID uint64) error { + r.deletedBuild = buildID + return nil +} + +func (r *fakeWikiRepo) ListPagesByBuild(context.Context, uint64, uint64) ([]*model.WikiBuildPage, error) { + return r.pages, nil +} + +func (r *fakeWikiRepo) GetSource(context.Context, uint64, uint64) (*model.Source, error) { + if r.sourceErr != nil { + return nil, r.sourceErr + } + return &model.Source{ID: 1}, nil +} + +func (r *fakeWikiRepo) DeleteSource(_ context.Context, _ uint64, id uint64) error { + r.deletedSources = append(r.deletedSources, id) + return nil +} + +func (r *fakeWikiRepo) GetVersion(_ context.Context, _ uint64, id uint64) (*model.SourceVersion, error) { + if r.versions != nil { + if version, ok := r.versions[id]; ok { + return version, nil + } + return nil, errs.ErrNotFound + } + if r.version == nil { + return nil, errs.ErrNotFound + } + return r.version, nil +} + +func (r *fakeWikiRepo) MarkSourceStatus(_ context.Context, _, id uint64, status string) error { + if status == model.SourceFailed { + r.failedSources = append(r.failedSources, id) + } + return nil +} + +func (r *fakeWikiRepo) UpsertSourceVersion(_ context.Context, source *model.Source, version *model.SourceVersion) (*model.Source, *model.SourceVersion, bool, error) { + for index, existing := range r.sources { + if existing.TenantID != source.TenantID || existing.SourceKey != source.SourceKey { + continue + } + changed := existing.ContentSHA256 != source.ContentSHA256 + stored := *source + stored.ID = existing.ID + if changed || existing.CurrentVersionID == nil { + r.nextVersionID++ + version.ID = r.nextVersionID + stored.CurrentVersionID = &version.ID + } else { + stored.CurrentVersionID = existing.CurrentVersionID + } + r.sources[index] = &stored + return &stored, version, changed, nil + } + + stored := *source + stored.ID = uint64(len(r.sources) + 1) + r.nextVersionID++ + version.ID = r.nextVersionID + stored.CurrentVersionID = &version.ID + r.sources = append(r.sources, &stored) + return &stored, version, true, nil +} + +// ListSources mimics the store: it answers with one capped page, so a test that +// needs every source has to go through ListSourcesAfter like the usecase does. +func (r *fakeWikiRepo) ListSources(context.Context, uint64, string, int) ([]*model.Source, int64, error) { + limit := 200 + if limit > len(r.sources) { + limit = len(r.sources) + } + return r.sources[:limit], int64(len(r.sources)), nil +} + +func (r *fakeWikiRepo) ListSourcesAfter(_ context.Context, _ uint64, _ string, afterID uint64, limit int) ([]*model.Source, error) { + page := make([]*model.Source, 0, limit) + for _, source := range r.sources { + if source.ID <= afterID { + continue + } + page = append(page, source) + if len(page) == limit { + break + } + } + return page, nil +} + +func newTestUsecase(t *testing.T, repo Repository) (*Usecase, *FileStore) { + t.Helper() + files, err := NewFileStore(t.TempDir()) + require.NoError(t, err) + uc, err := newUsecase(context.Background(), repo, files, mustNotCallLLM(), nil, slog.New(slog.NewTextHandler(io.Discard, nil)), nil) + require.NoError(t, err) + return uc, files +} + +// seedMirroredSource registers one source whose version snapshot exists on disk. +// A size below the corpus minimum makes the document drop out of chunking, which +// is how a corpus ends up empty. +func seedMirroredSource(t *testing.T, repo *fakeWikiRepo, files *FileStore, sizeBytes int, content string) { + t.Helper() + snapshot := "versions/" + shortHash(content) + ".md" + absolute, err := files.resolve("", snapshot) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(absolute), 0o750)) + require.NoError(t, os.WriteFile(absolute, []byte(content), 0o640)) + + versionID := uint64(11) + repo.version = &model.SourceVersion{ID: versionID, TenantID: DefaultTenantID, SHA256: contentSHA256([]byte(content)), SizeBytes: uint64(sizeBytes), SnapshotPath: snapshot} + repo.sources = []*model.Source{{ + ID: 7, + TenantID: DefaultTenantID, + SourceKey: "organization:7", + SourceType: "organization", + RawPath: "docs/runbook.md", + ContentSHA256: contentSHA256([]byte(content)), + Status: model.SourcePending, + CurrentVersionID: &versionID, + }} +} + +// mustNotCallLLM fails a test loudly if a stage reaches the model. +func mustNotCallLLM() CompilerLLM { + return compilerLLMFunc{ + version: "test", + call: func(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return nil, errors.New("llm must not be called") + }, + } +} + +// TestRunJob_MarksJobFailedAndRecordsStage — a job that fails must reach the +// failed terminal state with the step it stopped at, instead of staying running +// forever and blocking every later compile. +func TestRunJob_MarksJobFailedAndRecordsStage(t *testing.T) { + repo := &fakeWikiRepo{ + job: &model.CompileJob{ID: 5, TenantID: DefaultTenantID, Status: model.JobRunning, Stage: "queued"}, + createBuildErr: errors.New("mysql is down"), + } + uc, _ := newTestUsecase(t, repo) + + err := uc.runJob(context.Background(), repo.job) + + require.Error(t, err) + require.Len(t, repo.updates, 1) + assert.Equal(t, model.JobFailed, repo.updates[0].status) + assert.Equal(t, "create build", repo.updates[0].stage) + assert.Contains(t, repo.updates[0].message, "mysql is down") +} + +// TestRunJob_KeepsCancelledJobTerminal — CancelJob persists `cancelled` as soon +// as the request arrives, so the pipeline failing afterwards must not rewrite it. +func TestRunJob_KeepsCancelledJobTerminal(t *testing.T) { + repo := &fakeWikiRepo{ + job: &model.CompileJob{ID: 6, TenantID: DefaultTenantID, Status: model.JobCancelled}, + createBuildErr: errors.New("mysql is down"), + } + uc, _ := newTestUsecase(t, repo) + + require.Error(t, uc.runJob(context.Background(), repo.job)) + + assert.Empty(t, repo.updates) + assert.Equal(t, model.JobCancelled, repo.job.Status) +} + +// TestDeleteNode_RefusesWhileCompileIsRunning — the compiler reads the source +// set once, so a delete that lands mid-build is refused rather than raced. +func TestDeleteNode_RefusesWhileCompileIsRunning(t *testing.T) { + repo := &fakeWikiRepo{hasActive: true} + uc, _ := newTestUsecase(t, repo) + + err := uc.DeleteNode(context.Background(), encodeNodeID("raw", "docs/runbook.md")) + + require.Error(t, err) + assert.ErrorIs(t, err, errs.ErrConflict) + assert.Empty(t, repo.deletedSources) +} + +func TestSyncOrganizationSources_RefusesWhileCompileIsRunning(t *testing.T) { + repo := &fakeWikiRepo{hasActive: true} + uc, _ := newTestUsecase(t, repo) + + result, err := uc.SyncOrganizationSources(context.Background(), []OrganizationSource{{ID: 1, Title: "Runbook", Path: "docs", Content: "body"}}) + + require.Error(t, err) + assert.ErrorIs(t, err, errs.ErrConflict) + assert.Nil(t, result) +} + +// TestSyncOrganizationSources_RepositoryDocumentKeepsOrganizationIdentity — +// repository documents enter this usecase through the organization projection. +// Their Qdrant document ID remains the stable Wiki identity, while the raw path +// follows the same path/title layout as manual and uploaded documents. +func TestSyncOrganizationSources_RepositoryDocumentKeepsOrganizationIdentity(t *testing.T) { + repo := &fakeWikiRepo{} + uc, files := newTestUsecase(t, repo) + doc := OrganizationSource{ID: 99, Title: "main", Path: "cmd", Content: "package main"} + + created, err := uc.SyncOrganizationSources(context.Background(), []OrganizationSource{doc}) + require.NoError(t, err) + assert.Equal(t, 1, created.Created) + require.Len(t, repo.sources, 1) + assert.Equal(t, "organization:99", repo.sources[0].SourceKey) + assert.Equal(t, "organization", repo.sources[0].SourceType) + assert.Equal(t, "cmd/main.md", repo.sources[0].RawPath) + rawPath, err := files.resolve("raw", repo.sources[0].RawPath) + require.NoError(t, err) + body, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, "package main", string(body)) + + unchanged, err := uc.SyncOrganizationSources(context.Background(), []OrganizationSource{doc}) + require.NoError(t, err) + assert.Equal(t, 1, unchanged.Unchanged) + + doc.Content = "package main\n\nfunc main() {}" + updated, err := uc.SyncOrganizationSources(context.Background(), []OrganizationSource{doc}) + require.NoError(t, err) + assert.Equal(t, 1, updated.Updated) + + preserved, err := uc.SyncOrganizationSources(context.Background(), nil) + require.NoError(t, err) + assert.Zero(t, preserved.Deleted) + assert.Empty(t, repo.deletedSources) +} + +// TestPublish_RefusesBuildWhoseSourceWasDeleted — the source-set check races job +// creation, so the citation check at the activation boundary is what keeps a +// deleted source from being republished. +func TestPublish_RefusesBuildWhoseSourceWasDeleted(t *testing.T) { + repo := &fakeWikiRepo{ + build: &model.WikiBuild{ID: 9, TenantID: DefaultTenantID, Status: model.BuildValidated}, + pages: []*model.WikiBuildPage{{ + BuildID: 9, + TenantID: DefaultTenantID, + PageID: "dns-overview", + SourceRefsJSON: `[{"source_id":42,"source_version_id":1,"chunk_ordinal":0,"content_hash":"a"}]`, + }}, + sourceErr: errs.ErrNotFound, + } + files, err := NewFileStore(t.TempDir()) + require.NoError(t, err) + require.NoError(t, files.Ensure(context.Background())) + compiler := newBuildCompiler(repo, nil, nil, files, mustNotCallLLM(), nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + + err = compiler.Publish(context.Background(), DefaultTenantID, 9) + + require.Error(t, err) + assert.ErrorIs(t, err, errs.ErrConflict) + assert.False(t, repo.activated) + assert.Equal(t, uint64(9), repo.deletedBuild) +} + +// TestSyncOrganizationSources_PreservesSourcesMissingFromSnapshot — knowledge +// repository sync clears and repopulates Qdrant, so a snapshot can temporarily +// omit every document. Missing entries must not delete Raw sources or the active +// Wiki build, including sources beyond the first metadata page. +func TestSyncOrganizationSources_PreservesSourcesMissingFromSnapshot(t *testing.T) { + const total = 205 + repo := &fakeWikiRepo{activeBuild: &model.WikiBuild{ID: 77, TenantID: DefaultTenantID, Status: model.BuildActive}} + for i := 1; i <= total; i++ { + repo.sources = append(repo.sources, &model.Source{ + ID: uint64(i), + TenantID: DefaultTenantID, + SourceKey: fmt.Sprintf("organization:%d", i), + SourceType: "organization", + RawPath: fmt.Sprintf("docs/%03d.md", i), + Status: model.SourcePending, + }) + } + uc, files := newTestUsecase(t, repo) + rawPath, err := files.resolve("raw", repo.sources[0].RawPath) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(rawPath), 0o750)) + require.NoError(t, os.WriteFile(rawPath, []byte("existing raw"), 0o640)) + artifacts := newArtifactStore(files) + wikiPath, _, err := artifacts.writePageBody(context.Background(), repo.activeBuild.ID, "existing-wiki", "existing wiki") + require.NoError(t, err) + wikiAbsolute, err := files.resolve("wiki", wikiPath) + require.NoError(t, err) + + // The knowledge-base snapshot is temporarily empty during re-indexing. + result, err := uc.SyncOrganizationSources(context.Background(), nil) + + require.NoError(t, err) + assert.Zero(t, result.Deleted) + assert.Empty(t, repo.deletedSources) + assert.Zero(t, repo.deletedBuild, "sync must not invalidate the published Wiki") + assert.FileExists(t, rawPath) + assert.FileExists(t, wikiAbsolute) +} + +// TestReconcile_SkipsSourcesWhoseVersionRowIsGone — reconcile used to return the +// missing-version error, which failed Wiki construction and left every route +// unregistered, so the SPA answered 404. One broken source must not stop the +// Wiki from starting, or nobody can re-sync the source that broke it. +func TestReconcile_SkipsSourcesWhoseVersionRowIsGone(t *testing.T) { + repo := &fakeWikiRepo{} + uc, files := newTestUsecase(t, repo) + + // The healthy source's snapshot exists on disk, its raw file does not. + snapshot := "versions/" + shortHash("runbook body") + ".md" + absolute, err := files.resolve("", snapshot) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(absolute), 0o750)) + require.NoError(t, os.WriteFile(absolute, []byte("runbook body"), 0o640)) + + orphanVersionID := uint64(99) + healthyVersionID := uint64(21) + repo.sources = []*model.Source{ + {ID: 1, TenantID: DefaultTenantID, SourceKey: "organization:1", RawPath: "docs/orphan.md", CurrentVersionID: &orphanVersionID}, + {ID: 2, TenantID: DefaultTenantID, SourceKey: "organization:2", RawPath: "docs/runbook.md", CurrentVersionID: &healthyVersionID}, + } + repo.versions = map[uint64]*model.SourceVersion{ + healthyVersionID: {ID: healthyVersionID, TenantID: DefaultTenantID, SnapshotPath: snapshot, SizeBytes: 12}, + } + + require.NoError(t, uc.Reconcile(context.Background(), DefaultTenantID)) + + assert.Equal(t, []uint64{1}, repo.failedSources, "the source with no version row is marked failed") + rawPath, err := files.resolve("raw", "docs/runbook.md") + require.NoError(t, err) + body, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, "runbook body", string(body), "a source after the broken one is still restored") +} + +// TestCompile_SkipsWhenTheCorpusHasNoChunks — a corpus that chunks to nothing has +// no evidence to plan from. Asking the planner anyway would either invent pages +// without sources or fail on an empty plan, and publishing would wipe the Wiki. +func TestCompile_SkipsWhenTheCorpusHasNoChunks(t *testing.T) { + repo := &fakeWikiRepo{createBuildID: 4} + uc, files := newTestUsecase(t, repo) + // Below the corpus minimum, so the document never reaches the chunker. + seedMirroredSource(t, repo, files, 20, "tiny") + job := &model.CompileJob{ID: 9, TenantID: DefaultTenantID, Status: model.JobRunning} + + require.NoError(t, uc.runJob(context.Background(), job)) + + require.Len(t, repo.updates, 1) + assert.Equal(t, model.JobSkipped, repo.updates[0].status) + assert.False(t, repo.activated, "an empty corpus must not publish a build") + assert.Equal(t, uint64(4), repo.deletedBuild, "the empty staging build must be dropped") +} + +// TestCompile_CleansUpBuildAfterContextCancellation — cleanup has to survive the +// context that cancelled the compile, otherwise a timed-out build leaves its +// staging directory and rows behind forever. +func TestCompile_CleansUpBuildAfterContextCancellation(t *testing.T) { + repo := &fakeWikiRepo{createBuildID: 5} + _, files := newTestUsecase(t, repo) + seedMirroredSource(t, repo, files, 400, strings.Repeat("content ", 50)) + compiler := newBuildCompiler(repo, nil, nil, files, mustNotCallLLM(), nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := compiler.Compile(ctx, DefaultTenantID) + + require.False(t, result.Success) + assert.Equal(t, uint64(5), repo.deletedBuild) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/language.go b/internal/manager/biz/knowledge/llm_wiki/language.go new file mode 100644 index 000000000..962b2537f --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/language.go @@ -0,0 +1,8 @@ +package llm_wiki + +// sourceLanguageRule keeps every user-visible Wiki stage aligned with the +// language of the evidence instead of the language used by the prompt. +const sourceLanguageRule = `Write every human-readable field and sentence in the same language as the source material. +Do not translate the source into English or any other language. +When a page cites sources in multiple languages, use the dominant language of that page's cited source text. +Preserve code, commands, paths, identifiers, API names, and schema keys exactly.` diff --git a/internal/manager/biz/knowledge/llm_wiki/language_test.go b/internal/manager/biz/knowledge/llm_wiki/language_test.go new file mode 100644 index 000000000..b70861ede --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/language_test.go @@ -0,0 +1,25 @@ +package llm_wiki + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWikiPromptsPreserveSourceLanguage(t *testing.T) { + prompts := map[string]string{ + "summarizer system": summarizerSystemPrompt, + "summarizer user": summarizeChunkPrompt("示例文档"), + "planner system": plannerSystemPrompt, + "planner user": plannerPrompt("示例文档"), + "writer system": sectionWriterSystemPrompt, + "writer user": sectionWriterPrompt("示例页面", "概述", "示例内容"), + } + + for name, prompt := range prompts { + t.Run(name, func(t *testing.T) { + assert.Contains(t, prompt, "same language as the source material") + assert.Contains(t, prompt, "Do not translate the source") + }) + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/llm.go b/internal/manager/biz/knowledge/llm_wiki/llm.go new file mode 100644 index 000000000..23d758672 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/llm.go @@ -0,0 +1,109 @@ +package llm_wiki + +import ( + "context" + "errors" + "log/slog" + "strings" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// llmAdapter binds Wiki compilation to one provider/model on top of the shared +// LLM client. Stage-specific request construction stays inside the compilation +// stages; this adapter only executes the request and exposes the configured +// model identity. +type llmAdapter struct { + client llm.Client + provider string + model string + modelVersion string +} + +// NewLLMAdapter binds Wiki compilation to one provider/model. Provider and +// model are explicit on purpose: Wiki output is a machine-validated contract +// and must not silently follow the interactive chat default when multiple +// providers are configured. +func NewLLMAdapter(client llm.Client, provider, model, modelVersion string) *llmAdapter { + provider = strings.TrimSpace(provider) + model = strings.TrimSpace(model) + modelVersion = strings.TrimSpace(modelVersion) + if modelVersion == "" { + modelVersion = "unknown" + } + return &llmAdapter{client: client, provider: provider, model: model, modelVersion: modelVersion} +} + +// ModelVersion reports the configured model identity. +func (a *llmAdapter) ModelVersion() string { + if a == nil || strings.TrimSpace(a.modelVersion) == "" { + return "unknown" + } + return a.modelVersion +} + +// Complete executes one compiler stage request. +func (a *llmAdapter) Complete(ctx context.Context, request llm.ChatReq) (*llm.ChatResp, error) { + if a == nil || a.client == nil { + return nil, errors.New("llmwiki: LLM client is not configured") + } + request.Provider = a.provider + request.Model = a.model + resp, err := a.client.Chat(ctx, request) + if err != nil { + return nil, err + } + if resp == nil { + return nil, errors.New("llmwiki: compiler returned an empty response") + } + if strings.TrimSpace(resp.Assistant.Content) == "" { + return resp, errors.New("llmwiki: compiler returned empty JSON") + } + return resp, nil +} + +var _ CompilerLLM = (*llmAdapter)(nil) + +// usageRecordingLLM records the provider-reported usage of every successful +// compiler call. Accounting is best-effort: a persistence failure is logged +// but must not turn a valid provider response into a failed Wiki build. +type usageRecordingLLM struct { + inner CompilerLLM + sink TokenUsageSink + log *slog.Logger +} + +func newUsageRecordingLLM(inner CompilerLLM, sink TokenUsageSink, log *slog.Logger) *usageRecordingLLM { + if log == nil { + log = slog.Default() + } + return &usageRecordingLLM{inner: inner, sink: sink, log: log} +} + +func (a *usageRecordingLLM) Complete(ctx context.Context, request llm.ChatReq) (*llm.ChatResp, error) { + if a == nil || a.inner == nil { + return nil, errors.New("llmwiki: usage recorder has no compiler LLM") + } + resp, err := a.inner.Complete(ctx, request) + if err != nil { + return resp, err + } + if resp == nil { + return nil, errors.New("llmwiki: usage recorder received an empty response") + } + if a.sink != nil { + if err := a.sink.Record(ctx, resp.Usage); err != nil { + a.log.ErrorContext(ctx, "llmwiki: record token usage failed", slog.Any("err", err)) + } + } + return resp, nil +} + +func (a *usageRecordingLLM) ModelVersion() string { + if a == nil || a.inner == nil { + return "unknown" + } + return a.inner.ModelVersion() +} + +var _ CompilerLLM = (*usageRecordingLLM)(nil) diff --git a/internal/manager/biz/knowledge/llm_wiki/llm_test.go b/internal/manager/biz/knowledge/llm_wiki/llm_test.go new file mode 100644 index 000000000..ca42dbec3 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/llm_test.go @@ -0,0 +1,131 @@ +package llm_wiki + +import ( + "context" + "errors" + "io" + "log/slog" + "testing" + + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type compilerLLMFunc struct { + version string + call func(context.Context, llm.ChatReq) (*llm.ChatResp, error) +} + +func (f compilerLLMFunc) Complete(ctx context.Context, req llm.ChatReq) (*llm.ChatResp, error) { + return f.call(ctx, req) +} + +func (f compilerLLMFunc) ModelVersion() string { + return f.version +} + +type recordingUsageSink struct { + usages []llm.Usage + recordErr error +} + +func (s *recordingUsageSink) Record(_ context.Context, usage llm.Usage) error { + s.usages = append(s.usages, usage) + return s.recordErr +} + +func (s *recordingUsageSink) Close(context.Context) error { + return nil +} + +func testWikiLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func TestUsageRecordingLLM_RecordsEverySuccessfulCall(t *testing.T) { + expected := []llm.Usage{ + {PromptTokens: 11, CompletionTokens: 3, TotalTokens: 14}, + {PromptTokens: 17, CompletionTokens: 5, TotalTokens: 22}, + } + callCount := 0 + inner := compilerLLMFunc{ + version: "wiki-model-v1", + call: func(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + resp := &llm.ChatResp{ + Assistant: llm.Message{Role: "assistant", Content: "ok"}, + Usage: expected[callCount], + } + callCount++ + return resp, nil + }, + } + sink := &recordingUsageSink{} + recorder := newUsageRecordingLLM(inner, sink, testWikiLogger()) + + for range expected { + resp, err := recorder.Complete(context.Background(), llm.ChatReq{}) + require.NoError(t, err) + assert.Equal(t, "ok", resp.Assistant.Content) + } + + assert.Equal(t, expected, sink.usages) + assert.Equal(t, "wiki-model-v1", recorder.ModelVersion()) +} + +func TestUsageRecordingLLM_WhenInnerCallFails_DoesNotRecord(t *testing.T) { + innerErr := errors.New("provider unavailable") + inner := compilerLLMFunc{ + call: func(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return nil, innerErr + }, + } + sink := &recordingUsageSink{} + recorder := newUsageRecordingLLM(inner, sink, testWikiLogger()) + + resp, err := recorder.Complete(context.Background(), llm.ChatReq{}) + + require.ErrorIs(t, err, innerErr) + assert.Nil(t, resp) + assert.Empty(t, sink.usages) +} + +func TestUsageRecordingLLM_WhenRecordFails_ReturnsProviderResponse(t *testing.T) { + expected := &llm.ChatResp{ + Assistant: llm.Message{Role: "assistant", Content: "ok"}, + Usage: llm.Usage{PromptTokens: 9, CompletionTokens: 4, TotalTokens: 13}, + } + inner := compilerLLMFunc{ + call: func(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return expected, nil + }, + } + sink := &recordingUsageSink{recordErr: errors.New("database unavailable")} + recorder := newUsageRecordingLLM(inner, sink, testWikiLogger()) + + resp, err := recorder.Complete(context.Background(), llm.ChatReq{}) + + require.NoError(t, err) + assert.Same(t, expected, resp) + assert.Equal(t, []llm.Usage{expected.Usage}, sink.usages) +} + +func TestBuildCompilerWithUsageSink_WrapsAllLLMStagesWithoutMutatingBase(t *testing.T) { + inner := compilerLLMFunc{ + version: "wiki-model-v1", + call: func(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return &llm.ChatResp{Assistant: llm.Message{Role: "assistant", Content: "ok"}}, nil + }, + } + base := &buildCompiler{llm: inner, log: testWikiLogger()} + + perJob := base.withUsageSink(&recordingUsageSink{}) + + require.NotSame(t, base, perJob) + assert.Nil(t, base.summarizer) + assert.Nil(t, base.planner) + assert.Nil(t, base.writer) + assert.IsType(t, &usageRecordingLLM{}, perJob.summarizer.llm) + assert.IsType(t, &usageRecordingLLM{}, perJob.planner.llm) + assert.IsType(t, &usageRecordingLLM{}, perJob.writer.llm) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/markdown.go b/internal/manager/biz/knowledge/llm_wiki/markdown.go new file mode 100644 index 000000000..30f683709 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/markdown.go @@ -0,0 +1,127 @@ +package llm_wiki + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +const sectionWriterSystemPrompt = `You are a documentation writer. Output clean Markdown content only. +` + sourceLanguageRule + +// pageWriter renders resolved pages as Markdown. Sections that already carry +// Markdown structure are kept verbatim; the rest is polished by the LLM. +type pageWriter struct { + llm CompilerLLM + log *slog.Logger +} + +// newPageWriter creates a Markdown page writer. +func newPageWriter(llm CompilerLLM, log *slog.Logger) *pageWriter { + return &pageWriter{llm: llm, log: log} +} + +// WritePage renders one page: title, every section, then the source footer. +func (w *pageWriter) WritePage(ctx context.Context, page *resolvedPage) (string, error) { + w.log.InfoContext(ctx, "writer: generating page content", + slog.String("page_id", page.PageID), + slog.Int("sections", len(page.Sections))) + + var content strings.Builder + content.WriteString(fmt.Sprintf("# %s\n\n", page.Title)) + + for _, section := range page.Sections { + if err := ctx.Err(); err != nil { + return "", err + } + + sectionContent, err := w.writeSection(ctx, page.Title, section) + if err != nil { + return "", fmt.Errorf("write section %s: %w", section.Heading, err) + } + + content.WriteString(sectionContent) + content.WriteString("\n\n") + } + + if len(page.Sources) > 0 { + // The footer is written here rather than by the model, so it cannot follow + // the source language the way the page body does, and the pipeline does no + // language detection by design (HLD-002 §6.4). A hardcoded "## Sources" + // heading was therefore English prose on every page in every language, so + // the footer carries none: a rule and the cited titles, which are already + // in the source language because they are the documents' own. + content.WriteString("---\n\n") + seenSources := make(map[uint64]bool) + for _, source := range page.Sources { + if seenSources[source.SourceID] { + continue + } + seenSources[source.SourceID] = true + content.WriteString(fmt.Sprintf("- %s (ID: %d)\n", source.DocumentTitle, source.SourceID)) + } + } + + return content.String(), nil +} + +// writeSection renders one section. The raw section content is used whenever it +// is already Markdown and as the fallback when the LLM call fails, so a build +// never loses text to a provider error. +func (w *pageWriter) writeSection(ctx context.Context, pageTitle string, section resolvedSection) (string, error) { + if hasMarkdownFormatting(section.Content) { + return renderSection(section.Heading, section.Content), nil + } + + resp, err := w.llm.Complete(ctx, llm.ChatReq{ + Messages: []llm.Message{ + {Role: "system", Content: sectionWriterSystemPrompt}, + {Role: "user", Content: sectionWriterPrompt(pageTitle, section.Heading, section.Content)}, + }, + }) + if err != nil { + w.log.WarnContext(ctx, "writer: LLM failed, using raw content", + slog.String("section", section.Heading), + slog.String("error", err.Error())) + return renderSection(section.Heading, section.Content), nil + } + + return renderSection(section.Heading, strings.TrimSpace(resp.Assistant.Content)), nil +} + +func renderSection(heading, content string) string { + return fmt.Sprintf("## %s\n\n%s", heading, content) +} + +func sectionWriterPrompt(pageTitle, heading, content string) string { + return fmt.Sprintf(`You are writing one section of a Wiki page titled "%s". + +Write the content for the section "%s" based on the following source material. + +Requirements: +- Write clear, concise documentation for the readers of this Wiki +- Use Markdown formatting (headers, lists, tables, code blocks where appropriate) +- Preserve the details the source treats as important — names, dates, amounts, roles, steps — and keep any code or configuration exact +- Keep the source material's own register and point of view; do not turn a policy, a report, or a note into a technical tutorial +- Make the content self-contained and easy to understand +- Add nothing the source material does not contain: never introduce a command, parameter, value, date, name, role or step of your own, and never state a guess as a fact +- %s +- Do not include the section heading (it will be added separately) + +Source material: +%s`, pageTitle, heading, sourceLanguageRule, content) +} + +// hasMarkdownFormatting reports whether content already carries Markdown +// structure worth keeping instead of rewriting it through the LLM. +func hasMarkdownFormatting(content string) bool { + for _, indicator := range []string{"```", "- ", "1. ", "> ", "**", "__", "~~"} { + if strings.Contains(content, indicator) { + return true + } + } + return len(strings.Split(content, "\n")) > 3 +} diff --git a/internal/manager/biz/knowledge/llm_wiki/markdown_test.go b/internal/manager/biz/knowledge/llm_wiki/markdown_test.go new file mode 100644 index 000000000..e95b2717e --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/markdown_test.go @@ -0,0 +1,93 @@ +package llm_wiki + +import ( + "context" + "errors" + "testing" + + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestWritePage_SourceFooterCarriesNoLanguageSpecificProse — the footer is +// written by this code, not by the model, so it cannot follow the source +// language the way the page body does, and the pipeline does no language +// detection by design. The hardcoded "## Sources" heading put English prose on +// every page in every language; the footer now carries a rule and the cited +// titles only, and those titles are already in the source language. +func TestWritePage_SourceFooterCarriesNoLanguageSpecificProse(t *testing.T) { + page := &resolvedPage{ + PageID: "contracts", + Title: "采购合同", + Sections: []resolvedSection{ + {Heading: "范围", Content: "本合同自 2026 年 1 月起生效。"}, + }, + Sources: []resolvedSource{ + {SourceID: 4, DocumentTitle: "采购合同.md"}, + {SourceID: 5, DocumentTitle: "排障手册.md"}, + {SourceID: 4, DocumentTitle: "采购合同.md"}, + }, + } + writer := newPageWriter(writerLLM("本节说明采购范围。"), testWikiLogger()) + + content, err := writer.WritePage(context.Background(), page) + + require.NoError(t, err) + assert.Equal(t, "# 采购合同\n\n## 范围\n\n本节说明采购范围。\n\n---\n\n- 采购合同.md (ID: 4)\n- 排障手册.md (ID: 5)\n", content) + assert.NotContains(t, content, "## Sources") +} + +// TestWriteSection_AlreadyFormattedContentSkipsTheLLM — the verbatim path is the +// reason a section can reach a page untouched: it must not round-trip through +// the model, and it must not be able to lose text to one. +func TestWriteSection_AlreadyFormattedContentSkipsTheLLM(t *testing.T) { + writer := newPageWriter(mustNotCallLLM(), testWikiLogger()) + + got, err := writer.writeSection(context.Background(), "排障", resolvedSection{ + Heading: "步骤", + Content: "- 先看 resolver\n- 再看上游 DNS", + }) + + require.NoError(t, err) + assert.Equal(t, "## 步骤\n\n- 先看 resolver\n- 再看上游 DNS", got) +} + +// TestWriteSection_FallsBackToRawContentWhenTheLLMFails — a build must never +// lose a section to a provider error. +func TestWriteSection_FallsBackToRawContentWhenTheLLMFails(t *testing.T) { + failing := compilerLLMFunc{ + version: "test", + call: func(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return nil, errors.New("provider unavailable") + }, + } + writer := newPageWriter(failing, testWikiLogger()) + + got, err := writer.writeSection(context.Background(), "排障", resolvedSection{ + Heading: "背景", + Content: "一次解析故障的背景说明。", + }) + + require.NoError(t, err) + assert.Equal(t, "## 背景\n\n一次解析故障的背景说明。", got) +} + +// TestSectionWriterPrompt_ForbidsInventingMaterial — the writer is the last stage +// before publication, so an invented command or value lands in the Wiki with no +// reviewer: the prompt must forbid adding what the source material lacks. +func TestSectionWriterPrompt_ForbidsInventingMaterial(t *testing.T) { + prompt := sectionWriterPrompt("排障", "步骤", "先看 resolver") + + assert.Contains(t, prompt, "Add nothing the source material does not contain") +} + +// writerLLM answers every call with the same content. +func writerLLM(content string) CompilerLLM { + return compilerLLMFunc{ + version: "test", + call: func(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return &llm.ChatResp{Assistant: llm.Message{Role: "assistant", Content: content}}, nil + }, + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/paths.go b/internal/manager/biz/knowledge/llm_wiki/paths.go new file mode 100644 index 000000000..c0946b173 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/paths.go @@ -0,0 +1,188 @@ +package llm_wiki + +import ( + "fmt" + "path/filepath" + "regexp" + "slices" + "strings" + "unicode" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" +) + +// unsafeNameRE matches everything that is not safe inside a generated file name. +var unsafeNameRE = regexp.MustCompile(`[^a-zA-Z0-9._-]+`) + +// rawRelativePath is where a source file is mirrored inside raw/: uploaded files +// keep their name, repo files keep their path below the source namespace, and +// non-default tenants get their own top-level directory. +func rawRelativePath(tenantID uint64, sourceType, sourceKey, name string) string { + if sourceType == "upload" { + fileName := safeUploadFileName(name) + if tenantID == DefaultTenantID { + return fileName + } + return filepath.ToSlash(filepath.Join(fmt.Sprintf("tenant-%d", tenantID), fileName)) + } + parts := make([]string, 0, 3) + if tenantID != DefaultTenantID { + parts = append(parts, fmt.Sprintf("tenant-%d", tenantID)) + } + parts = append(parts, sourceNamespaceDir(sourceType, sourceKey)) + switch sourceType { + case "repo", "git": + parts = append(parts, safeRelativePath(name)) + default: + parts = append(parts, safeWikiFileName(name)) + } + return filepath.ToSlash(filepath.Join(parts...)) +} + +// safeWikiFileName reduces a name to a single Markdown file name inside the Wiki +// tree. +func safeWikiFileName(name string) string { + name = safeDirName(filepath.Base(strings.TrimSpace(name))) + if name == "" { + name = "source" + } + if filepath.Ext(name) == "" { + name += ".md" + } + return name +} + +// safeUploadFileName keeps the uploaded file name, including Unicode, spaces and +// its original extension. Only path separators and control characters are +// removed, so the raw tree does not silently rename documents such as +// "设计方案.pdf" to an unrelated markdown file name. +func safeUploadFileName(name string) string { + parts := strings.FieldsFunc(strings.TrimSpace(name), func(r rune) bool { return r == '/' || r == '\\' }) + if len(parts) == 0 { + return "source" + } + name = strings.Map(func(r rune) rune { + if r == 0 || unicode.IsControl(r) { + return -1 + } + return r + }, parts[len(parts)-1]) + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." { + return "source" + } + return name +} + +func safeOrganizationPath(name string) string { + parts := strings.FieldsFunc(filepath.ToSlash(name), func(r rune) bool { return r == '/' || r == '\\' }) + clean := make([]string, 0, len(parts)) + for _, part := range parts { + if part == "." || part == ".." { + continue + } + if safe := safeUploadFileName(part); safe != "source" || part == "source" { + clean = append(clean, safe) + } + } + if len(clean) == 0 { + return "source.md" + } + clean[len(clean)-1] = strings.TrimSuffix(clean[len(clean)-1], filepath.Ext(clean[len(clean)-1])) + ".md" + return filepath.ToSlash(filepath.Join(clean...)) +} + +// wikiPageRelativePath places a generated page below the directory of its +// primary source. SourceRefs are sorted by SourceID so the primary source is +// stable across rebuilds. Old pages without SourcePath keep the historical +// flat layout. +func wikiPageRelativePath(pageID string, refs []model.SourceRef) string { + filename := safeWikiFileName(pageID) + sorted := slices.Clone(refs) + slices.SortFunc(sorted, func(left, right model.SourceRef) int { + switch { + case left.SourceID < right.SourceID: + return -1 + case left.SourceID > right.SourceID: + return 1 + default: + return 0 + } + }) + + for _, ref := range sorted { + sourcePath := strings.TrimSpace(ref.SourcePath) + if sourcePath == "" { + continue + } + dir := filepath.ToSlash(filepath.Dir(filepath.FromSlash(sourcePath))) + if safeDir := safeRelativeDirectory(dir); safeDir != "" { + return filepath.ToSlash(filepath.Join(safeDir, filename)) + } + return filename + } + return filename +} + +// safeRelativeDirectory sanitizes every directory segment while preserving the +// source-language folder names. +func safeRelativeDirectory(name string) string { + parts := strings.FieldsFunc(filepath.ToSlash(name), func(r rune) bool { return r == '/' || r == '\\' }) + clean := make([]string, 0, len(parts)) + for _, part := range parts { + if part == "." || part == ".." { + continue + } + if safe := safeUploadFileName(part); safe != "source" || part == "source" { + clean = append(clean, safe) + } + } + return filepath.ToSlash(filepath.Join(clean...)) +} + +// sourceNamespaceDir is the directory that groups all files of one source, for +// example "repo-ongrid" for the source key "github:ongrid". +func sourceNamespaceDir(sourceType, sourceKey string) string { + parts := strings.Split(sourceKey, ":") + identifier := sourceKey + if len(parts) > 1 { + identifier = parts[1] + } + if identifier == "" { + identifier = "source" + } + if name := safeDirName(sourceType + "-" + identifier); name != "" { + return name + } + return "source" +} + +// safeDirName reduces a value to a safe single path element. +func safeDirName(name string) string { + name = unsafeNameRE.ReplaceAllString(strings.TrimSpace(name), "-") + return strings.Trim(name, ".-") +} + +// safeRelativePath keeps the directory structure of a repo-relative path while +// sanitising every element and forcing a Markdown extension on the last one. +func safeRelativePath(name string) string { + parts := strings.FieldsFunc(filepath.ToSlash(name), func(r rune) bool { return r == '/' || r == '\\' }) + clean := make([]string, 0, len(parts)) + for _, part := range parts { + if part == "." || part == ".." { + continue + } + part = unsafeNameRE.ReplaceAllString(strings.Trim(part, ".-"), "-") + if part == "" { + continue + } + clean = append(clean, part) + } + if len(clean) == 0 { + return "source.md" + } + if filepath.Ext(clean[len(clean)-1]) == "" { + clean[len(clean)-1] += ".md" + } + return filepath.Join(clean...) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/paths_test.go b/internal/manager/biz/knowledge/llm_wiki/paths_test.go new file mode 100644 index 000000000..97a6f206d --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/paths_test.go @@ -0,0 +1,38 @@ +package llm_wiki + +import ( + "testing" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" +) + +func TestSafeOrganizationPathPreservesFolders(t *testing.T) { + got := safeOrganizationPath("产品/运维手册/故障排查.pdf") + if got != "产品/运维手册/故障排查.md" { + t.Fatalf("safeOrganizationPath() = %q", got) + } +} + +func TestSafeOrganizationPathDropsTraversal(t *testing.T) { + got := safeOrganizationPath("../平台/../../密钥") + if got != "平台/密钥.md" { + t.Fatalf("safeOrganizationPath() = %q", got) + } +} + +func TestWikiPageRelativePathUsesPrimarySourceDirectory(t *testing.T) { + got := wikiPageRelativePath("dns-diagnosis", []model.SourceRef{ + {SourceID: 2, SourcePath: "network/dns/troubleshooting.md"}, + {SourceID: 1, SourcePath: "network/dns/overview.md"}, + }) + if got != "network/dns/dns-diagnosis.md" { + t.Fatalf("wikiPageRelativePath() = %q", got) + } +} + +func TestWikiPageRelativePathFallsBackForLegacyPages(t *testing.T) { + got := wikiPageRelativePath("legacy-page", []model.SourceRef{{SourceID: 1}}) + if got != "legacy-page.md" { + t.Fatalf("wikiPageRelativePath() = %q", got) + } +} diff --git a/internal/manager/biz/knowledge/llm_wiki/plan.go b/internal/manager/biz/knowledge/llm_wiki/plan.go new file mode 100644 index 000000000..f84192c9b --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/plan.go @@ -0,0 +1,255 @@ +package llm_wiki + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "regexp" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// plan is the page structure the LLM proposes for one build. +type plan struct { + Pages []planPage `json:"pages"` +} + +// planPage is one page the planner wants written. +type planPage struct { + PageID string `json:"page_id"` + Title string `json:"title"` + Sections []planSection `json:"sections"` +} + +// planSection is one section of a planned page. SourceIDs point at digest items +// so the evidence resolver can trace the section back to raw documents. +type planSection struct { + Heading string `json:"heading"` + Content string `json:"content"` + SourceIDs []int `json:"source_ids"` +} + +// planResponseSchema is the strict JSON Schema the planner asks providers for. +// Providers that cannot honor it fall back to json_object mode. +const planResponseSchema = `{ + "type": "object", + "properties": { + "pages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "page_id": { + "type": "string", + "description": "Unique identifier for the page, using kebab-case" + }, + "title": { + "type": "string", + "description": "Human-readable page title" + }, + "sections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "heading": { + "type": "string", + "description": "Section heading" + }, + "content": { + "type": "string", + "description": "Section content in Markdown format" + }, + "source_ids": { + "type": "array", + "items": {"type": "integer", "minimum": 0}, + "description": "Zero-based Digest source_id indices that provide source material; these are not database IDs" + } + }, + "required": ["heading", "content", "source_ids"] + } + } + }, + "required": ["page_id", "title", "sections"] + } + } + }, + "required": ["pages"] +}` + +// plannerSystemPrompt is the system prompt for the planning stage. +const plannerSystemPrompt = `You are a Wiki structure planner for an organization's knowledge base. + +Your task is to analyze a digest of documents and create a well-organized Wiki structure. + +The material may be of any kind: technical references, policies, meeting notes, contracts, reports, research, or general prose. Let the material itself decide the structure instead of assuming a technical manual — the same rules apply to a process guide, a policy, and an API reference. + +Guidelines: +1. Each page should cover a coherent, self-contained topic +2. Pages should be logically ordered and cross-referenced where appropriate +3. Sections within a page should follow a logical flow +4. Preserve what the material itself treats as important — names, dates, amounts, terms, roles, decisions, steps and examples — along with any code or configuration it contains +5. Use clear, descriptive page IDs in kebab-case format +6. source_ids MUST contain only the zero-based "Digest source_id" values printed in the digest; never invent or use database IDs +7. Ground every heading and every line of content in what the digest states. Never invent an API, command, parameter, value, date, name, role or step, and never present a guess as something the material says — a plan that leaves a gap is correct, a plan that fills a gap with invention is not + +` + sourceLanguageRule + ` + +Output valid JSON only. Do not include any explanations or markdown.` + +// planner turns the reduced corpus digest into the page structure of a build. +type planner struct { + llm CompilerLLM + log *slog.Logger +} + +// newPlanner creates a Wiki structure planner. +func newPlanner(llm CompilerLLM, log *slog.Logger) *planner { + return &planner{llm: llm, log: log} +} + +// Plan asks the LLM for a page structure and validates it locally. It asks for a +// strict JSON Schema first and falls back to json_object mode when the provider +// refuses the schema. +func (p *planner) Plan(ctx context.Context, digest string) (*plan, error) { + p.log.InfoContext(ctx, "planner: planning Wiki structure", slog.Int("digest_length", len(digest))) + + created, err := p.planWithJSONSchema(ctx, digest) + if err != nil { + p.log.WarnContext(ctx, "planner: JSON schema planning failed, trying json_object", + slog.String("error", err.Error())) + created, err = p.planWithJSONObject(ctx, digest) + if err != nil { + return nil, fmt.Errorf("planner failed: %w", err) + } + } + + if err := validatePlan(created); err != nil { + return nil, fmt.Errorf("invalid plan: %w", err) + } + + p.log.InfoContext(ctx, "planner: plan created", slog.Int("pages", len(created.Pages))) + return created, nil +} + +// planWithJSONSchema asks for the plan with a JSON Schema response format. +func (p *planner) planWithJSONSchema(ctx context.Context, digest string) (*plan, error) { + return p.requestPlan(ctx, digest, &llm.ResponseFormat{ + Type: llm.ResponseFormatJSONSchema, + Name: "wiki_plan", + Schema: json.RawMessage(planResponseSchema), + }) +} + +// planWithJSONObject asks for the plan in plain json_object mode. +func (p *planner) planWithJSONObject(ctx context.Context, digest string) (*plan, error) { + return p.requestPlan(ctx, digest, &llm.ResponseFormat{Type: llm.ResponseFormatJSONObject}) +} + +// requestPlan performs one planning request and decodes the response. Both +// response-format attempts share it so they stay identical apart from the format. +func (p *planner) requestPlan(ctx context.Context, digest string, format *llm.ResponseFormat) (*plan, error) { + resp, err := p.llm.Complete(ctx, llm.ChatReq{ + Messages: []llm.Message{ + {Role: "system", Content: plannerSystemPrompt}, + {Role: "user", Content: plannerPrompt(digest)}, + }, + ResponseFormat: format, + }) + if err != nil { + return nil, err + } + + var created plan + if err := json.Unmarshal([]byte(resp.Assistant.Content), &created); err != nil { + return nil, fmt.Errorf("decode plan: %w", err) + } + return &created, nil +} + +// maxPageIDLength matches the page_id column of the build page table, so an +// over-long id is rejected before it can reach the database. +const maxPageIDLength = 64 + +// pageIDPattern is the kebab-case whitelist the planner is asked for. It also +// keeps an id usable as one artifact file name below the build directory and as +// one segment of a Wiki tree path. +var pageIDPattern = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) + +// validatePlan rejects structurally unusable plans before any page is written. +func validatePlan(created *plan) error { + if created == nil { + return fmt.Errorf("nil plan") + } + if len(created.Pages) == 0 { + return fmt.Errorf("plan has no pages") + } + + seenPageIDs := make(map[string]bool) + for i, page := range created.Pages { + if err := validatePageID(page.PageID); err != nil { + return fmt.Errorf("page %d: %w", i, err) + } + if page.Title == "" { + return fmt.Errorf("page %d has empty title", i) + } + if seenPageIDs[page.PageID] { + return fmt.Errorf("duplicate page_id: %s", page.PageID) + } + seenPageIDs[page.PageID] = true + + if len(page.Sections) == 0 { + return fmt.Errorf("page %s has no sections", page.PageID) + } + + for j, section := range page.Sections { + if section.Heading == "" { + return fmt.Errorf("page %s section %d has empty heading", page.PageID, j) + } + if section.Content == "" { + return fmt.Errorf("page %s section %d has empty content", page.PageID, j) + } + } + } + + return nil +} + +// validatePageID enforces the page id contract: a non-empty kebab-case id that +// fits the database column. Page ids come from the LLM, so an unvalidated id +// could otherwise escape the build artifact directory as a path component. +func validatePageID(pageID string) error { + if pageID == "" { + return errors.New("empty page_id") + } + if len(pageID) > maxPageIDLength { + return fmt.Errorf("page_id %q is longer than %d characters", pageID, maxPageIDLength) + } + if !pageIDPattern.MatchString(pageID) { + return fmt.Errorf("page_id %q is not kebab-case", pageID) + } + return nil +} + +// plannerPrompt renders the user prompt for planning. +func plannerPrompt(digest string) string { + return fmt.Sprintf(`Based on the following digest of an organization's documents, plan a Wiki structure. + +Requirements: +- Create pages that cover distinct topics +- Each page should have well-organized sections +- Include source_ids using only the zero-based "Digest source_id" values shown below +- Each entry opens with its "Digest source_id" and, when the document has one, a "Document:" title; both are labels, not content, and the title names the document the passage was taken from +- Pages should be self-contained but may reference other pages +- %s + +Digest of documents: +%s + +Output a JSON object with a "pages" array. Each page has: +- "page_id": unique identifier in kebab-case +- "title": human-readable title +- "sections": array of sections, each with "heading", "content" (Markdown), and "source_ids" (array of zero-based Digest source_id integers)`, sourceLanguageRule, digest) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/plan_test.go b/internal/manager/biz/knowledge/llm_wiki/plan_test.go new file mode 100644 index 000000000..44f0861dd --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/plan_test.go @@ -0,0 +1,57 @@ +package llm_wiki + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestValidatePlan_RejectsUnsafePageIDs — page ids come from the LLM and are +// used as artifact file names, so anything outside the kebab-case whitelist must +// be refused before a page is written. +func TestValidatePlan_RejectsUnsafePageIDs(t *testing.T) { + tests := []struct { + name string + pageID string + }{ + {name: "empty", pageID: ""}, + {name: "parent traversal", pageID: "../../../builds/7/pages/other"}, + {name: "nested path", pageID: "topics/dns"}, + {name: "space", pageID: "dns overview"}, + {name: "uppercase", pageID: "DNS-Overview"}, + {name: "dot segment", pageID: ".."}, + {name: "leading dash", pageID: "-dns"}, + {name: "trailing dash", pageID: "dns-"}, + {name: "extension", pageID: "dns.md"}, + {name: "longer than the column", pageID: strings.Repeat("a", maxPageIDLength+1)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validatePlan(&plan{Pages: []planPage{{ + PageID: tt.pageID, + Title: "DNS", + Sections: []planSection{{Heading: "Overview", Content: "body"}}, + }}}) + require.Error(t, err) + }) + } +} + +func TestValidatePlan_AcceptsKebabCasePageIDs(t *testing.T) { + err := validatePlan(&plan{Pages: []planPage{ + {PageID: "dns-overview", Title: "DNS", Sections: []planSection{{Heading: "h", Content: "c"}}}, + {PageID: "2024", Title: "Archive", Sections: []planSection{{Heading: "h", Content: "c"}}}, + {PageID: strings.Repeat("a", maxPageIDLength), Title: "Long", Sections: []planSection{{Heading: "h", Content: "c"}}}, + }}) + require.NoError(t, err) +} + +func TestValidatePlan_RejectsDuplicatePageIDs(t *testing.T) { + page := planPage{PageID: "dns-overview", Title: "DNS", Sections: []planSection{{Heading: "h", Content: "c"}}} + err := validatePlan(&plan{Pages: []planPage{page, page}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate page_id") +} diff --git a/internal/manager/biz/knowledge/llm_wiki/raw.go b/internal/manager/biz/knowledge/llm_wiki/raw.go new file mode 100644 index 000000000..dc10c2f97 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/raw.go @@ -0,0 +1,95 @@ +package llm_wiki + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// MirrorInput describes one source file to mirror into the Wiki tree. +type MirrorInput struct { + TenantID uint64 + SourceKey string + SourceType string + Name string + Content []byte + RelativePath string +} + +// MirroredFile is where a mirrored source file and its snapshot ended up. +type MirroredFile struct { + RawPath string + SnapshotPath string + SHA256 string + Size uint64 +} + +// MirrorSource writes a raw file plus an immutable, content-addressed snapshot. +// Compilation reads the snapshot, so replacing a raw file never changes a page +// that was generated from an earlier version. +func (s *FileStore) MirrorSource(ctx context.Context, input MirrorInput) (*MirroredFile, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if strings.TrimSpace(input.SourceKey) == "" { + return nil, errors.New("llmwiki: source key is required") + } + if len(input.Content) > MaxSourceBytes { + return nil, fmt.Errorf("llmwiki: source exceeds %d bytes", MaxSourceBytes) + } + + digest := contentSHA256(input.Content) + rawRelative := rawRelativePath(input.TenantID, input.SourceType, input.SourceKey, input.Name) + if input.RelativePath != "" { + rawRelative = safeOrganizationPath(input.RelativePath) + } + rawPath, err := s.resolve("raw", rawRelative) + if err != nil { + return nil, err + } + + versionRelative := filepath.ToSlash(filepath.Join( + ".llm-wiki/versions", + shortHash(fmt.Sprintf("%d:%s", input.TenantID, input.SourceKey)), + digest+".md", + )) + versionPath, err := s.resolve("", versionRelative) + if err != nil { + return nil, err + } + + if err := atomicWrite(versionPath, input.Content, 0o640); err != nil { + return nil, fmt.Errorf("llmwiki: write version: %w", err) + } + if err := atomicWrite(rawPath, input.Content, 0o640); err != nil { + return nil, fmt.Errorf("llmwiki: write raw: %w", err) + } + + return &MirroredFile{ + RawPath: rawRelative, + SnapshotPath: versionRelative, + SHA256: digest, + Size: uint64(len(input.Content)), + }, nil +} + +// RemoveSourceFile deletes one mirrored raw file. A missing file is not an error. +func (s *FileStore) RemoveSourceFile(ctx context.Context, relative string) error { + if err := ctx.Err(); err != nil { + return err + } + path, err := s.resolve("raw", relative) + if err != nil { + return err + } + if err := os.Remove(path); errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("llmwiki: remove raw file: %w", err) + } + return nil +} diff --git a/internal/manager/biz/knowledge/llm_wiki/search.go b/internal/manager/biz/knowledge/llm_wiki/search.go new file mode 100644 index 000000000..f41e8b592 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/search.go @@ -0,0 +1,33 @@ +package llm_wiki + +import ( + "context" + "errors" + "log/slog" + "strings" +) + +// Search answers one Wiki query through the derived search index. The index is +// required: Qdrant vector search is a hard dependency, so an index failure is +// returned to the caller instead of being masked by a file scan. +func (u *Usecase) Search(ctx context.Context, tenantID uint64, query string, limit int) ([]SearchHit, error) { + unlock := u.files.LockArtifacts() + defer unlock() + + query = strings.ToLower(strings.TrimSpace(query)) + if query == "" { + return nil, nil + } + if limit <= 0 || limit > 50 { + limit = 10 + } + if u.indexer == nil { + return nil, errors.New("llmwiki: search index is unavailable") + } + hits, err := u.indexer.Search(ctx, tenantID, query, limit) + if err != nil { + u.log.WarnContext(ctx, "wiki index search failed", slog.Any("err", err)) + return nil, err + } + return hits, nil +} diff --git a/internal/manager/biz/knowledge/llm_wiki/source.go b/internal/manager/biz/knowledge/llm_wiki/source.go new file mode 100644 index 000000000..7b33a62a1 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/source.go @@ -0,0 +1,146 @@ +package llm_wiki + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + "strings" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" +) + +// Mirror stores one source file and records the new version. changed reports +// whether the content differed from the stored version, so callers can skip a +// recompilation for an unchanged upload. +func (u *Usecase) Mirror(ctx context.Context, input MirrorInput) (*model.Source, bool, error) { + mirrored, err := u.files.MirrorSource(ctx, input) + if err != nil { + return nil, false, err + } + source := &model.Source{ + TenantID: input.TenantID, + SourceKey: input.SourceKey, + SourceType: input.SourceType, + RawPath: mirrored.RawPath, + ContentSHA256: mirrored.SHA256, + Status: model.SourcePending, + } + version := &model.SourceVersion{ + TenantID: input.TenantID, + SHA256: mirrored.SHA256, + SizeBytes: mirrored.Size, + SnapshotPath: mirrored.SnapshotPath, + SchemaVersion: SchemaVersion, + } + stored, _, changed, err := u.repo.UpsertSourceVersion(ctx, source, version) + if err != nil { + return nil, false, fmt.Errorf("llmwiki: save mirror metadata: %w", err) + } + return stored, changed, nil +} + +type OrganizationSource struct { + ID uint64 + Title, Path, Content string +} +type SyncResult struct { + Created int `json:"created"` + Updated int `json:"updated"` + Unchanged int `json:"unchanged"` + Deleted int `json:"deleted"` + Total int `json:"total"` +} + +// sourcePageSize is the page size listAllSources walks with. It matches the +// store's default so a page never gets clamped mid-walk. +const sourcePageSize = 200 + +// sourceLister is the narrow read surface needed to walk a tenant's sources. +type sourceLister interface { + ListSourcesAfter(ctx context.Context, tenantID uint64, status string, afterID uint64, limit int) ([]*model.Source, error) +} + +// listAllSources returns every source of a tenant. ListSources answers with one +// page, so a tenant with more sources than a page holds would silently lose the +// remainder — which for sync means deleted documents never get cleaned up, and +// for a compile means a corpus that quietly omits sources. +func listAllSources(ctx context.Context, repo sourceLister, tenantID uint64) ([]*model.Source, error) { + all := make([]*model.Source, 0, sourcePageSize) + var afterID uint64 + for { + page, err := repo.ListSourcesAfter(ctx, tenantID, "", afterID, sourcePageSize) + if err != nil { + return nil, err + } + if len(page) == 0 { + return all, nil + } + all = append(all, page...) + afterID = page[len(page)-1].ID + if len(page) < sourcePageSize { + return all, nil + } + } +} + +// SyncOrganizationSources adds or refreshes the organization knowledge base in +// the Wiki raw tree while retaining its folder hierarchy. It deliberately does +// not remove sources missing from this snapshot: repository re-indexing clears +// and repopulates Qdrant, so absence can be transient and must not invalidate +// the active Wiki build. Explicit Raw deletion remains the destructive path. +func (u *Usecase) SyncOrganizationSources(ctx context.Context, docs []OrganizationSource) (*SyncResult, error) { + if err := u.ensureNoActiveCompile(ctx, DefaultTenantID); err != nil { + return nil, err + } + + unlock := u.files.LockArtifacts() + defer unlock() + existing, err := listAllSources(ctx, u.repo, DefaultTenantID) + if err != nil { + return nil, fmt.Errorf("llmwiki: list sources for sync: %w", err) + } + byKey := make(map[string]*model.Source) + for _, source := range existing { + if source.SourceType == "organization" { + byKey[source.SourceKey] = source + } + } + result := &SyncResult{Total: len(docs)} + for _, doc := range docs { + key := organizationSourcePrefix + strconv.FormatUint(doc.ID, 10) + name := strings.TrimSpace(doc.Title) + if name == "" { + name = fmt.Sprintf("document-%d", doc.ID) + } + relative := filepath.ToSlash(filepath.Join(doc.Path, name+".md")) + old := byKey[key] + stored, changed, mirrorErr := u.Mirror(ctx, MirrorInput{TenantID: DefaultTenantID, SourceKey: key, SourceType: "organization", Name: name, Content: []byte(doc.Content), RelativePath: relative}) + if mirrorErr != nil { + return nil, mirrorErr + } + if old == nil { + result.Created++ + } else if changed || old.RawPath != stored.RawPath { + result.Updated++ + } else { + result.Unchanged++ + } + if old != nil && old.RawPath != stored.RawPath { + if err := u.files.RemoveSourceFile(ctx, old.RawPath); err != nil { + return nil, err + } + } + } + return result, nil +} + +// organizationSourcePrefix marks a source mirrored from one organization +// knowledge-base document, whose id follows the prefix. +const organizationSourcePrefix = "organization:" + +// ListSources lists the mirrored sources of a tenant, optionally filtered by +// status. +func (u *Usecase) ListSources(ctx context.Context, tenantID uint64, status string, limit int) ([]*model.Source, int64, error) { + return u.repo.ListSources(ctx, tenantID, status, limit) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/summarize.go b/internal/manager/biz/knowledge/llm_wiki/summarize.go new file mode 100644 index 000000000..c8199303d --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/summarize.go @@ -0,0 +1,178 @@ +package llm_wiki + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// summarizeTokenBudget is the corpus size (in estimated tokens) above which the +// Map phase summarizes every chunk instead of passing the text through. +const summarizeTokenBudget = 32_000 + +// digestItem is the plain-text digest of one chunk plus the sources it came +// from. The planner consumes the reduced digest, the evidence resolver maps the +// referenced indices back to source documents. +type digestItem struct { + ChunkIndex int + Text string + SourceIDs []uint64 + // DocumentTitle is the document the chunk came from. The planner never sees + // the corpus, so without it a digest of an organization's documents reaches + // the model as anonymous text: it cannot tell a runbook from a contract, and + // it cannot name a page after the document it was compiled from. + DocumentTitle string +} + +// summarizer performs the Map phase of Wiki compilation: it turns every corpus +// chunk into one digest item, summarizing chunks only when the corpus is large +// enough to need it. +type summarizer struct { + llm CompilerLLM + log *slog.Logger +} + +const summarizerSystemPrompt = `You are a knowledge base summarizer. Output plain text only. Be concise but preserve the details the material treats as important, whatever kind of material it is. +` + sourceLanguageRule + +// newSummarizer creates a corpus summarizer. +func newSummarizer(llm CompilerLLM, log *slog.Logger) *summarizer { + return &summarizer{llm: llm, log: log} +} + +// Summarize turns the corpus into digest items. Short corpora are returned +// verbatim so small installs need no extra LLM round trips. +func (s *summarizer) Summarize(ctx context.Context, corpus *corpus) ([]digestItem, error) { + // Estimate total tokens + totalTokens := 0 + for _, chunk := range corpus.Chunks { + totalTokens += estimateTokens(chunk.Text) + } + + // Short corpus: skip summarization, use chunks directly + if totalTokens < summarizeTokenBudget { + s.log.InfoContext(ctx, "summarizer: short corpus, skipping summarization", + slog.Int("chunks", len(corpus.Chunks)), + slog.Int("estimated_tokens", totalTokens)) + + items := make([]digestItem, len(corpus.Chunks)) + for i, chunk := range corpus.Chunks { + document := corpus.Documents[chunk.DocumentIndex] + items[i] = digestItem{ + ChunkIndex: i, + Text: chunk.Text, + SourceIDs: []uint64{document.SourceID}, + DocumentTitle: digestTitle(document.Title), + } + } + return items, nil + } + + // Long corpus: perform Map summarization + s.log.InfoContext(ctx, "summarizer: performing map summarization", + slog.Int("chunks", len(corpus.Chunks)), + slog.Int("estimated_tokens", totalTokens)) + + items := make([]digestItem, len(corpus.Chunks)) + for i, chunk := range corpus.Chunks { + if err := ctx.Err(); err != nil { + return nil, err + } + + summary, err := s.summarizeChunk(ctx, chunk.Text, i) + if err != nil { + return nil, fmt.Errorf("summarize chunk %d: %w", i, err) + } + + document := corpus.Documents[chunk.DocumentIndex] + items[i] = digestItem{ + ChunkIndex: i, + Text: summary, + SourceIDs: []uint64{document.SourceID}, + DocumentTitle: digestTitle(document.Title), + } + } + + return items, nil +} + +// summarizeChunk generates a plain-text summary of a chunk. +func (s *summarizer) summarizeChunk(ctx context.Context, text string, index int) (string, error) { + prompt := summarizeChunkPrompt(text) + + resp, err := s.llm.Complete(ctx, llm.ChatReq{ + Messages: []llm.Message{ + {Role: "system", Content: summarizerSystemPrompt}, + {Role: "user", Content: prompt}, + }, + }) + if err != nil { + return "", fmt.Errorf("llm summarize: %w", err) + } + + summary := strings.TrimSpace(resp.Assistant.Content) + if summary == "" { + return "", fmt.Errorf("empty summary returned for chunk %d", index) + } + + return summary, nil +} + +// summarizeChunkPrompt renders the summary request for one source chunk. +func summarizeChunkPrompt(text string) string { + return fmt.Sprintf(`Summarize the following document in plain language. + +Requirements: +- Focus on the key facts, concepts, procedures, decisions, and figures the document carries +- Keep names, roles, dates, amounts, and values exact, and keep any code, command, path, or configuration value exact +- Describe the material as what it is; do not reframe a policy, a report, or a note as a technical manual +- Keep the summary concise but complete +- Output plain text only, no JSON or markdown formatting +- %s + +Text to summarize: +%s`, sourceLanguageRule, text) +} + +// maxDigestTitleRunes caps the document title carried into the digest. The +// title is prompt scaffolding, not content, and must not crowd out the passage +// it labels. +const maxDigestTitleRunes = 200 + +// digestTitle normalizes a document title for the digest's one-line header. A +// title carrying a newline would otherwise break the per-item framing and make +// the rest of the title read as part of the passage. +func digestTitle(title string) string { + title = strings.Join(strings.Fields(title), " ") + if runes := []rune(title); len(runes) > maxDigestTitleRunes { + title = string(runes[:maxDigestTitleRunes]) + } + return title +} + +// buildDigest joins digest items into the single digest string the planner reads. +func buildDigest(items []digestItem) string { + var digest strings.Builder + for i, item := range items { + // Only expose the stable, zero-based digest index. Exposing database + // source IDs here made the planner confuse those IDs with item indices. + digest.WriteString(fmt.Sprintf("--- Digest source_id: %d ---\n", i)) + // The planner never sees the corpus itself, so the title is what tells it + // whether a passage came from a runbook, a contract or a meeting note — + // and what to name a page compiled from that document. + if item.DocumentTitle != "" { + digest.WriteString("Document: " + item.DocumentTitle + "\n") + } + digest.WriteString(item.Text) + digest.WriteString("\n\n") + } + return digest.String() +} + +// estimateTokens provides a rough token estimate (charsPerToken characters per token). +func estimateTokens(text string) int { + return len(text) / charsPerToken +} diff --git a/internal/manager/biz/knowledge/llm_wiki/summarize_test.go b/internal/manager/biz/knowledge/llm_wiki/summarize_test.go new file mode 100644 index 000000000..4ad1dd199 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/summarize_test.go @@ -0,0 +1,60 @@ +package llm_wiki + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSummarize_CarriesTheDocumentTitleIntoTheDigest — the planner only ever +// sees the digest, so a digest without titles reached the model as anonymous +// text: it could not tell a runbook from a contract, and it could not name a +// page after the document it was compiled from. +func TestSummarize_CarriesTheDocumentTitleIntoTheDigest(t *testing.T) { + corpus := &corpus{ + Documents: []corpusDocument{ + {SourceID: 4, SourceVersionID: 7, Title: "2026 年度采购合同.md"}, + {SourceID: 5, SourceVersionID: 8, Title: "on-call 排障手册.md"}, + }, + Chunks: []corpusChunk{ + {DocumentIndex: 0, Ordinal: 0, Text: "合同期自 2026 年 1 月起。"}, + {DocumentIndex: 1, Ordinal: 0, Text: "解析失败时先看 resolver。"}, + }, + } + summarizer := newSummarizer(mustNotCallLLM(), testWikiLogger()) + + items, err := summarizer.Summarize(context.Background(), corpus) + + require.NoError(t, err) + require.Len(t, items, 2) + assert.Equal(t, "2026 年度采购合同.md", items[0].DocumentTitle) + assert.Equal(t, "on-call 排障手册.md", items[1].DocumentTitle) + digest := buildDigest(items) + assert.Contains(t, digest, "Document: 2026 年度采购合同.md") + assert.Contains(t, digest, "Document: on-call 排障手册.md") +} + +// TestDigestTitle_KeepsTheDigestHeaderToOneLine — the title is rendered into the +// digest's per-item header, so a title carrying a newline would push part of +// itself into the passage the planner reads. +func TestDigestTitle_KeepsTheDigestHeaderToOneLine(t *testing.T) { + assert.Equal(t, "危险 变更", digestTitle(" 危险\n变更\t")) + assert.Equal(t, "", digestTitle("\n\n")) + + long := strings.Repeat("标", maxDigestTitleRunes+50) + got := digestTitle(long) + assert.Equal(t, maxDigestTitleRunes, len([]rune(got))) + assert.True(t, strings.HasPrefix(long, got), "truncation must not corrupt the runes it keeps") +} + +// TestBuildDigestOmitsTheTitleLineWhenTheDocumentHasNone — an empty title must +// not render a stray label the planner could read as content. +func TestBuildDigestOmitsTheTitleLineWhenTheDocumentHasNone(t *testing.T) { + digest := buildDigest([]digestItem{{ChunkIndex: 0, Text: "no title here", SourceIDs: []uint64{1}}}) + + require.Equal(t, "--- Digest source_id: 0 ---\nno title here\n\n", digest) + assert.NotContains(t, digest, "Document:") +} diff --git a/internal/manager/biz/knowledge/llm_wiki/tree.go b/internal/manager/biz/knowledge/llm_wiki/tree.go new file mode 100644 index 000000000..a2ff397b0 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/tree.go @@ -0,0 +1,509 @@ +package llm_wiki + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/docextract" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// ListTree returns the children of one Wiki tree node. layer selects the tree +// ("raw" or "wiki") and an empty parentID starts at its root. +func (u *Usecase) ListTree(ctx context.Context, layer, parentID string) ([]TreeNode, error) { + unlock := u.files.LockArtifacts() + defer unlock() + if layer != "raw" && layer != "wiki" { + return nil, errors.Join(errs.ErrInvalid, errors.New("invalid layer")) + } + relative := "" + if parentID != "" { + decodedLayer, decodedRelative, err := decodeNodeID(parentID) + if err != nil || decodedLayer != layer { + return nil, errors.Join(errs.ErrInvalid, errors.New("invalid parent")) + } + relative = decodedRelative + } + return u.listTreeFromMetadata(ctx, layer, relative, parentID) +} + +// listTreeFromMetadata builds the complete flat tree from one repository +// snapshot. The filesystem is consulted only to discard stale rows and to obtain +// a fallback timestamp; folder counts and hierarchy are derived from durable +// metadata, so expanding a directory never triggers another directory scan. +func (u *Usecase) listTreeFromMetadata(ctx context.Context, layer, parentRelative, parentID string) ([]TreeNode, error) { + type fileMetadata struct { + source *model.Source + buildPage *model.WikiBuildPage + } + files := make(map[string]fileMetadata) + if layer == "raw" { + sources, err := listAllSources(ctx, u.repo, DefaultTenantID) + if err != nil { + return nil, err + } + for _, source := range sources { + if source == nil || strings.TrimSpace(source.RawPath) == "" { + continue + } + files[filepath.ToSlash(filepath.Clean(source.RawPath))] = fileMetadata{source: source} + } + } + if layer == "wiki" { + buildPages, active, err := u.activeBuildPages(ctx) + if err != nil { + return nil, err + } + if active { + for _, page := range buildPages { + if page == nil || strings.TrimSpace(page.PageID) == "" || strings.TrimSpace(page.BodyPath) == "" { + continue + } + relative, err := wikiBuildPagePath(page) + if err != nil { + return nil, err + } + files[relative] = fileMetadata{buildPage: page} + } + } + } + + tree := make(map[string]*TreeNode) + for relative, metadata := range files { + storedRelative := relative + if metadata.buildPage != nil { + storedRelative = metadata.buildPage.BodyPath + } + path, err := u.files.resolve(layer, storedRelative) + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return nil, err + } + if info.IsDir() { + continue + } + fileNode := TreeNode{ + ID: encodeNodeID(layer, relative), Layer: layer, Kind: "file", + Name: filepath.Base(relative), RelativePath: relative, DocumentCount: 1, + } + updated := info.ModTime() + if metadata.source != nil { + fileNode.SourceID = strconv.FormatUint(metadata.source.ID, 10) + fileNode.Status = metadata.source.Status + if !metadata.source.UpdatedAt.IsZero() { + updated = metadata.source.UpdatedAt + } + } + if metadata.buildPage != nil { + fileNode.PageID = metadata.buildPage.PageID + fileNode.PageType = metadata.buildPage.PageType + fileNode.Name = metadata.buildPage.Title + fileNode.Status = "ready" + if !metadata.buildPage.CreatedAt.IsZero() { + updated = metadata.buildPage.CreatedAt + } + } + fileNode.UpdatedAt = &updated + tree[relative] = &fileNode + for parent := filepath.ToSlash(filepath.Dir(relative)); parent != "." && parent != ""; parent = filepath.ToSlash(filepath.Dir(parent)) { + if _, exists := tree[parent]; !exists { + tree[parent] = &TreeNode{ID: encodeNodeID(layer, parent), Layer: layer, Kind: "folder", Name: filepath.Base(parent), RelativePath: parent} + } + } + } + + rollUpFolderStats(tree) + return selectTreeLevel(tree, layer, parentRelative, parentID), nil +} + +// rollUpFolderStats counts the documents and the newest update time of every +// folder node. +func rollUpFolderStats(tree map[string]*TreeNode) { + for relative, node := range tree { + if node.Kind != "file" { + continue + } + for parent := filepath.ToSlash(filepath.Dir(relative)); parent != "." && parent != ""; parent = filepath.ToSlash(filepath.Dir(parent)) { + folder := tree[parent] + if folder == nil { + continue + } + folder.DocumentCount++ + if node.UpdatedAt != nil && (folder.UpdatedAt == nil || node.UpdatedAt.After(*folder.UpdatedAt)) { + updated := *node.UpdatedAt + folder.UpdatedAt = &updated + } + } + } + for relative, node := range tree { + if node.Kind == "file" { + continue + } + for childRelative := range tree { + if filepath.ToSlash(filepath.Dir(childRelative)) != relative { + continue + } + node.ChildCount++ + node.HasChildren = true + } + } +} + +// selectTreeLevel returns the nodes directly below parentRelative, folders first +// and then by name. +func selectTreeLevel(tree map[string]*TreeNode, layer, parentRelative, parentID string) []TreeNode { + items := make([]TreeNode, 0, len(tree)) + for relative, node := range tree { + parent := filepath.ToSlash(filepath.Dir(relative)) + if parent == "." { + parent = "" + } + if parentID != "" && parent != parentRelative { + continue + } + if parentID != "" { + node.ParentID = parentID + } else if parent != "" { + node.ParentID = encodeNodeID(layer, parent) + } + items = append(items, *node) + } + sort.Slice(items, func(i, j int) bool { + if items[i].Kind != items[j].Kind { + return items[i].Kind == "folder" + } + left := strings.ToLower(items[i].Name) + right := strings.ToLower(items[j].Name) + if left != right { + return left < right + } + return items[i].RelativePath < items[j].RelativePath + }) + return items +} + +// GetNode returns one tree node with its content and metadata. +func (u *Usecase) GetNode(ctx context.Context, id string) (*NodeDetail, error) { + unlock := u.files.LockArtifacts() + defer unlock() + + layer, relative, err := decodeNodeID(id) + if err != nil { + return nil, err + } + if layer != "raw" && layer != "wiki" { + return nil, errs.ErrInvalid + } + storedRelative := relative + var buildPage *model.WikiBuildPage + if layer == "wiki" { + buildPage, err = u.findActiveBuildPage(ctx, relative) + if err != nil { + return nil, err + } + if buildPage != nil { + storedRelative = buildPage.BodyPath + } else { + return nil, errs.ErrNotFound + } + } + path, err := u.files.resolve(layer, storedRelative) + if err != nil { + return nil, err + } + storeRelative := filepath.ToSlash(filepath.Join(layer, storedRelative)) + maxBytes := int64(MaxPageBytes) + if layer == "raw" { + maxBytes = MaxSourceBytes + } + body, err := u.files.Read(ctx, storeRelative, maxBytes) + if errors.Is(err, fs.ErrNotExist) { + return nil, errs.ErrNotFound + } + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, errors.Join(errs.ErrInvalid, errors.New("folder has no content")) + } + + updated := info.ModTime() + displayBody := body + if layer == "raw" && binaryPreviewFormat(relative) != "" { + displayBody = nil + } + detail := &NodeDetail{ + TreeNode: TreeNode{ID: id, Layer: layer, Kind: "file", Name: filepath.Base(relative), RelativePath: relative, UpdatedAt: &updated}, + Content: string(displayBody), + Metadata: map[string]any{"sha256": contentSHA256(body)}, + } + if buildPage != nil { + detail.Name = buildPage.Title + detail.PageID = buildPage.PageID + detail.PageType = buildPage.PageType + detail.Status = "ready" + } + if layer == "raw" { + if err := u.attachSourceFiles(ctx, detail, relative); err != nil { + return nil, err + } + } + if layer == "wiki" { + if buildPage != nil { + if err := u.attachBuildPageMetadata(ctx, detail, buildPage); err != nil { + return nil, err + } + } + } + return detail, nil +} + +// activeBuildPages returns the published build snapshot. The active build is +// the sole source of truth for generated Wiki pages. +func (u *Usecase) activeBuildPages(ctx context.Context) ([]*model.WikiBuildPage, bool, error) { + return u.activeBuildPagesForTenant(ctx, DefaultTenantID) +} + +func (u *Usecase) activeBuildPagesForTenant(ctx context.Context, tenantID uint64) ([]*model.WikiBuildPage, bool, error) { + build, err := u.repo.GetActiveBuild(ctx, tenantID) + if errors.Is(err, errs.ErrNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + pages, err := u.repo.ListPagesByBuild(ctx, tenantID, build.ID) + return pages, true, err +} + +func (u *Usecase) findActiveBuildPage(ctx context.Context, relative string) (*model.WikiBuildPage, error) { + pages, active, err := u.activeBuildPages(ctx) + if err != nil || !active { + return nil, err + } + for _, page := range pages { + virtualPath, err := wikiBuildPagePath(page) + if err != nil { + return nil, err + } + if virtualPath == relative { + return page, nil + } + } + return nil, nil +} + +// wikiBuildPagePath derives the visible tree path of a generated page from its +// source provenance. Pages written before SourcePath existed stay flat. +func wikiBuildPagePath(page *model.WikiBuildPage) (string, error) { + if page == nil { + return "", nil + } + var refs []model.SourceRef + if raw := strings.TrimSpace(page.SourceRefsJSON); raw != "" { + if err := json.Unmarshal([]byte(raw), &refs); err != nil { + return "", fmt.Errorf("llmwiki: decode build page source refs for %s: %w", page.PageID, err) + } + } + return wikiPageRelativePath(page.PageID, refs), nil +} + +func (u *Usecase) attachBuildPageMetadata(ctx context.Context, detail *NodeDetail, page *model.WikiBuildPage) error { + detail.Metadata["page_type"] = page.PageType + var refs []model.SourceRef + if err := json.Unmarshal([]byte(page.SourceRefsJSON), &refs); err != nil { + return fmt.Errorf("llmwiki: decode build page source refs: %w", err) + } + type sourceReference struct { + Path string `json:"path"` + NodeID string `json:"node_id"` + VersionID string `json:"version_id"` + } + references := make([]sourceReference, 0, len(refs)) + seen := make(map[uint64]bool, len(refs)) + for _, ref := range refs { + if seen[ref.SourceID] { + continue + } + seen[ref.SourceID] = true + source, err := u.repo.GetSource(ctx, page.TenantID, ref.SourceID) + if err != nil { + return err + } + references = append(references, sourceReference{Path: source.RawPath, NodeID: encodeNodeID("raw", source.RawPath), VersionID: strconv.FormatUint(ref.SourceVersionID, 10)}) + } + if len(references) > 0 { + detail.Metadata["source_files"] = references + detail.Metadata["source_file"] = references[0].Path + detail.Metadata["source_node_id"] = references[0].NodeID + detail.Metadata["source_version"] = references[0].VersionID + } + return nil +} + +// attachSourceFiles adds the source row of one raw file to a node: its id, type, +// status and the Wiki pages generated from it. +func (u *Usecase) attachSourceFiles(ctx context.Context, detail *NodeDetail, relative string) error { + sources, err := listAllSources(ctx, u.repo, DefaultTenantID) + if err != nil { + return err + } + for _, source := range sources { + if source.RawPath != relative { + continue + } + detail.SourceID = strconv.FormatUint(source.ID, 10) + detail.Status = source.Status + detail.Metadata["source_type"] = source.SourceType + if source.CurrentVersionID != nil { + detail.Metadata["source_version"] = strconv.FormatUint(*source.CurrentVersionID, 10) + } + return u.attachWikiFiles(ctx, detail.Metadata, source.ID) + } + return nil +} + +// attachWikiFiles lists the Wiki pages generated from one source. +func (u *Usecase) attachWikiFiles(ctx context.Context, metadata map[string]any, sourceID uint64) error { + pages, active, err := u.activeBuildPages(ctx) + if err != nil { + return err + } + type wikiReference struct { + Title string `json:"title"` + Path string `json:"path"` + NodeID string `json:"node_id"` + PageID string `json:"page_id"` + } + references := make([]wikiReference, 0, len(pages)) + if !active { + metadata["wiki_files"] = references + return nil + } + for _, page := range pages { + var refs []model.SourceRef + if err := json.Unmarshal([]byte(page.SourceRefsJSON), &refs); err != nil { + return fmt.Errorf("llmwiki: decode build page source refs: %w", err) + } + matched := false + for _, ref := range refs { + if ref.SourceID == sourceID { + matched = true + break + } + } + if !matched { + continue + } + path, err := wikiBuildPagePath(page) + if err != nil { + return err + } + references = append(references, wikiReference{ + Title: page.Title, + Path: path, + NodeID: encodeNodeID("wiki", path), + PageID: page.PageID, + }) + } + metadata["wiki_files"] = references + return nil +} + +// PreviewNode returns the binary preview of one raw file. Only PDF and DOCX +// files have one: both are converted to a browser-friendly representation. +func (u *Usecase) PreviewNode(ctx context.Context, id string) (*NodePreview, error) { + unlock := u.files.LockArtifacts() + defer unlock() + + layer, relative, err := decodeNodeID(id) + if err != nil || layer != "raw" { + return nil, errors.Join(errs.ErrInvalid, errors.New("only raw files can be previewed")) + } + previewFormat := binaryPreviewFormat(relative) + if previewFormat == "" { + return nil, errors.Join(errs.ErrInvalid, errors.New("only PDF and DOCX files support binary preview")) + } + body, err := u.files.Read(ctx, filepath.ToSlash(filepath.Join("raw", relative)), MaxSourceBytes) + if errors.Is(err, fs.ErrNotExist) { + return nil, errs.ErrNotFound + } + if err != nil { + return nil, err + } + + contentType := "application/pdf" + if previewFormat == "docx" { + text, extractErr := docextract.ExtractPlainText(relative, body) + if extractErr != nil { + return nil, fmt.Errorf("llmwiki: preview source %q: %w", relative, extractErr) + } + body = []byte(text) + contentType = "text/plain; charset=utf-8" + } + return &NodePreview{Name: filepath.Base(relative), ContentType: contentType, Content: body}, nil +} + +// binaryPreviewFormat reports the preview format of a raw file, or "" when the +// file has no binary preview. +func binaryPreviewFormat(relative string) string { + switch strings.ToLower(filepath.Ext(relative)) { + case ".pdf": + return "pdf" + case ".docx": + return "docx" + default: + return "" + } +} + +// encodeNodeID encodes a tree position as an opaque, URL-safe node id. +func encodeNodeID(layer, relative string) string { + return base64.RawURLEncoding.EncodeToString([]byte(layer + ":" + relative)) +} + +// decodeNodeID splits a node id back into its layer and store-relative path. +func decodeNodeID(id string) (string, string, error) { + body, err := base64.RawURLEncoding.DecodeString(id) + if err != nil { + return "", "", errors.Join(errs.ErrInvalid, err) + } + parts := strings.SplitN(string(body), ":", 2) + if len(parts) != 2 { + return "", "", errs.ErrInvalid + } + relative := filepath.Clean(filepath.FromSlash(parts[1])) + if filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", "", errs.ErrInvalid + } + return parts[0], filepath.ToSlash(relative), nil +} + +// ParseID parses a numeric id from an API path parameter. +func ParseID(id string) (uint64, error) { + value, err := strconv.ParseUint(id, 10, 64) + if err != nil { + return 0, errors.Join(errs.ErrInvalid, err) + } + return value, nil +} diff --git a/internal/manager/biz/knowledge/llm_wiki/usecase.go b/internal/manager/biz/knowledge/llm_wiki/usecase.go new file mode 100644 index 000000000..94cb07243 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/usecase.go @@ -0,0 +1,192 @@ +package llm_wiki + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +// Usecase is the LLM Wiki entry point. It owns the Wiki file tree, the durable +// repository and the compilation pipeline, and exposes the operations the HTTP +// layer calls. +type Usecase struct { + compiler *buildCompiler + repo Repository + files *FileStore + summarizer CompilerLLM + indexer SearchIndex + log *slog.Logger + runCtx context.Context + trigger CompileTriggerOption + usage TokenUsageRecorder +} + +// jobChecker is the narrow cancellation surface of the durable repository. +type jobChecker interface { + IsCancelRequested(ctx context.Context, tenantID, jobID uint64) (bool, error) +} + +// NewWithUsageRecorder wires Wiki compilation to the existing AIOps chat +// transcript accounting without adding a Wiki-specific statistics table. +func NewWithUsageRecorder(ctx context.Context, repo Repository, files *FileStore, summarizer CompilerLLM, indexer SearchIndex, log *slog.Logger, usage TokenUsageRecorder, trigger ...CompileTriggerOption) (*Usecase, error) { + return newUsecase(ctx, repo, files, summarizer, indexer, log, usage, trigger...) +} + +func newUsecase(ctx context.Context, repo Repository, files *FileStore, summarizer CompilerLLM, indexer SearchIndex, log *slog.Logger, usage TokenUsageRecorder, trigger ...CompileTriggerOption) (*Usecase, error) { + if repo == nil || files == nil { + return nil, errors.New("llmwiki: repository and file store are required") + } + if ctx == nil { + ctx = context.Background() + } + if log == nil { + log = slog.Default() + } + if err := files.Ensure(ctx); err != nil { + return nil, err + } + var compileTrigger CompileTriggerOption + if len(trigger) > 0 { + compileTrigger = trigger[0] + } + if compileTrigger.Timeout <= 0 { + compileTrigger.Timeout = 10 * time.Minute + } + + created := &Usecase{ + repo: repo, + files: files, + summarizer: summarizer, + indexer: indexer, + log: log, + runCtx: ctx, + trigger: compileTrigger, + usage: usage, + compiler: newBuildCompiler(repo, nil, repo, files, summarizer, indexer, log), + } + if err := created.Reconcile(ctx, DefaultTenantID); err != nil { + return nil, err + } + if err := created.rebuildSearchIndex(ctx, DefaultTenantID); err != nil { + // The index is derived data. Keep the Wiki available and let the next + // compile rebuild it instead of blocking startup. + log.WarnContext(ctx, "llmwiki: search index backfill failed", slog.Any("err", err)) + } + return created, nil +} + +// vectorIndexState is the optional capability the derived index exposes so the +// usecase can backfill Qdrant when the collection is empty, for example right +// after upgrading from table-backed page vectors. +type vectorIndexState interface { + HasVectors(ctx context.Context, tenantID uint64) (bool, error) +} + +// rebuildSearchIndex repopulates lexical and vector entries from the active +// immutable build when the vector store has no points for the tenant. It never +// calls the LLM. +func (u *Usecase) rebuildSearchIndex(ctx context.Context, tenantID uint64) error { + if u.indexer == nil { + return nil + } + state, ok := u.indexer.(vectorIndexState) + if !ok { + return nil + } + has, err := state.HasVectors(ctx, tenantID) + if err != nil || has { + return err + } + active, err := u.repo.GetActiveBuild(ctx, tenantID) + if errors.Is(err, errs.ErrNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("llmwiki: load active build for index backfill: %w", err) + } + pages, err := u.repo.ListPagesByBuild(ctx, tenantID, active.ID) + if err != nil { + return fmt.Errorf("llmwiki: list pages for index backfill: %w", err) + } + artifacts := newArtifactStore(u.files) + for _, page := range pages { + if err := ctx.Err(); err != nil { + return err + } + content, err := artifacts.readPageBody(ctx, active.ID, page.PageID) + if err != nil { + return fmt.Errorf("llmwiki: read page %s for index backfill: %w", page.PageID, err) + } + if err := u.indexer.IndexPage(ctx, newIndexDocument(page, content)); err != nil { + return fmt.Errorf("llmwiki: backfill index page %s: %w", page.PageID, err) + } + } + u.log.InfoContext(ctx, "llmwiki: backfilled search index from active build", + slog.Uint64("build_id", active.ID), slog.Int("pages", len(pages))) + return nil +} + +// Reconcile restores mirrored raw files from their durable source snapshots. +// One source that cannot be restored is marked failed and skipped: it must not +// stop the Wiki from starting, because a Wiki that never opens cannot be +// repaired by re-syncing. +func (u *Usecase) Reconcile(ctx context.Context, tenantID uint64) error { + sources, err := listAllSources(ctx, u.repo, tenantID) + if err != nil { + return fmt.Errorf("llmwiki: list sources for reconcile: %w", err) + } + for _, source := range sources { + if source.CurrentVersionID == nil { + continue + } + version, err := u.repo.GetVersion(ctx, tenantID, *source.CurrentVersionID) + if err != nil { + if !errors.Is(err, errs.ErrNotFound) { + return err + } + // The version row is gone, so there is no snapshot left to restore + // this source from. It stays unusable until the next sync mirrors + // it again. + u.markUnrestorable(ctx, tenantID, source, err) + continue + } + body, err := u.files.Read(ctx, version.SnapshotPath, MaxSourceBytes) + if err != nil { + u.markUnrestorable(ctx, tenantID, source, err) + continue + } + rawPath, err := u.files.resolve("raw", source.RawPath) + if err != nil { + return err + } + if _, err := os.Stat(rawPath); errors.Is(err, fs.ErrNotExist) { + if err := atomicWrite(rawPath, body, 0o640); err != nil { + return err + } + } else if err != nil { + return err + } + } + return nil +} + +// markUnrestorable records a source Reconcile could not put back, without +// letting the recording itself stop the reconcile. +func (u *Usecase) markUnrestorable(ctx context.Context, tenantID uint64, source *model.Source, cause error) { + if err := u.repo.MarkSourceStatus(ctx, tenantID, source.ID, model.SourceFailed); err != nil && !errors.Is(err, errs.ErrNotFound) { + u.log.ErrorContext(ctx, "llmwiki: failed to mark an unrestorable source", + slog.Uint64("source_id", source.ID), + slog.String("error", err.Error())) + } + u.log.WarnContext(ctx, "llmwiki: source left out of reconcile", + slog.Uint64("source_id", source.ID), + slog.String("raw_path", source.RawPath), + slog.String("error", cause.Error())) +} diff --git a/internal/manager/biz/knowledge/llm_wiki/wiki.go b/internal/manager/biz/knowledge/llm_wiki/wiki.go new file mode 100644 index 000000000..7172cba11 --- /dev/null +++ b/internal/manager/biz/knowledge/llm_wiki/wiki.go @@ -0,0 +1,76 @@ +// Package llm_wiki compiles raw knowledge into a Markdown Wiki: sources are +// mirrored and snapshotted, the resulting corpus is chunked and summarized, the +// LLM plans a page structure, and every page is written into an isolated build +// that is published atomically. +package llm_wiki + +import "time" + +const ( + // SchemaVersion is stamped on every source version and Materialized Wiki page. + SchemaVersion = "v7" + // MaxSourceBytes caps one mirrored source snapshot. + MaxSourceBytes = 16 << 20 + // MaxPageBytes caps one generated Wiki page. + MaxPageBytes = 1 << 20 + // TargetChunkTokens is the chunk size the corpus chunker aims for. + TargetChunkTokens = 4000 + // DefaultTenantID is the id of the single-tenant installation. + DefaultTenantID = uint64(0) + // charsPerToken is the rough ratio behind estimateTokens and chunk sizing. + charsPerToken = 4 +) + +// SearchHit is one Wiki search result. +type SearchHit struct { + Layer string `json:"layer"` + PageType string `json:"page_type,omitempty"` + PageID string `json:"page_id,omitempty"` + SourceVersionID string `json:"source_version_id,omitempty"` + Title string `json:"title"` + Preview string `json:"preview"` + Score float64 `json:"score"` + MatchedNode string `json:"matched_node,omitempty"` +} + +// IndexDocument is one Wiki page handed to the search index. +type IndexDocument struct { + TenantID uint64 + PageID string + PageType string + Title string + Content string +} + +// TreeNode is one node of the Wiki tree the UI browses: a raw file, a Wiki page +// or a folder derived from the durable metadata. +type TreeNode struct { + ID string `json:"id"` + ParentID string `json:"parent_id"` + Layer string `json:"layer"` + Kind string `json:"kind"` + Name string `json:"name"` + RelativePath string `json:"relative_path"` + HasChildren bool `json:"has_children"` + ChildCount int `json:"child_count"` + DocumentCount int `json:"document_count"` + SourceID string `json:"source_id,omitempty"` + PageID string `json:"page_id,omitempty"` + PageType string `json:"page_type,omitempty"` + Status string `json:"status,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` +} + +// NodeDetail is one tree node with its content and metadata. +type NodeDetail struct { + TreeNode + Content string `json:"content"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +// NodePreview is the binary preview of one raw file. +type NodePreview struct { + Name string + ContentType string + Content []byte +} diff --git a/internal/manager/biz/knowledge/usecase.go b/internal/manager/biz/knowledge/usecase.go index 765aeca22..62e60955a 100644 --- a/internal/manager/biz/knowledge/usecase.go +++ b/internal/manager/biz/knowledge/usecase.go @@ -260,7 +260,7 @@ func (u *Usecase) UploadDoc(ctx context.Context, in UploadDocInput) (*model.Doc, title = title[:256] } now := time.Now().UTC() - return u.ingestUpload(ctx, model.Doc{ + doc := model.Doc{ SourceType: model.SourceUpload, URL: url, Title: title, @@ -269,7 +269,8 @@ func (u *Usecase) UploadDoc(ctx context.Context, in UploadDocInput) (*model.Doc, Tags: normalizeTags(in.Tags), CreatedAt: now, UpdatedAt: now, - }) + } + return u.ingestUpload(ctx, doc) } // ingestUpload (re)chunks one org-uploaded file into qdrant under @@ -508,6 +509,31 @@ func (u *Usecase) ListDocs(ctx context.Context, f ListDocsFilter) ([]*model.Doc, // works as "contains". must["tags"] = f.Tag } + if f.All { + const pageSize = 1000 + points := make([]qdrantx.SearchHit, 0, pageSize) + var offset *uint64 + for { + res, err := u.vec.Scroll(ctx, CollectionName, qdrantx.ScrollOpts{ + MustMatch: must, + Limit: pageSize, + Offset: offset, + }) + if err != nil { + return nil, fmt.Errorf("knowledge: scroll all: %w", err) + } + points = append(points, res.Points...) + if res.NextOffset == nil { + return dedupeByIDAlias(points, 0), nil + } + if offset != nil && *res.NextOffset == *offset { + return nil, errors.New("knowledge: scroll all returned a repeated offset") + } + next := *res.NextOffset + offset = &next + } + } + limit := f.Limit if limit <= 0 { limit = 200 @@ -820,6 +846,7 @@ func (u *Usecase) Search(ctx context.Context, q string, opts SearchOptions) ([]S out = append(out, SearchHit{ Doc: payloadToDoc(h.ID, h.Payload), Score: h.Score, + Layer: "raw", }) if len(out) >= limit { break @@ -1066,7 +1093,6 @@ func (u *Usecase) Sync(ctx context.Context, id uint64) (*model.Repository, error if err != nil { return u.recordSyncFailure(ctx, repo, fmt.Errorf("scan files: %w", err)) } - // Drop the previous point set first; if embedding/upsert fails // downstream we'd rather show "0 indexed, last_sync_error=…" than // keep stale rows mixed with new ones. @@ -1077,89 +1103,8 @@ func (u *Usecase) Sync(ctx context.Context, id uint64) (*model.Repository, error return u.recordSyncFailure(ctx, repo, fmt.Errorf("drop prior: %w", err)) } - // Expand each scanned file into 1+ chunks of ≤chunkChars runes each. - // Small docs become a single chunk (identical to the pre-chunking - // behaviour); large docs (RFCs, long kernel admin guides) become N - // chunks so semantic search can hit content past the first ~2500 - // chars instead of being stuck on the lead. Each chunk becomes its - // own qdrant point with its own embedding; payload dedup ('parent_url' - // + 'chunk_index') lets listings collapse to one entry per file. - type chunkRef struct { - file *scannedFile - chunkIndex int - chunkTotal int - body string - } - now := time.Now().UTC() - chunks := make([]chunkRef, 0, len(files)) - for i := range files { - parts := splitForChunks(files[i].Content) - for j, p := range parts { - // Chunk 0 prepends the title so the embedding picks up the - // "what is this doc" signal — same as the pre-chunking - // behaviour for short docs. Chunks beyond 0 carry only - // their slice (the title would dominate the vector - // otherwise). - var body string - if j == 0 { - body = files[i].Title + "\n\n" + p - } else { - body = p - } - chunks = append(chunks, chunkRef{ - file: &files[i], - chunkIndex: j, - chunkTotal: len(parts), - body: body, - }) - } - } - - // Embed in batches of 32 — keeps each request well under the - // embedding provider's per-request input cap (Zhipu = 3072 tokens - // per single input; we cap each input to chunkChars=2500 runes - // before truncateForEmbedding clips further if needed). - const batch = 32 - for i := 0; i < len(chunks); i += batch { - end := i + batch - if end > len(chunks) { - end = len(chunks) - } - texts := make([]string, 0, end-i) - for _, c := range chunks[i:end] { - texts = append(texts, truncateForEmbedding(c.body)) - } - vectors, err := u.embed.Embed(ctx, texts) - if err != nil { - return u.recordSyncFailure(ctx, repo, fmt.Errorf("embed batch %d: %w", i, err)) - } - points := make([]qdrantx.Point, 0, len(vectors)) - for j, v := range vectors { - c := chunks[i+j] - // Path: derive from URL directory so the SPA folder-tree - // view groups docs by their repo subdirectory (concepts/, - // reference/external/dns/, etc.). Repo docs never set Path - // explicitly — without this derivation the folder tree was - // silently empty for the entire repo corpus. - folder := filepath.Dir(c.file.URL) - if folder == "." || folder == "/" { - folder = "" - } - pt := repoChunkPoint(repo.ID, c.file.URL, c.chunkIndex, c.chunkTotal, v, model.Doc{ - SourceType: model.SourceRepo, - RepoID: ptrU64(repo.ID), - URL: c.file.URL, - Title: c.file.Title, - Content: c.file.Content, - Path: folder, - CreatedAt: now, - UpdatedAt: now, - }, c.body) - points = append(points, pt) - } - if err := u.vec.Upsert(ctx, CollectionName, points); err != nil { - return u.recordSyncFailure(ctx, repo, fmt.Errorf("upsert batch %d: %w", i, err)) - } + if err := u.embedScannedFiles(ctx, files, model.SourceRepo, ptrU64(repo.ID), time.Now().UTC()); err != nil { + return u.recordSyncFailure(ctx, repo, err) } // file_count tracks distinct files (the operator-facing "how many // docs are in this repo"), not the chunk fanout count. @@ -1220,61 +1165,109 @@ func (u *Usecase) SyncBuiltinVault(ctx context.Context) (int, string, error) { }); err != nil { return 0, "", fmt.Errorf("knowledge: drop prior vault: %w", err) } - now := time.Now().UTC() - type chunkRef struct { - file *scannedFile - chunkIndex, chunkN int - body string + if err := u.embedScannedFiles(ctx, files, model.SourceVault, nil, time.Now().UTC()); err != nil { + return 0, "", err } - chunks := make([]chunkRef, 0, len(files)) + u.log.Info("knowledge: built-in vault synced", + slog.String("source", source), slog.Int("file_count", len(files))) + return len(files), source, nil +} + +const knowledgeEmbeddingBatchSize = 32 + +type embeddingChunk struct { + file *scannedFile + chunkIndex int + chunkTotal int + body string +} + +func buildEmbeddingChunks(files []scannedFile) []embeddingChunk { + chunks := make([]embeddingChunk, 0, len(files)) for i := range files { parts := splitForChunks(files[i].Content) - for j, p := range parts { - body := p + for j, part := range parts { + body := part if j == 0 { - body = files[i].Title + "\n\n" + p + body = files[i].Title + "\n\n" + part } - chunks = append(chunks, chunkRef{file: &files[i], chunkIndex: j, chunkN: len(parts), body: body}) + chunks = append(chunks, embeddingChunk{ + file: &files[i], + chunkIndex: j, + chunkTotal: len(parts), + body: body, + }) } } - const batch = 32 - for i := 0; i < len(chunks); i += batch { - end := i + batch + return chunks +} + +func (u *Usecase) embedScannedFiles(ctx context.Context, files []scannedFile, sourceType string, repoID *uint64, now time.Time) error { + switch sourceType { + case model.SourceRepo: + if repoID == nil { + return errors.New("knowledge: repo embedding requires repo id") + } + case model.SourceVault: + if repoID != nil { + return errors.New("knowledge: vault embedding cannot have repo id") + } + default: + return fmt.Errorf("knowledge: unsupported scanned source type %q", sourceType) + } + chunks := buildEmbeddingChunks(files) + label := sourceType + if sourceType == model.SourceRepo { + label = "repo" + } else if sourceType == model.SourceVault { + label = "vault" + } + for i := 0; i < len(chunks); i += knowledgeEmbeddingBatchSize { + end := i + knowledgeEmbeddingBatchSize if end > len(chunks) { end = len(chunks) } texts := make([]string, 0, end-i) - for _, c := range chunks[i:end] { - texts = append(texts, truncateForEmbedding(c.body)) + for _, chunk := range chunks[i:end] { + texts = append(texts, truncateForEmbedding(chunk.body)) } vectors, err := u.embed.Embed(ctx, texts) if err != nil { - return 0, "", fmt.Errorf("knowledge: embed vault batch %d: %w", i, err) + return fmt.Errorf("knowledge: embed %s batch %d: %w", label, i, err) + } + if len(vectors) != end-i { + return fmt.Errorf("knowledge: embed %s batch %d returned %d vectors, want %d", label, i, len(vectors), end-i) } points := make([]qdrantx.Point, 0, len(vectors)) - for j, v := range vectors { - c := chunks[i+j] - folder := filepath.Dir(c.file.URL) + for j, vector := range vectors { + chunk := chunks[i+j] + folder := filepath.Dir(chunk.file.URL) if folder == "." || folder == "/" { folder = "" } - points = append(points, vaultChunkPoint(c.file.URL, c.chunkIndex, c.chunkN, v, model.Doc{ - SourceType: model.SourceVault, - URL: c.file.URL, - Title: c.file.Title, - Content: c.file.Content, + doc := model.Doc{ + SourceType: sourceType, + RepoID: repoID, + URL: chunk.file.URL, + Title: chunk.file.Title, + Content: chunk.file.Content, Path: folder, CreatedAt: now, UpdatedAt: now, - }, c.body)) + } + var point qdrantx.Point + if repoID != nil { + point = repoChunkPoint(*repoID, chunk.file.URL, chunk.chunkIndex, chunk.chunkTotal, vector, doc, chunk.body) + } else { + point = vaultChunkPoint(chunk.file.URL, chunk.chunkIndex, chunk.chunkTotal, vector, doc, chunk.body) + } + points = append(points, point) } if err := u.vec.Upsert(ctx, CollectionName, points); err != nil { - return 0, "", fmt.Errorf("knowledge: upsert vault batch %d: %w", i, err) + return fmt.Errorf("knowledge: upsert %s batch %d: %w", label, i, err) } } - u.log.Info("knowledge: built-in vault synced", - slog.String("source", source), slog.Int("file_count", len(files))) - return len(files), source, nil + return nil } // cloudVaultAttempts / cloudVaultPerTry tune the retry loop in fetchCloudVault. @@ -1982,6 +1975,7 @@ func ptrU64(v uint64) *uint64 { return &v } func manualDocID(title string) uint64 { return docID("manual||" + title) } + func repoDocID(repoID uint64, url string) uint64 { return docID(fmt.Sprintf("repo||%d||%s", repoID, url)) } diff --git a/internal/manager/data/knowledge/llm_wiki/index/index.go b/internal/manager/data/knowledge/llm_wiki/index/index.go new file mode 100644 index 000000000..432c0b64a --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/index/index.go @@ -0,0 +1,420 @@ +// Package index implements the rebuildable Wiki lexical and vector indexes. +// Lexical matching lives in the application database (portable LIKE over +// wiki_lexical); page vectors live in a dedicated Qdrant collection. +package index + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "sort" + "strconv" + "strings" + + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/embedding" + "github.com/ongridio/ongrid/internal/pkg/qdrantx" + "gorm.io/gorm" +) + +// CollectionName is the dedicated Qdrant collection for LLM Wiki page vectors. +const CollectionName = "ongrid_llm_wiki" + +// Payload keys written to Qdrant. tenant_id is stored as a decimal string so +// server-side filters never hit qdrant's int64 JSON number limit. +const ( + payloadTenantID = "tenant_id" + payloadPageID = "page_id" + payloadPageType = "page_type" + payloadTitle = "title" + payloadBodyHash = "body_hash" +) + +// VectorStore is the narrow qdrant surface the Wiki index consumes. +// *qdrantx.Client satisfies it; tests can inject a fake. +type VectorStore interface { + EnsureCollection(ctx context.Context, name string, dim int) error + EnsurePayloadIndex(ctx context.Context, collection, field, schema string) error + Upsert(ctx context.Context, collection string, points []qdrantx.Point) error + DeleteByID(ctx context.Context, collection string, id uint64) error + DeleteByFilter(ctx context.Context, collection string, mustMatch map[string]any) error + GetPoints(ctx context.Context, collection string, ids []uint64) ([]qdrantx.SearchHit, error) + Search(ctx context.Context, collection string, vector []float32, opts qdrantx.SearchOpts) ([]qdrantx.SearchHit, error) + Scroll(ctx context.Context, collection string, opts qdrantx.ScrollOpts) (*qdrantx.ScrollResult, error) +} + +type Index struct { + db *gorm.DB + vec VectorStore + embed embedding.Embedder + dim int +} + +// New wires the search index onto the migrated Wiki schema and the Qdrant +// collection. Callers run store.Migrate first (startup migrations do this for +// every backend). When an embedder is configured, Qdrant is a hard dependency: +// a missing client or unreachable collection fails startup. +func New(ctx context.Context, db *gorm.DB, vec VectorStore, embed embedding.Embedder, dim int, log *slog.Logger) (*Index, error) { + if db == nil { + return nil, errors.New("llmwiki index: database is required") + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if log == nil { + log = slog.Default() + } + if embed != nil { + // The embedder is the source of truth for the vector size; the + // caller-supplied dim only matters when no embedder is wired. + if embedDim := embed.Dim(); embedDim > 0 { + dim = embedDim + } + } + if embed != nil && vec == nil { + return nil, errors.New("llmwiki index: qdrant client is required when embedding is configured") + } + if embed != nil { + if err := vec.EnsureCollection(ctx, CollectionName, dim); err != nil { + return nil, fmt.Errorf("llmwiki index: ensure qdrant collection: %w", err) + } + if err := vec.EnsurePayloadIndex(ctx, CollectionName, payloadTenantID, "keyword"); err != nil { + return nil, fmt.Errorf("llmwiki index: ensure qdrant payload index: %w", err) + } + } + log.Info("llm wiki search index ready", + slog.String("collection", CollectionName), + slog.Bool("vector_search", embed != nil), + slog.Int("dim", dim)) + return &Index{db: db, vec: vec, embed: embed, dim: dim}, nil +} + +func (i *Index) IndexPage(ctx context.Context, document biz.IndexDocument) error { + if strings.TrimSpace(document.PageID) == "" { + return errors.New("llmwiki index: page id is required") + } + pageKey := fmt.Sprintf("%d:%s", document.TenantID, document.PageID) + if err := i.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Where("page_key = ?", pageKey).Delete(&model.WikiLexical{}).Error; err != nil { + return fmt.Errorf("delete old lexical page: %w", err) + } + row := model.WikiLexical{ + PageKey: pageKey, + TenantID: document.TenantID, + PageID: document.PageID, + PageType: document.PageType, + Title: document.Title, + Content: document.Content, + } + if err := tx.Create(&row).Error; err != nil { + return fmt.Errorf("insert lexical page: %w", err) + } + return nil + }); err != nil { + return fmt.Errorf("llmwiki index: update lexical page: %w", err) + } + + if i.embed == nil { + return nil + } + bodyHash := contentHash(document.Content) + id := pointID(document.TenantID, document.PageID) + existing, err := i.vec.GetPoints(ctx, CollectionName, []uint64{id}) + if err != nil { + return fmt.Errorf("llmwiki index: load existing vector: %w", err) + } + if len(existing) > 0 { + if hash, _ := existing[0].Payload[payloadBodyHash].(string); hash == bodyHash { + return nil + } + } + vectors, err := i.embed.Embed(ctx, []string{document.Title + "\n\n" + document.Content}) + if err != nil { + return errors.Join(fmt.Errorf("llmwiki index: embed page: %w", err), i.deleteVector(ctx, document.TenantID, document.PageID)) + } + if len(vectors) != 1 || len(vectors[0]) == 0 { + cause := errors.New("llmwiki index: embedder returned an unexpected vector count") + return errors.Join(cause, i.deleteVector(ctx, document.TenantID, document.PageID)) + } + if i.dim > 0 && len(vectors[0]) != i.dim { + cause := fmt.Errorf("llmwiki index: vector dimension %d does not match %d", len(vectors[0]), i.dim) + return errors.Join(cause, i.deleteVector(ctx, document.TenantID, document.PageID)) + } + point := qdrantx.Point{ + ID: id, + Vector: vectors[0], + Payload: map[string]any{ + payloadTenantID: strconv.FormatUint(document.TenantID, 10), + payloadPageID: document.PageID, + payloadPageType: document.PageType, + payloadTitle: document.Title, + payloadBodyHash: bodyHash, + }, + } + if err := i.vec.Upsert(ctx, CollectionName, []qdrantx.Point{point}); err != nil { + return fmt.Errorf("llmwiki index: save vector: %w", err) + } + return nil +} + +// Clear removes every derived search entry for one tenant: first the Qdrant +// points, then the lexical rows. Qdrant and SQL cannot share a transaction, so +// the derived state is briefly inconsistent after a partial failure; the next +// compile rebuilds both sides. Without an embedder the collection may not +// exist, so only the lexical side is cleared. +func (i *Index) Clear(ctx context.Context, tenantID uint64) error { + if i.embed != nil && i.vec != nil { + if err := i.vec.DeleteByFilter(ctx, CollectionName, map[string]any{payloadTenantID: tenantIDKey(tenantID)}); err != nil { + return fmt.Errorf("llmwiki index: clear vectors: %w", err) + } + } + return i.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Where("tenant_id = ?", tenantID).Delete(&model.WikiLexical{}).Error; err != nil { + return fmt.Errorf("llmwiki index: clear lexical pages: %w", err) + } + return nil + }) +} + +// HasVectors reports whether the tenant has at least one page vector. Disabled +// vector search reports true so callers skip backfilling. +func (i *Index) HasVectors(ctx context.Context, tenantID uint64) (bool, error) { + if i.embed == nil || i.vec == nil { + return true, nil + } + result, err := i.vec.Scroll(ctx, CollectionName, qdrantx.ScrollOpts{ + MustMatch: map[string]any{payloadTenantID: tenantIDKey(tenantID)}, + Limit: 1, + }) + if err != nil { + return false, fmt.Errorf("llmwiki index: probe vectors: %w", err) + } + return len(result.Points) > 0, nil +} + +func (i *Index) deleteVector(ctx context.Context, tenantID uint64, pageID string) error { + if i.vec == nil { + return nil + } + if err := i.vec.DeleteByID(ctx, CollectionName, pointID(tenantID, pageID)); err != nil { + return fmt.Errorf("llmwiki index: delete stale vector: %w", err) + } + return nil +} + +func (i *Index) Search(ctx context.Context, tenantID uint64, query string, limit int) ([]biz.SearchHit, error) { + query = strings.TrimSpace(query) + if query == "" { + return nil, nil + } + if limit <= 0 { + limit = 10 + } + lexical, lexicalErr := i.searchLexical(ctx, tenantID, query, limit*2) + vector, vectorErr := i.searchVector(ctx, tenantID, query, limit*2) + // Qdrant is a hard dependency: a vector failure must not be masked by + // lexical-only results. + if vectorErr != nil { + return nil, vectorErr + } + if lexicalErr != nil { + return mergeRRF(query, nil, vector, limit), nil + } + return mergeRRF(query, lexical, vector, limit), nil +} + +func (i *Index) searchLexical(ctx context.Context, tenantID uint64, query string, limit int) ([]biz.SearchHit, error) { + type lexicalRow struct { + PageID string `gorm:"column:page_id"` + PageType string `gorm:"column:page_type"` + Title string `gorm:"column:title"` + Preview string `gorm:"column:preview"` + MatchRank float64 `gorm:"column:match_rank"` + } + var rows []lexicalRow + pattern := "%" + escapeLike(query) + "%" + // SUBSTR, CASE and ESCAPE are portable across MySQL and SQLite. match_rank + // prefers title matches, then aliases, then body-only matches. RANK is a + // reserved word on MySQL 8, so the alias is not named rank. + const statement = `SELECT page_id, page_type, title, SUBSTR(content, 1, 800) AS preview, + CASE WHEN title LIKE ? ESCAPE '!' THEN 2 WHEN aliases LIKE ? ESCAPE '!' THEN 1 ELSE 0 END AS match_rank + FROM wiki_lexical + WHERE tenant_id = ? AND (title LIKE ? ESCAPE '!' OR aliases LIKE ? ESCAPE '!' OR content LIKE ? ESCAPE '!') + ORDER BY match_rank DESC, page_id ASC LIMIT ?` + err := i.db.WithContext(ctx).Raw(statement, pattern, pattern, tenantID, pattern, pattern, pattern, limit).Scan(&rows).Error + if err != nil { + return nil, fmt.Errorf("llmwiki index: search lexical: %w", err) + } + hits := make([]biz.SearchHit, 0, len(rows)) + for _, row := range rows { + hits = append(hits, biz.SearchHit{Layer: "wiki", PageID: row.PageID, PageType: row.PageType, Title: row.Title, Preview: row.Preview, Score: (row.MatchRank + 1) / 3}) + } + return hits, nil +} + +func (i *Index) searchVector(ctx context.Context, tenantID uint64, query string, limit int) ([]biz.SearchHit, error) { + if i.embed == nil || i.vec == nil { + return nil, nil + } + vectors, err := i.embed.Embed(ctx, []string{query}) + if err != nil { + return nil, fmt.Errorf("llmwiki index: embed query: %w", err) + } + if len(vectors) != 1 || len(vectors[0]) == 0 { + return nil, errors.New("llmwiki index: embedder returned an unexpected query vector count") + } + points, err := i.vec.Search(ctx, CollectionName, vectors[0], qdrantx.SearchOpts{ + Limit: limit, + MustMatch: map[string]any{payloadTenantID: tenantIDKey(tenantID)}, + }) + if err != nil { + return nil, fmt.Errorf("llmwiki index: search vectors: %w", err) + } + pageIDs := make([]string, 0, len(points)) + for _, point := range points { + if pageID, _ := point.Payload[payloadPageID].(string); pageID != "" { + pageIDs = append(pageIDs, pageID) + } + } + previews, err := i.lexicalRows(ctx, tenantID, pageIDs) + if err != nil { + return nil, err + } + hits := make([]biz.SearchHit, 0, len(points)) + for _, point := range points { + pageID, _ := point.Payload[payloadPageID].(string) + if pageID == "" { + continue + } + hit := biz.SearchHit{Layer: "wiki", PageID: pageID, Score: point.Score} + if row, ok := previews[pageID]; ok { + hit.PageType = row.PageType + hit.Title = row.Title + hit.Preview = row.Preview + } else { + // The point is authoritative enough to answer with payload + // metadata when the lexical mirror is missing. + hit.PageType, _ = point.Payload[payloadPageType].(string) + hit.Title, _ = point.Payload[payloadTitle].(string) + } + hits = append(hits, hit) + } + sort.SliceStable(hits, func(left, right int) bool { return hits[left].Score > hits[right].Score }) + if len(hits) > limit { + hits = hits[:limit] + } + return hits, nil +} + +type lexicalPreview struct { + PageID string `gorm:"column:page_id"` + PageType string `gorm:"column:page_type"` + Title string `gorm:"column:title"` + Preview string `gorm:"column:preview"` +} + +// lexicalRows loads the display metadata of vector hits in one query. +func (i *Index) lexicalRows(ctx context.Context, tenantID uint64, pageIDs []string) (map[string]lexicalPreview, error) { + out := make(map[string]lexicalPreview, len(pageIDs)) + if len(pageIDs) == 0 { + return out, nil + } + var rows []lexicalPreview + err := i.db.WithContext(ctx).Raw( + "SELECT page_id, page_type, title, SUBSTR(content, 1, 800) AS preview FROM wiki_lexical WHERE tenant_id = ? AND page_id IN ?", + tenantID, pageIDs, + ).Scan(&rows).Error + if err != nil { + return nil, fmt.Errorf("llmwiki index: load vector previews: %w", err) + } + for _, row := range rows { + out[row.PageID] = row + } + return out, nil +} + +// pointID derives the stable Qdrant point id of one page. Rewriting a page +// overwrites its point instead of duplicating it. +func pointID(tenantID uint64, pageID string) uint64 { + sum := sha256.Sum256([]byte("wiki:" + strconv.FormatUint(tenantID, 10) + ":" + pageID)) + return binary.BigEndian.Uint64(sum[:8]) +} + +func tenantIDKey(tenantID uint64) string { + return strconv.FormatUint(tenantID, 10) +} + +// escapeLike neutralises LIKE wildcards so a query matches literal text. The +// caller pairs it with ESCAPE '!'. +func escapeLike(query string) string { + replacer := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_") + return replacer.Replace(query) +} + +// mergeRRF fuses the lexical and vector legs by reciprocal rank fusion. RRF +// decides the order only: 1/(60+rank) is a rank signal that spans the single +// hundredth between 0.0246 and 0.0143 across the top ten, so it must never leave +// this function as a Score a caller thresholds on. Each hit keeps the relevance +// its own leg produced — the vector cosine, or the three-level lexical match +// rank — and a page found by both legs reports the cosine. +func mergeRRF(query string, lexical []biz.SearchHit, vector []biz.SearchHit, limit int) []biz.SearchHit { + legs := []struct { + hits []biz.SearchHit + fromVector bool + }{{hits: lexical}, {hits: vector, fromVector: true}} + + byID := make(map[string]biz.SearchHit) + fused := make(map[string]float64) + for _, leg := range legs { + for rank, hit := range leg.hits { + existing, seen := byID[hit.PageID] + switch { + case !seen: + existing = hit + case leg.fromVector: + existing.Score = hit.Score + } + byID[hit.PageID] = existing + fused[hit.PageID] += 1 / float64(60+rank+1) + } + } + + // A title match outranks an equally ranked body match. The nudge belongs to + // the fusion value alone: adding it to Score would inflate a number callers + // read as relevance. + needle := strings.ToLower(strings.TrimSpace(query)) + for pageID, hit := range byID { + if strings.Contains(strings.ToLower(hit.Title), needle) { + fused[pageID] += 0.005 + } + } + + out := make([]biz.SearchHit, 0, len(byID)) + for _, hit := range byID { + out = append(out, hit) + } + sort.Slice(out, func(left, right int) bool { + leftFused, rightFused := fused[out[left].PageID], fused[out[right].PageID] + if leftFused == rightFused { + return out[left].PageID < out[right].PageID + } + return leftFused > rightFused + }) + if len(out) > limit { + out = out[:limit] + } + return out +} + +func contentHash(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/manager/data/knowledge/llm_wiki/index/index_test.go b/internal/manager/data/knowledge/llm_wiki/index/index_test.go new file mode 100644 index 000000000..2b46da064 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/index/index_test.go @@ -0,0 +1,438 @@ +package index + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/glebarez/sqlite" + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + store "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki/store" + "github.com/ongridio/ongrid/internal/pkg/embedding" + "github.com/ongridio/ongrid/internal/pkg/qdrantx" + "gorm.io/gorm" +) + +type indexEmbedder struct { + calls int + err error +} + +func (*indexEmbedder) Dim() int { return 2 } + +func (e *indexEmbedder) Embed(context.Context, []string) ([][]float32, error) { + e.calls++ + if e.err != nil { + return nil, e.err + } + return [][]float32{{1, 0}}, nil +} + +// fakeVectorStore is a minimal in-memory stand-in for *qdrantx.Client. +type fakeVectorStore struct { + points map[uint64]qdrantx.SearchHit + order []uint64 + ensureErr error + searchErr error + upserts int + deletesByID int + deletesByFilt int + lastSearch qdrantx.SearchOpts + lastFilter map[string]any +} + +func newFakeVectorStore() *fakeVectorStore { + return &fakeVectorStore{points: map[uint64]qdrantx.SearchHit{}} +} + +func (f *fakeVectorStore) EnsureCollection(context.Context, string, int) error { return f.ensureErr } + +func (f *fakeVectorStore) EnsurePayloadIndex(context.Context, string, string, string) error { + return nil +} + +func (f *fakeVectorStore) Upsert(_ context.Context, _ string, points []qdrantx.Point) error { + f.upserts++ + for _, point := range points { + if _, exists := f.points[point.ID]; !exists { + f.order = append(f.order, point.ID) + } + f.points[point.ID] = qdrantx.SearchHit{ID: point.ID, Payload: point.Payload} + } + return nil +} + +func (f *fakeVectorStore) DeleteByID(_ context.Context, _ string, id uint64) error { + f.deletesByID++ + delete(f.points, id) + f.dropOrder(id) + return nil +} + +func (f *fakeVectorStore) DeleteByFilter(_ context.Context, _ string, must map[string]any) error { + if len(must) == 0 { + return errors.New("fake qdrant: empty filter") + } + f.deletesByFilt++ + f.lastFilter = must + for id, point := range f.points { + if testPayloadMatches(point.Payload, must) { + delete(f.points, id) + f.dropOrder(id) + } + } + return nil +} + +func (f *fakeVectorStore) GetPoints(_ context.Context, _ string, ids []uint64) ([]qdrantx.SearchHit, error) { + out := make([]qdrantx.SearchHit, 0, len(ids)) + for _, id := range ids { + if point, ok := f.points[id]; ok { + out = append(out, point) + } + } + return out, nil +} + +func (f *fakeVectorStore) Search(_ context.Context, _ string, _ []float32, opts qdrantx.SearchOpts) ([]qdrantx.SearchHit, error) { + f.lastSearch = opts + if f.searchErr != nil { + return nil, f.searchErr + } + out := make([]qdrantx.SearchHit, 0, len(f.order)) + for rank, id := range f.order { + point := f.points[id] + if testPayloadMatches(point.Payload, opts.MustMatch) { + point.Score = 1 / float64(rank+1) + out = append(out, point) + } + } + if opts.Limit > 0 && len(out) > opts.Limit { + out = out[:opts.Limit] + } + return out, nil +} + +func (f *fakeVectorStore) Scroll(_ context.Context, _ string, opts qdrantx.ScrollOpts) (*qdrantx.ScrollResult, error) { + out := make([]qdrantx.SearchHit, 0, len(f.order)) + for _, id := range f.order { + point := f.points[id] + if testPayloadMatches(point.Payload, opts.MustMatch) { + out = append(out, point) + } + } + if opts.Limit > 0 && len(out) > opts.Limit { + out = out[:opts.Limit] + } + return &qdrantx.ScrollResult{Points: out}, nil +} + +func (f *fakeVectorStore) dropOrder(id uint64) { + kept := f.order[:0] + for _, existing := range f.order { + if existing != id { + kept = append(kept, existing) + } + } + f.order = kept +} + +func testPayloadMatches(payload map[string]any, must map[string]any) bool { + for key, want := range must { + got, ok := payload[key] + if !ok || fmt.Sprint(got) != fmt.Sprint(want) { + return false + } + } + return true +} + +func testIndex(t *testing.T, vec VectorStore, embedder embedding.Embedder, dimension int) (*Index, *gorm.DB) { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := store.Migrate(db); err != nil { + t.Fatal(err) + } + idx, err := New(context.Background(), db, vec, embedder, dimension, nil) + if err != nil { + t.Fatal(err) + } + return idx, db +} + +func TestIndex_SearchesDocumentsAndIsolatesTenant(t *testing.T) { + idx, _ := testIndex(t, nil, nil, 0) + documents := []biz.IndexDocument{ + {TenantID: 1, PageID: "dns", PageType: "generated", Title: "DNS 排障", Content: "检查解析超时"}, + {TenantID: 2, PageID: "dns-other", PageType: "generated", Title: "DNS", Content: "另一个租户的解析文档"}, + } + for _, document := range documents { + if err := idx.IndexPage(context.Background(), document); err != nil { + t.Fatal(err) + } + } + hits, err := idx.Search(context.Background(), 1, "解析", 10) + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 || hits[0].PageID != "dns" { + t.Fatalf("hits = %+v", hits) + } +} + +func TestIndex_ClearRemovesOnlyRequestedTenant(t *testing.T) { + vec := newFakeVectorStore() + idx, _ := testIndex(t, vec, &indexEmbedder{}, 2) + for _, document := range []biz.IndexDocument{ + {TenantID: 1, PageID: "one", PageType: "generated", Title: "One", Content: "shared keyword"}, + {TenantID: 2, PageID: "two", PageType: "generated", Title: "Two", Content: "shared keyword"}, + } { + if err := idx.IndexPage(context.Background(), document); err != nil { + t.Fatal(err) + } + } + if err := idx.Clear(context.Background(), 1); err != nil { + t.Fatal(err) + } + if vec.deletesByFilt != 1 { + t.Fatalf("qdrant clear calls = %d", vec.deletesByFilt) + } + if vec.lastFilter[payloadTenantID] != "1" { + t.Fatalf("clear filter = %+v", vec.lastFilter) + } + hits, err := idx.Search(context.Background(), 1, "keyword", 10) + if err != nil { + t.Fatal(err) + } + if len(hits) != 0 { + t.Fatalf("cleared tenant hits = %+v", hits) + } + hits, err = idx.Search(context.Background(), 2, "keyword", 10) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("other tenant lost its index entry") + } +} + +func TestIndex_TreatsLikeWildcardsLiterally(t *testing.T) { + idx, _ := testIndex(t, nil, nil, 0) + for _, document := range []biz.IndexDocument{ + {TenantID: 1, PageID: "percent", PageType: "generated", Title: "Disk", Content: "disk 50% full"}, + {TenantID: 1, PageID: "plain", PageType: "generated", Title: "Disk", Content: "disk cpu usage"}, + } { + if err := idx.IndexPage(context.Background(), document); err != nil { + t.Fatal(err) + } + } + hits, err := idx.Search(context.Background(), 1, "50%", 10) + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 || hits[0].PageID != "percent" { + t.Fatalf("literal wildcard hits = %+v", hits) + } + hits, err = idx.Search(context.Background(), 1, "c_u", 10) + if err != nil { + t.Fatal(err) + } + if len(hits) != 0 { + t.Fatalf("underscore wildcard hits = %+v", hits) + } +} + +func TestIndex_ReusesBodyHashAndDropsStaleVectorOnEmbeddingFailure(t *testing.T) { + embedder := &indexEmbedder{} + vec := newFakeVectorStore() + idx, _ := testIndex(t, vec, embedder, 2) + document := biz.IndexDocument{TenantID: 0, PageID: "hash-page", PageType: "generated", Title: "Hash", Content: "稳定正文"} + if err := idx.IndexPage(context.Background(), document); err != nil { + t.Fatal(err) + } + if err := idx.IndexPage(context.Background(), document); err != nil { + t.Fatal(err) + } + if embedder.calls != 1 { + t.Fatalf("embedding calls = %d; want body-hash reuse", embedder.calls) + } + if vec.upserts != 1 { + t.Fatalf("qdrant upserts = %d; want 1", vec.upserts) + } + embedder.err = errors.New("embedding offline") + document.Content = "更新后的正文" + if err := idx.IndexPage(context.Background(), document); err == nil { + t.Fatal("embedding failure was ignored") + } + if len(vec.points) != 0 { + t.Fatalf("stale vector count = %d", len(vec.points)) + } + if vec.deletesByID != 1 { + t.Fatalf("qdrant delete calls = %d; want 1", vec.deletesByID) + } + hits, err := idx.searchLexical(context.Background(), 0, "更新", 10) + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 || hits[0].PageID != document.PageID { + t.Fatalf("lexical fallback hits = %+v", hits) + } +} + +func TestIndex_VectorSearchFiltersTenantAndEnrichesFromLexical(t *testing.T) { + vec := newFakeVectorStore() + idx, _ := testIndex(t, vec, &indexEmbedder{}, 2) + for _, document := range []biz.IndexDocument{ + {TenantID: 1, PageID: "dns", PageType: "generated", Title: "DNS 排障", Content: "检查解析超时"}, + {TenantID: 2, PageID: "dns-other", PageType: "generated", Title: "另一个租户", Content: "另一个租户的文档"}, + } { + if err := idx.IndexPage(context.Background(), document); err != nil { + t.Fatal(err) + } + } + // "zzz" matches no lexical row, so every hit must come from the vector leg. + hits, err := idx.Search(context.Background(), 1, "zzz", 10) + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 || hits[0].PageID != "dns" { + t.Fatalf("vector hits = %+v", hits) + } + if hits[0].Title != "DNS 排障" || hits[0].Preview == "" || hits[0].PageType != "generated" { + t.Fatalf("vector hit not enriched from lexical row: %+v", hits[0]) + } + if vec.lastSearch.MustMatch[payloadTenantID] != "1" { + t.Fatalf("search filter = %+v", vec.lastSearch.MustMatch) + } +} + +func TestIndex_VectorFailureIsNotMaskedByLexicalResults(t *testing.T) { + vec := newFakeVectorStore() + idx, _ := testIndex(t, vec, &indexEmbedder{}, 2) + document := biz.IndexDocument{TenantID: 1, PageID: "dns", PageType: "generated", Title: "DNS", Content: "解析超时"} + if err := idx.IndexPage(context.Background(), document); err != nil { + t.Fatal(err) + } + vec.searchErr = errors.New("qdrant down") + if _, err := idx.Search(context.Background(), 1, "DNS", 10); err == nil { + t.Fatal("vector failure was masked by lexical results") + } +} + +func TestIndex_RequiresQdrantWhenEmbeddingConfigured(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := store.Migrate(db); err != nil { + t.Fatal(err) + } + if _, err := New(context.Background(), db, nil, &indexEmbedder{}, 2, nil); err == nil { + t.Fatal("missing qdrant client was accepted with an embedder") + } +} + +func TestIndex_EnsureCollectionFailureIsFatal(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := store.Migrate(db); err != nil { + t.Fatal(err) + } + vec := newFakeVectorStore() + vec.ensureErr = errors.New("qdrant unreachable") + if _, err := New(context.Background(), db, vec, &indexEmbedder{}, 2, nil); err == nil { + t.Fatal("qdrant ensure failure was ignored") + } +} + +func TestIndex_PointIDIsStableAndTenantScoped(t *testing.T) { + first := pointID(1, "dns") + if first != pointID(1, "dns") { + t.Fatal("point id is not stable") + } + if first == pointID(2, "dns") || first == pointID(1, "other") { + t.Fatal("point id is not scoped to tenant and page") + } +} + +func TestIndex_HasVectorsReportsTenantState(t *testing.T) { + vec := newFakeVectorStore() + idx, _ := testIndex(t, vec, &indexEmbedder{}, 2) + has, err := idx.HasVectors(context.Background(), 1) + if err != nil || has { + t.Fatalf("empty store has vectors=%v err=%v", has, err) + } + if err := idx.IndexPage(context.Background(), biz.IndexDocument{TenantID: 1, PageID: "dns", PageType: "generated", Title: "DNS", Content: "解析超时"}); err != nil { + t.Fatal(err) + } + has, err = idx.HasVectors(context.Background(), 1) + if err != nil || !has { + t.Fatalf("populated store has vectors=%v err=%v", has, err) + } +} + +// sameScore compares a reported relevance against the value a leg produced. +func sameScore(got, want float64) bool { + diff := got - want + return diff < 1e-9 && diff > -1e-9 +} + +// TestMergeRRF_OrdersByFusionAndReportsRelevance pins the score contract of a +// Wiki search: RRF decides the order, but Score stays the relevance the leg +// produced. Publishing 1/(60+rank) instead made every hit report ~0.016 — a +// uniform "0.02" on the Knowledge page, and a silent fail against the 0.6 gate +// the agent's knowledge prologue applies. +func TestMergeRRF_OrdersByFusionAndReportsRelevance(t *testing.T) { + lexical := []biz.SearchHit{ + {PageID: "body-only", Title: "unrelated", Score: 1.0 / 3}, + {PageID: "both", Title: "unrelated", Score: 1.0 / 3}, + } + vector := []biz.SearchHit{ + {PageID: "both", Title: "unrelated", Score: 0.82}, + {PageID: "vector-only", Title: "unrelated", Score: 0.71}, + } + + hits := mergeRRF("needle", lexical, vector, 10) + + if len(hits) != 3 { + t.Fatalf("hits = %+v", hits) + } + // The page both legs agree on wins the fusion, and the cosine the vector leg + // measured is what callers get to see. + if hits[0].PageID != "both" || !sameScore(hits[0].Score, 0.82) { + t.Fatalf("top hit = %+v, want both at its vector cosine", hits[0]) + } + byID := make(map[string]float64, len(hits)) + for _, hit := range hits { + byID[hit.PageID] = hit.Score + } + if !sameScore(byID["body-only"], 1.0/3) { + t.Fatalf("lexical-only score = %v, want the match rank tier", byID["body-only"]) + } + if !sameScore(byID["vector-only"], 0.71) { + t.Fatalf("vector-only score = %v, want the cosine", byID["vector-only"]) + } +} + +// TestMergeRRF_TitleBonusOrdersButDoesNotInflateScore — the title match bonus is +// an ordering nudge. Adding it to Score, as the old code did, reported a +// relevance the search never measured. +func TestMergeRRF_TitleBonusOrdersButDoesNotInflateScore(t *testing.T) { + lexical := []biz.SearchHit{{PageID: "titled", Title: "DNS 排障", Score: 1.0 / 3}} + vector := []biz.SearchHit{{PageID: "other", Title: "别的东西", Score: 0.9}} + + hits := mergeRRF("DNS", lexical, vector, 10) + + if len(hits) != 2 || hits[0].PageID != "titled" { + t.Fatalf("hits = %+v, want the title match first", hits) + } + if !sameScore(hits[0].Score, 1.0/3) { + t.Fatalf("score = %v, want the lexical tier without the ordering bonus", hits[0].Score) + } +} diff --git a/internal/manager/data/knowledge/llm_wiki/mysql_integration_test.go b/internal/manager/data/knowledge/llm_wiki/mysql_integration_test.go new file mode 100644 index 000000000..ed23f3e59 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/mysql_integration_test.go @@ -0,0 +1,99 @@ +package llm_wiki + +import ( + "context" + "os" + "testing" + + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + store "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki/store" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + gormmysql "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +const mysqlTestTenant = 9 + +// TestMySQLBackendIntegration exercises the MySQL dialect on a real server. +// Set WIKI_MYSQL_TEST_DSN to a scratch database DSN, for example: +// +// WIKI_MYSQL_TEST_DSN='user:pass@tcp(127.0.0.1:3306)/ongrid_wiki_test?parseTime=true' go test ./... +func TestMySQLBackendIntegration(t *testing.T) { + dsn := os.Getenv("WIKI_MYSQL_TEST_DSN") + if dsn == "" { + t.Skip("WIKI_MYSQL_TEST_DSN not set") + } + ctx := context.Background() + db, err := gorm.Open(gormmysql.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := store.Migrate(db); err != nil { + t.Fatalf("migrate: %v", err) + } + cleanupMySQLTestTenant(t, db) + opened, err := Open(ctx, db, nil, nil, 0, nil) + if err != nil { + t.Fatal(err) + } + repo := opened.Repository() + + source, version, changed, err := repo.UpsertSourceVersion(ctx, + &model.Source{TenantID: mysqlTestTenant, SourceKey: "manual:mysql", SourceType: "manual", RawPath: "mysql.md", Status: model.SourcePending}, + &model.SourceVersion{TenantID: mysqlTestTenant, SHA256: "hash-mysql", SnapshotPath: "mysql.md", SchemaVersion: "v1"}, + ) + if err != nil || !changed || source == nil || version == nil { + t.Fatalf("upsert source: changed=%v err=%v", changed, err) + } + + build, err := repo.CreateBuild(ctx, mysqlTestTenant) + if err != nil { + t.Fatal(err) + } + page := &model.WikiBuildPage{BuildID: build.ID, TenantID: mysqlTestTenant, PageID: "manual-page", PageType: "generated", Title: "磁盘容量", BodyPath: "manual-page.md", BodySHA256: "hash-page", SourceRefsJSON: "[]"} + if err := repo.CreatePagesBatch(ctx, []*model.WikiBuildPage{page}); err != nil { + t.Fatal(err) + } + if err := repo.UpdateBuildStatus(ctx, mysqlTestTenant, build.ID, model.BuildValidated, ""); err != nil { + t.Fatal(err) + } + if err := repo.ActivateBuild(ctx, mysqlTestTenant, build.ID); err != nil { + t.Fatal(err) + } + + document := biz.IndexDocument{TenantID: mysqlTestTenant, PageID: page.PageID, PageType: page.PageType, Title: page.Title, Content: "磁盘容量 50% 使用率，请检查 disk usage"} + if err := opened.SearchIndex.IndexPage(ctx, document); err != nil { + t.Fatalf("index page: %v", err) + } + hits, err := opened.SearchIndex.Search(ctx, mysqlTestTenant, "50%", 10) + if err != nil { + t.Fatalf("search literal wildcard: %v", err) + } + if len(hits) != 1 || hits[0].PageID != page.PageID { + t.Fatalf("literal wildcard hits = %+v", hits) + } + hits, err = opened.SearchIndex.Search(ctx, mysqlTestTenant, "磁盘", 10) + if err != nil { + t.Fatalf("search cjk: %v", err) + } + if len(hits) != 1 || hits[0].Title != page.Title { + t.Fatalf("cjk hits = %+v", hits) + } + active, err := repo.GetActiveBuild(ctx, mysqlTestTenant) + if err != nil || active.ID != build.ID { + t.Fatalf("active build = %+v err=%v", active, err) + } + _, total, err := repo.ListSources(ctx, mysqlTestTenant, "", 10) + if err != nil || total != 1 { + t.Fatalf("list sources total=%d err=%v", total, err) + } +} + +func cleanupMySQLTestTenant(t *testing.T, db *gorm.DB) { + t.Helper() + for _, table := range []string{"wiki_build_pages", "wiki_builds", "wiki_compile_jobs", "wiki_source_versions", "wiki_sources", "wiki_lexical"} { + if err := db.Exec("DELETE FROM "+table+" WHERE tenant_id = ?", mysqlTestTenant).Error; err != nil { + t.Fatalf("cleanup %s: %v", table, err) + } + } +} diff --git a/internal/manager/data/knowledge/llm_wiki/store.go b/internal/manager/data/knowledge/llm_wiki/store.go new file mode 100644 index 000000000..b237f7640 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store.go @@ -0,0 +1,55 @@ +// Package llm_wiki wires the LLM Wiki repository and search index onto the +// application database selected by ONGRID_DB_DIALECT (MySQL or SQLite). +package llm_wiki + +import ( + "context" + "errors" + "log/slog" + + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki/index" + "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki/store" + "github.com/ongridio/ongrid/internal/pkg/embedding" + "gorm.io/gorm" +) + +// Store exposes the Wiki repository and search index over the shared +// application database. The pool is owned by the caller and is not closed +// here. +type Store struct { + SearchIndex biz.SearchIndex + + repo *store.Repo +} + +// Open wires the Wiki repository and search index onto db. Schema creation is +// performed by store.Migrate during application startup migrations. The search +// index keeps lexical rows in db and page vectors in the dedicated Qdrant +// collection; when an embedder is configured, vec must be reachable. +func Open(ctx context.Context, db *gorm.DB, vec index.VectorStore, embed embedding.Embedder, dim int, log *slog.Logger) (*Store, error) { + if db == nil { + return nil, errors.New("llmwiki store: database is required") + } + if ctx == nil { + return nil, errors.New("llmwiki store: context is required") + } + if err := ctx.Err(); err != nil { + return nil, err + } + searchIndex, err := index.New(ctx, db, vec, embed, dim, log) + if err != nil { + return nil, err + } + if log != nil { + log.Info("llm wiki store ready", slog.String("dialect", db.Dialector.Name())) + } + return &Store{SearchIndex: searchIndex, repo: store.New(db)}, nil +} + +func (s *Store) Repository() biz.Repository { + if s == nil { + return nil + } + return s.repo +} diff --git a/internal/manager/data/knowledge/llm_wiki/store/build_repo.go b/internal/manager/data/knowledge/llm_wiki/store/build_repo.go new file mode 100644 index 000000000..909bf84c0 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store/build_repo.go @@ -0,0 +1,227 @@ +package store + +import ( + "context" + "errors" + "fmt" + "time" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" + "gorm.io/gorm" +) + +// CreateBuild creates a new staging build for the tenant. +func (r *Repo) CreateBuild(ctx context.Context, tenantID uint64) (*model.WikiBuild, error) { + build := &model.WikiBuild{ + TenantID: tenantID, + Status: model.BuildStaging, + } + if err := r.db.WithContext(ctx).Create(build).Error; err != nil { + return nil, fmt.Errorf("create wiki build: %w", err) + } + return build, nil +} + +// GetBuild retrieves a build by ID. +func (r *Repo) GetBuild(ctx context.Context, tenantID, buildID uint64) (*model.WikiBuild, error) { + var build model.WikiBuild + err := r.db.WithContext(ctx). + Where("tenant_id = ? AND id = ?", tenantID, buildID). + First(&build).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errs.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("get wiki build: %w", err) + } + return &build, nil +} + +// GetActiveBuild returns the currently active build for the tenant. +func (r *Repo) GetActiveBuild(ctx context.Context, tenantID uint64) (*model.WikiBuild, error) { + var build model.WikiBuild + err := r.db.WithContext(ctx). + Where("tenant_id = ? AND status = ?", tenantID, model.BuildActive). + First(&build).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errs.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("get active wiki build: %w", err) + } + return &build, nil +} + +// UpdateBuildStatus updates the build status and optional error message. +func (r *Repo) UpdateBuildStatus(ctx context.Context, tenantID, buildID uint64, status, errorMsg string) error { + updates := map[string]any{ + "status": status, + } + if errorMsg != "" { + updates["error_msg"] = errorMsg + } + if status == model.BuildActive { + now := time.Now().UTC() + updates["activated_at"] = &now + } + + res := r.db.WithContext(ctx). + Model(&model.WikiBuild{}). + Where("tenant_id = ? AND id = ?", tenantID, buildID). + Updates(updates) + if res.Error != nil { + return fmt.Errorf("update wiki build status: %w", res.Error) + } + if res.RowsAffected == 0 { + return errs.ErrNotFound + } + return nil +} + +// UpdateBuildPageCount updates the page count for a build. +func (r *Repo) UpdateBuildPageCount(ctx context.Context, tenantID, buildID uint64, count int) error { + res := r.db.WithContext(ctx). + Model(&model.WikiBuild{}). + Where("tenant_id = ? AND id = ?", tenantID, buildID). + Update("page_count", count) + if res.Error != nil { + return fmt.Errorf("update wiki build page count: %w", res.Error) + } + if res.RowsAffected == 0 { + return errs.ErrNotFound + } + return nil +} + +// ActivateBuild atomically promotes a build to active and demotes any existing active build. +// This is the core of the publish protocol - it ensures only one build is visible at a time. +func (r *Repo) ActivateBuild(ctx context.Context, tenantID, buildID uint64) error { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // Verify the build exists and is in validated state + var build model.WikiBuild + err := tx.Where("tenant_id = ? AND id = ?", tenantID, buildID). + First(&build).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return errs.ErrNotFound + } + if err != nil { + return fmt.Errorf("get build for activation: %w", err) + } + if build.Status != model.BuildValidated { + return fmt.Errorf("cannot activate build in status %q: %w", build.Status, errs.ErrInvalid) + } + + // Demote any existing active build + res := tx.Model(&model.WikiBuild{}). + Where("tenant_id = ? AND status = ?", tenantID, model.BuildActive). + Update("status", "superseded") + if res.Error != nil { + return fmt.Errorf("demote old active build: %w", res.Error) + } + + // Promote the new build + now := time.Now().UTC() + res = tx.Model(&model.WikiBuild{}). + Where("tenant_id = ? AND id = ?", tenantID, buildID). + Updates(map[string]any{ + "status": model.BuildActive, + "activated_at": &now, + }) + if res.Error != nil { + return fmt.Errorf("promote new active build: %w", res.Error) + } + + return nil + }) +} + +// CreatePage creates a page within a staging build. +func (r *Repo) CreatePage(ctx context.Context, page *model.WikiBuildPage) error { + if err := r.db.WithContext(ctx).Create(page).Error; err != nil { + return fmt.Errorf("create wiki page: %w", err) + } + return nil +} + +// CreatePagesBatch creates multiple pages in a single transaction. +func (r *Repo) CreatePagesBatch(ctx context.Context, pages []*model.WikiBuildPage) error { + if len(pages) == 0 { + return nil + } + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // Batch insert in chunks of 100 + batchSize := 100 + for i := 0; i < len(pages); i += batchSize { + end := i + batchSize + if end > len(pages) { + end = len(pages) + } + batch := pages[i:end] + if err := tx.CreateInBatches(batch, len(batch)).Error; err != nil { + return fmt.Errorf("create wiki pages batch: %w", err) + } + } + return nil + }) +} + +// GetPage retrieves a page by build ID and page ID. +func (r *Repo) GetPage(ctx context.Context, tenantID, buildID uint64, pageID string) (*model.WikiBuildPage, error) { + var page model.WikiBuildPage + err := r.db.WithContext(ctx). + Where("tenant_id = ? AND build_id = ? AND page_id = ?", tenantID, buildID, pageID). + First(&page).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errs.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("get wiki page: %w", err) + } + return &page, nil +} + +// ListPagesByBuild returns all pages in a build. +func (r *Repo) ListPagesByBuild(ctx context.Context, tenantID, buildID uint64) ([]*model.WikiBuildPage, error) { + var pages []*model.WikiBuildPage + err := r.db.WithContext(ctx). + Where("tenant_id = ? AND build_id = ?", tenantID, buildID). + Order("page_id"). + Find(&pages).Error + if err != nil { + return nil, fmt.Errorf("list wiki pages: %w", err) + } + return pages, nil +} + +// DeleteBuildPages removes all pages for a build (used for cleanup on failed builds). +func (r *Repo) DeleteBuildPages(ctx context.Context, tenantID, buildID uint64) error { + res := r.db.WithContext(ctx). + Where("tenant_id = ? AND build_id = ?", tenantID, buildID). + Delete(&model.WikiBuildPage{}) + if res.Error != nil { + return fmt.Errorf("delete wiki build pages: %w", res.Error) + } + return nil +} + +// DeleteBuild removes a build and its pages (cascading delete assumed via FK or manual). +func (r *Repo) DeleteBuild(ctx context.Context, tenantID, buildID uint64) error { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // Delete pages first + if err := tx.Where("tenant_id = ? AND build_id = ?", tenantID, buildID). + Delete(&model.WikiBuildPage{}).Error; err != nil { + return fmt.Errorf("delete wiki build pages: %w", err) + } + // Delete build + res := tx.Where("tenant_id = ? AND id = ?", tenantID, buildID). + Delete(&model.WikiBuild{}) + if res.Error != nil { + return fmt.Errorf("delete wiki build: %w", res.Error) + } + if res.RowsAffected == 0 { + return errs.ErrNotFound + } + return nil + }) +} diff --git a/internal/manager/data/knowledge/llm_wiki/store/job_repo.go b/internal/manager/data/knowledge/llm_wiki/store/job_repo.go new file mode 100644 index 000000000..32dd7b153 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store/job_repo.go @@ -0,0 +1,220 @@ +package store + +import ( + "context" + "errors" + "fmt" + "time" + + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" + "gorm.io/gorm" +) + +// CreateJob queues a full-corpus compile job. Only one pending/running job may +// exist for a tenant. +func (r *Repo) CreateJob(ctx context.Context, job *model.CompileJob) error { + active, err := r.HasActiveJob(ctx, job.TenantID) + if err != nil { + return err + } + if active { + return errors.Join(errs.ErrConflict, errors.New("an active compile job already covers these sources")) + } + err = r.db.WithContext(ctx).Create(job).Error + if err != nil { + if isDuplicateKey(err) { + return errors.Join(errs.ErrConflict, errors.New("an active compile job already covers these sources")) + } + if errors.Is(err, errs.ErrConflict) { + return err + } + return fmt.Errorf("create wiki job: %w", err) + } + return nil +} + +// HasActiveJob reports whether the tenant has a queued or running compile job. +// It is the tenant's "a build is in flight" signal, so callers that would change +// the source set can refuse instead of racing the compiler. +func (r *Repo) HasActiveJob(ctx context.Context, tenantID uint64) (bool, error) { + var active int64 + if err := r.db.WithContext(ctx).Model(&model.CompileJob{}).Where("tenant_id = ? AND status IN ? AND deleted_at IS NULL", tenantID, []string{model.JobPending, model.JobRunning}).Count(&active).Error; err != nil { + return false, fmt.Errorf("check active wiki job: %w", err) + } + return active > 0, nil +} + +// ListJobs lists the compile jobs of a tenant, newest first. +func (r *Repo) ListJobs(ctx context.Context, tenantID uint64, limit int) ([]*model.CompileJob, int64, error) { + if limit <= 0 || limit > 200 { + limit = 50 + } + q := r.db.WithContext(ctx).Model(&model.CompileJob{}).Where("tenant_id = ? AND deleted_at IS NULL", tenantID) + var total int64 + if err := q.Count(&total).Error; err != nil { + return nil, 0, fmt.Errorf("count wiki jobs: %w", err) + } + var rows []*model.CompileJob + if err := q.Order("id DESC").Limit(limit).Find(&rows).Error; err != nil { + return nil, 0, fmt.Errorf("list wiki jobs: %w", err) + } + return rows, total, nil +} + +func (r *Repo) GetJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) { + var row model.CompileJob + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ? AND deleted_at IS NULL", tenantID, id).First(&row).Error; err != nil { + return nil, mapNotFound(err) + } + return &row, nil +} + +// RetryJob re-queues a failed or cancelled full-corpus job. +func (r *Repo) RetryJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) { + job, err := r.GetJob(ctx, tenantID, id) + if err != nil { + return nil, err + } + activeKey := model.ActiveJobKey(tenantID) + err = r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + res := tx.Model(&model.CompileJob{}).Where("tenant_id = ? AND id = ? AND status IN ?", tenantID, id, []string{model.JobFailed, model.JobCancelled}).Updates(map[string]any{"status": model.JobPending, "stage": "queued", "cancel_requested": false, "error_message": "", "lease_owner": "", "lease_expires_at": nil, "active_key": activeKey}) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errs.ErrInvalid + } + job.ActiveKey = &activeKey + return nil + }) + if err != nil { + if isDuplicateKey(err) || errors.Is(err, errs.ErrConflict) { + return nil, errors.Join(errs.ErrConflict, errors.New("an active compile job already covers these sources")) + } + return nil, fmt.Errorf("retry wiki job: %w", err) + } + return r.GetJob(ctx, tenantID, id) +} + +// CancelJob marks a pending, running or failed job as cancelled. A running +// worker notices the request at its next cancellation check. +func (r *Repo) CancelJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) { + job, err := r.GetJob(ctx, tenantID, id) + if err != nil { + return nil, err + } + if job.Status != model.JobPending && job.Status != model.JobRunning && job.Status != model.JobFailed { + return nil, errs.ErrInvalid + } + updates := map[string]any{"cancel_requested": true, "status": model.JobCancelled, "active_key": nil, "lease_owner": "", "lease_expires_at": nil} + res := r.db.WithContext(ctx).Model(&model.CompileJob{}).Where("tenant_id = ? AND id = ? AND status = ?", tenantID, id, job.Status).Updates(updates) + err = res.Error + if err == nil && res.RowsAffected == 0 { + err = errs.ErrInvalid + } + if err != nil { + return nil, fmt.Errorf("cancel wiki job: %w", err) + } + return r.GetJob(ctx, tenantID, id) +} + +// ClaimJob leases the oldest claimable job of a tenant: a pending job, or a +// running job whose lease expired. +func (r *Repo) ClaimJob(ctx context.Context, tenantID uint64, owner string, lease time.Duration) (*model.CompileJob, error) { + now := time.Now().UTC() + var candidates []model.CompileJob + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND deleted_at IS NULL AND cancel_requested = ? AND status IN ?", tenantID, false, []string{model.JobPending, model.JobRunning}).Order("id ASC").Limit(32).Find(&candidates).Error; err != nil { + return nil, fmt.Errorf("find wiki job: %w", err) + } + var candidate *model.CompileJob + for i := range candidates { + if candidates[i].Status == model.JobPending || (candidates[i].LeaseExpiresAt != nil && candidates[i].LeaseExpiresAt.Before(now)) { + candidate = &candidates[i] + break + } + } + if candidate == nil { + return nil, biz.ErrNoPendingJob + } + if err := claimJob(ctx, r.db, *candidate, owner, lease); err != nil { + return nil, err + } + return r.GetJob(ctx, tenantID, candidate.ID) +} + +// ClaimJobByID leases one specific job, used when a job is triggered directly. +func (r *Repo) ClaimJobByID(ctx context.Context, tenantID, id uint64, owner string, lease time.Duration) (*model.CompileJob, error) { + now := time.Now().UTC() + var candidate model.CompileJob + query := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ? AND deleted_at IS NULL AND cancel_requested = ?", tenantID, id, false). + Where("(status = ? OR (status = ? AND lease_expires_at IS NOT NULL AND lease_expires_at < ?))", model.JobPending, model.JobRunning, now) + if err := query.First(&candidate).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, biz.ErrNoPendingJob + } + return nil, fmt.Errorf("find wiki job %d: %w", id, err) + } + if err := claimJob(ctx, r.db, candidate, owner, lease); err != nil { + return nil, err + } + return r.GetJob(ctx, tenantID, id) +} + +// claimJob takes the lease on one candidate, guarded by its current status so a +// concurrent claimer cannot take the same job twice. +func claimJob(ctx context.Context, db *gorm.DB, candidate model.CompileJob, owner string, lease time.Duration) error { + claim := db.WithContext(ctx).Model(&model.CompileJob{}). + Where("tenant_id = ? AND id = ? AND cancel_requested = ?", candidate.TenantID, candidate.ID, false) + if candidate.Status == model.JobPending { + claim = claim.Where("status = ?", model.JobPending) + } else { + claim = claim.Where("status = ? AND lease_owner = ? AND lease_expires_at = ?", model.JobRunning, candidate.LeaseOwner, candidate.LeaseExpiresAt) + } + res := claim.Updates(map[string]any{ + "status": model.JobRunning, + "stage": "claimed", + "lease_owner": owner, + "lease_expires_at": time.Now().UTC().Add(lease), + "attempt": gorm.Expr("attempt + 1"), + }) + if res.Error != nil { + return fmt.Errorf("claim wiki job %d: %w", candidate.ID, res.Error) + } + if res.RowsAffected == 0 { + return biz.ErrNoPendingJob + } + return nil +} + +// UpdateJob writes the status of a job and releases its sources when the job is +// no longer running. A cancelled job is never resurrected. +func (r *Repo) UpdateJob(ctx context.Context, tenantID, id uint64, status, stage, errMessage string) error { + updates := map[string]any{"status": status, "stage": stage, "error_message": errMessage} + if status != model.JobRunning { + updates["lease_owner"] = "" + updates["lease_expires_at"] = nil + updates["active_key"] = nil + } + query := r.db.WithContext(ctx).Model(&model.CompileJob{}).Where("tenant_id = ? AND id = ?", tenantID, id) + if status != model.JobCancelled { + query = query.Where("status <> ?", model.JobCancelled) + } + res := query.Updates(updates) + if res.Error != nil { + return fmt.Errorf("update wiki job: %w", res.Error) + } + if res.RowsAffected == 0 { + return errs.ErrNotFound + } + return nil +} + +func (r *Repo) IsCancelRequested(ctx context.Context, tenantID, id uint64) (bool, error) { + var row struct{ CancelRequested bool } + if err := r.db.WithContext(ctx).Model(&model.CompileJob{}).Select("cancel_requested").Where("tenant_id = ? AND id = ?", tenantID, id).First(&row).Error; err != nil { + return false, mapNotFound(err) + } + return row.CancelRequested, nil +} diff --git a/internal/manager/data/knowledge/llm_wiki/store/repo.go b/internal/manager/data/knowledge/llm_wiki/store/repo.go new file mode 100644 index 000000000..a67cab45f --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store/repo.go @@ -0,0 +1,92 @@ +// Package store implements LLM Wiki persistence on the application database +// selected by ONGRID_DB_DIALECT (MySQL or SQLite): sources and versions, +// compile jobs, immutable build pages and search indexes. +package store + +import ( + "errors" + "fmt" + "strings" + + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" + "gorm.io/gorm" +) + +// Repo is the LLM Wiki repository. +type Repo struct{ db *gorm.DB } + +func New(db *gorm.DB) *Repo { return &Repo{db: db} } + +// Migrate creates the current source/build state and the derived search +// tables on MySQL or SQLite. +func Migrate(db *gorm.DB) error { + if db == nil { + return errors.New("migrate llm wiki: database is required") + } + dialect := db.Dialector.Name() + if dialect != "mysql" && dialect != "sqlite" { + return fmt.Errorf("migrate llm wiki: unsupported dialect %s", dialect) + } + if dialect == "sqlite" { + for _, statement := range []string{"PRAGMA foreign_keys = ON", "PRAGMA busy_timeout = 5000", "PRAGMA journal_mode = WAL"} { + if err := db.Exec(statement).Error; err != nil { + return fmt.Errorf("migrate llm wiki: configure sqlite: %w", err) + } + } + } + // The pre-RFC-005 wiki_builds carried corpus_fingerprint/published_at + // columns. Drop that retired leftover before AutoMigrate so the current + // build schema can own the table name. + if db.Migrator().HasTable("wiki_builds") && db.Migrator().HasColumn("wiki_builds", "corpus_fingerprint") { + if err := db.Migrator().DropTable("wiki_builds"); err != nil { + return fmt.Errorf("migrate llm wiki: drop legacy wiki_builds: %w", err) + } + } + // Page vectors moved to Qdrant (ADR-036). MySQL drops the retired table + // through db/migrations; SQLite has no SQL migration runner, so the + // single-instance database is cleaned here. + if dialect == "sqlite" && db.Migrator().HasTable("wiki_vectors") { + if err := db.Migrator().DropTable("wiki_vectors"); err != nil { + return fmt.Errorf("migrate llm wiki: drop legacy wiki_vectors: %w", err) + } + } + // Nothing ever wrote wiki_source_chunks, and the wiki_index_meta schema + // version had no reader, so both were retired. MySQL drops them through + // db/migrations; SQLite has no SQL migration runner, so the single-instance + // database is cleaned here. + if dialect == "sqlite" { + for _, table := range []string{"wiki_source_chunks", "wiki_index_meta"} { + if db.Migrator().HasTable(table) { + if err := db.Migrator().DropTable(table); err != nil { + return fmt.Errorf("migrate llm wiki: drop retired %s: %w", table, err) + } + } + } + } + if err := db.AutoMigrate( + &model.Source{}, &model.SourceVersion{}, &model.CompileJob{}, + &model.WikiBuild{}, &model.WikiBuildPage{}, + &model.WikiLexical{}, + ); err != nil { + return fmt.Errorf("migrate llm wiki: source and build tables: %w", err) + } + return nil +} + +// mapNotFound turns GORM's record-not-found into the shared not-found error. +func mapNotFound(err error) error { + if errors.Is(err, gorm.ErrRecordNotFound) { + return errs.ErrNotFound + } + return err +} + +// isDuplicateKey reports whether err is a unique-constraint violation. +func isDuplicateKey(err error) bool { + message := strings.ToLower(err.Error()) + return strings.Contains(message, "duplicate entry") || strings.Contains(message, "unique constraint failed") +} + +var _ biz.Repository = (*Repo)(nil) diff --git a/internal/manager/data/knowledge/llm_wiki/store/repo_test.go b/internal/manager/data/knowledge/llm_wiki/store/repo_test.go new file mode 100644 index 000000000..dcaba6526 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store/repo_test.go @@ -0,0 +1,285 @@ +package store + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/glebarez/sqlite" + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" + "gorm.io/gorm" +) + +func testRepo(t *testing.T) (*Repo, *gorm.DB) { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + repo := New(db) + if err := Migrate(db); err != nil { + t.Fatal(err) + } + return repo, db +} + +func TestClaimJob_WhenLeaseExpires_RecoversAfterRestart(t *testing.T) { + repo, db := testRepo(t) + activeKey := "tenant-0-full-corpus" + job := &model.CompileJob{TenantID: 0, ActiveKey: &activeKey, Status: model.JobPending, Stage: "queued"} + if err := repo.CreateJob(context.Background(), job); err != nil { + t.Fatal(err) + } + claimed, err := repo.ClaimJob(context.Background(), 0, "worker-a", time.Minute) + if err != nil { + t.Fatal(err) + } + if claimed.LeaseOwner != "worker-a" { + t.Fatalf("owner = %q", claimed.LeaseOwner) + } + if _, err := repo.ClaimJob(context.Background(), 0, "worker-b", time.Minute); !errors.Is(err, biz.ErrNoPendingJob) { + t.Fatalf("active lease claim error = %v", err) + } + past := time.Now().Add(-time.Minute) + if err := db.Model(&model.CompileJob{}).Where("id = ?", job.ID).Update("lease_expires_at", past).Error; err != nil { + t.Fatal(err) + } + recovered, err := repo.ClaimJob(context.Background(), 0, "worker-b", time.Minute) + if err != nil { + t.Fatal(err) + } + if recovered.LeaseOwner != "worker-b" || recovered.Attempt != 2 { + t.Fatalf("recovered = %+v", recovered) + } +} + +func TestCreateJob_RejectsSecondActiveTenantJob(t *testing.T) { + repo, _ := testRepo(t) + firstKey, secondKey := "first", "second" + if err := repo.CreateJob(context.Background(), &model.CompileJob{TenantID: 7, ActiveKey: &firstKey, Status: model.JobPending, Stage: "queued"}); err != nil { + t.Fatal(err) + } + err := repo.CreateJob(context.Background(), &model.CompileJob{TenantID: 7, ActiveKey: &secondKey, Status: model.JobPending, Stage: "queued"}) + if !errors.Is(err, errs.ErrConflict) { + t.Fatalf("second job error = %v", err) + } +} + +func TestCancelJob_AllowsFailedJob(t *testing.T) { + repo, db := testRepo(t) + activeKey := "failed-cancel-key" + job := &model.CompileJob{TenantID: 0, ActiveKey: &activeKey, Status: model.JobFailed, Stage: "compile", ErrorMessage: "compile failed"} + if err := db.Create(job).Error; err != nil { + t.Fatal(err) + } + cancelled, err := repo.CancelJob(context.Background(), 0, job.ID) + if err != nil { + t.Fatal(err) + } + if cancelled.Status != model.JobCancelled || !cancelled.CancelRequested || cancelled.ActiveKey != nil { + t.Fatalf("cancelled job = %+v", cancelled) + } +} + +func TestDeleteSource_RemovesOwnedVersions(t *testing.T) { + repo, db := testRepo(t) + source := &model.Source{TenantID: 0, SourceKey: "source:delete", SourceType: "organization", RawPath: "team/delete.md", Status: model.SourceSucceeded} + if err := db.Create(source).Error; err != nil { + t.Fatal(err) + } + version := &model.SourceVersion{TenantID: 0, SourceID: source.ID, SHA256: "delete-v1", SnapshotPath: "delete-v1.md", SchemaVersion: biz.SchemaVersion} + if err := db.Create(version).Error; err != nil { + t.Fatal(err) + } + if err := repo.DeleteSource(context.Background(), 0, source.ID); err != nil { + t.Fatal(err) + } + for name, target := range map[string]any{"sources": &model.Source{}, "versions": &model.SourceVersion{}} { + var count int64 + if err := db.Model(target).Count(&count).Error; err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("%s count = %d", name, count) + } + } +} + +func TestMigrate_ReplacesLegacyWikiBuildsSchema(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.Exec(`CREATE TABLE wiki_builds (id INTEGER PRIMARY KEY, status TEXT NOT NULL DEFAULT 'building', corpus_fingerprint TEXT NOT NULL DEFAULT '', published_at DATETIME)`).Error; err != nil { + t.Fatal(err) + } + if err := Migrate(db); err != nil { + t.Fatal(err) + } + if !db.Migrator().HasTable("wiki_builds") || !db.Migrator().HasColumn("wiki_builds", "tenant_id") { + t.Fatal("wiki_builds was not recreated with the current schema") + } + if db.Migrator().HasColumn("wiki_builds", "corpus_fingerprint") { + t.Fatal("retired wiki_builds columns still exist") + } +} + +// TestHasActiveJob_TracksPendingAndRunningJobs — callers that would change the +// source set use this signal to refuse while a build is in flight. +func TestHasActiveJob_TracksPendingAndRunningJobs(t *testing.T) { + repo, _ := testRepo(t) + ctx := context.Background() + + active, err := repo.HasActiveJob(ctx, 0) + if err != nil { + t.Fatal(err) + } + if active { + t.Fatal("tenant without jobs reported an active job") + } + + activeKey := "tenant-0-full-corpus" + job := &model.CompileJob{TenantID: 0, ActiveKey: &activeKey, Status: model.JobPending, Stage: "queued"} + if err := repo.CreateJob(ctx, job); err != nil { + t.Fatal(err) + } + if active, err = repo.HasActiveJob(ctx, 0); err != nil || !active { + t.Fatalf("pending job: active = %t, err = %v", active, err) + } + if active, err = repo.HasActiveJob(ctx, 1); err != nil || active { + t.Fatalf("other tenant: active = %t, err = %v", active, err) + } + + if _, err := repo.CancelJob(ctx, 0, job.ID); err != nil { + t.Fatal(err) + } + if active, err = repo.HasActiveJob(ctx, 0); err != nil || active { + t.Fatalf("cancelled job: active = %t, err = %v", active, err) + } +} + +// TestCreateJob_RejectsSecondJobWhenTheTenantKeyIsHeld — "one in-flight job per +// tenant" is enforced by the tenant-scoped active_key and uk_wiki_job_active, not +// by the pre-check: the pre-check only looks at pending/running rows, so two +// concurrent creates can both pass it, and only the unique index stops the +// second insert. +func TestCreateJob_RejectsSecondJobWhenTheTenantKeyIsHeld(t *testing.T) { + repo, db := testRepo(t) + ctx := context.Background() + + if model.ActiveJobKey(0) == model.ActiveJobKey(1) { + t.Fatal("the active job key must be scoped to the tenant") + } + + key := model.ActiveJobKey(0) + holder := &model.CompileJob{TenantID: 0, ActiveKey: &key, Status: model.JobSucceeded, Stage: "completed"} + if err := db.Create(holder).Error; err != nil { + t.Fatal(err) + } + + err := repo.CreateJob(ctx, &model.CompileJob{TenantID: 0, ActiveKey: &key, Status: model.JobPending, Stage: "queued"}) + + if !errors.Is(err, errs.ErrConflict) { + t.Fatalf("CreateJob error = %v, want conflict", err) + } +} + +// TestListSourcesAfter_WalksEverySource — ListSources answers with a single capped +// page, so callers that must see every source (sync, delete, the compile corpus) +// have to page with ListSourcesAfter or they silently lose the tail. +func TestListSourcesAfter_WalksEverySource(t *testing.T) { + repo, db := testRepo(t) + ctx := context.Background() + const total = 205 + for i := 1; i <= total; i++ { + source := &model.Source{ + TenantID: 0, + SourceKey: fmt.Sprintf("key-%03d", i), + SourceType: "organization", + RawPath: fmt.Sprintf("docs/%03d.md", i), + Status: model.SourcePending, + } + if err := db.Create(source).Error; err != nil { + t.Fatal(err) + } + } + + page, count, err := repo.ListSources(ctx, 0, "", 10000) + if err != nil { + t.Fatal(err) + } + if len(page) != 200 || count != total { + t.Fatalf("single page = %d rows of a reported %d, want 200 of %d", len(page), count, total) + } + + seen := 0 + var afterID uint64 + for { + rows, err := repo.ListSourcesAfter(ctx, 0, "", afterID, 200) + if err != nil { + t.Fatal(err) + } + if len(rows) == 0 { + break + } + if rows[0].ID <= afterID { + t.Fatalf("page did not advance past id %d", afterID) + } + afterID = rows[len(rows)-1].ID + seen += len(rows) + } + if seen != total { + t.Fatalf("paged walk saw %d sources, want %d", seen, total) + } +} + +// TestUpsertSourceVersion_ReattachesWhenTheCurrentVersionRowIsGone — a source +// whose current_version_id points at a row that no longer exists made the next +// sync fail, because identical content then looked already stored. It must +// attach a fresh version instead. +func TestUpsertSourceVersion_ReattachesWhenTheCurrentVersionRowIsGone(t *testing.T) { + repo, db := testRepo(t) + ctx := context.Background() + source := &model.Source{TenantID: 0, SourceKey: "organization:7", SourceType: "organization", RawPath: "PLAN.md"} + newVersion := func() *model.SourceVersion { + return &model.SourceVersion{SHA256: "abc123", SizeBytes: 3, SnapshotPath: "versions/x/abc123.md"} + } + + stored, _, _, err := repo.UpsertSourceVersion(ctx, source, newVersion()) + if err != nil { + t.Fatal(err) + } + if stored.CurrentVersionID == nil { + t.Fatal("first mirror did not record a current version") + } + + // The version row disappears while the source keeps pointing at it. + if err := db.Exec("DELETE FROM wiki_source_versions WHERE id = ?", *stored.CurrentVersionID).Error; err != nil { + t.Fatal(err) + } + + again, version, changed, err := repo.UpsertSourceVersion(ctx, source, newVersion()) + if err != nil { + t.Fatalf("re-mirror with a dangling current version: %v", err) + } + if !changed { + t.Fatal("attaching a lost version must report a change") + } + if version == nil || version.ID == 0 { + t.Fatalf("version = %+v", version) + } + if again.CurrentVersionID == nil || *again.CurrentVersionID != version.ID { + t.Fatalf("current_version_id = %v, want %d", again.CurrentVersionID, version.ID) + } + var rows int64 + if err := db.Model(&model.SourceVersion{}).Where("id = ?", version.ID).Count(&rows).Error; err != nil { + t.Fatal(err) + } + if rows != 1 { + t.Fatalf("version rows = %d, want 1", rows) + } +} diff --git a/internal/manager/data/knowledge/llm_wiki/store/source_repo.go b/internal/manager/data/knowledge/llm_wiki/store/source_repo.go new file mode 100644 index 000000000..57e63656c --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store/source_repo.go @@ -0,0 +1,173 @@ +package store + +import ( + "context" + "errors" + "fmt" + + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" + "gorm.io/gorm" +) + +// UpsertSourceVersion stores a mirrored source and its new version. changed +// reports whether the content differed from the stored version, in which case +// the source is marked stale until it is compiled again. +func (r *Repo) UpsertSourceVersion(ctx context.Context, source *model.Source, version *model.SourceVersion) (*model.Source, *model.SourceVersion, bool, error) { + var resultSource model.Source + var resultVersion model.SourceVersion + changed := false + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + err := tx.Where("tenant_id = ? AND source_key = ? AND deleted_at IS NULL", source.TenantID, source.SourceKey).First(&resultSource).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + resultSource = *source + if err := tx.Create(&resultSource).Error; err != nil { + return fmt.Errorf("create wiki source: %w", err) + } + } else if err != nil { + return fmt.Errorf("get wiki source: %w", err) + } + if resultSource.ContentSHA256 == version.SHA256 && resultSource.CurrentVersionID != nil { + err := tx.Where("tenant_id = ? AND id = ? AND deleted_at IS NULL", source.TenantID, *resultSource.CurrentVersionID).First(&resultVersion).Error + if err == nil { + if resultSource.RawPath != source.RawPath || resultSource.SourceType != source.SourceType { + updates := map[string]any{"source_type": source.SourceType, "raw_path": source.RawPath} + if err := tx.Model(&model.Source{}).Where("tenant_id = ? AND id = ?", source.TenantID, resultSource.ID).Updates(updates).Error; err != nil { + return fmt.Errorf("update wiki source metadata: %w", err) + } + resultSource.SourceType = source.SourceType + resultSource.RawPath = source.RawPath + } + return nil + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return fmt.Errorf("get current wiki version: %w", err) + } + // The row current_version_id points at is gone, so identical content + // only looks stored. Fall through and attach the version again + // instead of failing the sync on a dangling pointer. + } + changed = true + version.TenantID = source.TenantID + version.SourceID = resultSource.ID + if err := tx.Where("tenant_id = ? AND source_id = ? AND sha256 = ? AND deleted_at IS NULL", source.TenantID, resultSource.ID, version.SHA256).First(&resultVersion).Error; errors.Is(err, gorm.ErrRecordNotFound) { + resultVersion = *version + if err := tx.Create(&resultVersion).Error; err != nil { + return fmt.Errorf("create wiki version: %w", err) + } + } else if err != nil { + return fmt.Errorf("get wiki version: %w", err) + } + status := model.SourcePending + if resultSource.CurrentVersionID != nil { + status = model.SourceStale + } + updates := map[string]any{"source_type": source.SourceType, "raw_path": source.RawPath, "current_version_id": resultVersion.ID, "content_sha256": version.SHA256, "status": status} + if err := tx.Model(&model.Source{}).Where("tenant_id = ? AND id = ?", source.TenantID, resultSource.ID).Updates(updates).Error; err != nil { + return fmt.Errorf("update wiki source: %w", err) + } + resultSource.SourceType = source.SourceType + resultSource.RawPath = source.RawPath + resultSource.CurrentVersionID = &resultVersion.ID + resultSource.ContentSHA256 = version.SHA256 + resultSource.Status = status + return nil + }) + return &resultSource, &resultVersion, changed, err +} + +// ListSources lists the sources of a tenant, newest first, optionally filtered +// by status. +func (r *Repo) ListSources(ctx context.Context, tenantID uint64, status string, limit int) ([]*model.Source, int64, error) { + limit = clampSourceLimit(limit) + q := r.db.WithContext(ctx).Model(&model.Source{}).Where("tenant_id = ? AND deleted_at IS NULL", tenantID) + if status != "" { + q = q.Where("status = ?", status) + } + var total int64 + if err := q.Count(&total).Error; err != nil { + return nil, 0, fmt.Errorf("count wiki sources: %w", err) + } + var rows []*model.Source + if err := q.Order("updated_at DESC").Limit(limit).Find(&rows).Error; err != nil { + return nil, 0, fmt.Errorf("list wiki sources: %w", err) + } + return rows, total, nil +} + +// clampSourceLimit keeps one source page inside the size the store answers in a +// single query. +func clampSourceLimit(limit int) int { + if limit <= 0 || limit > 500 { + return 200 + } + return limit +} + +// ListSourcesAfter returns at most limit sources with an id greater than afterID +// in ascending id order, so a caller that needs every source of a tenant can +// walk stable pages. ListSources answers with one page, which silently drops the +// rest for a tenant that has more sources than a page holds. +func (r *Repo) ListSourcesAfter(ctx context.Context, tenantID uint64, status string, afterID uint64, limit int) ([]*model.Source, error) { + q := r.db.WithContext(ctx).Model(&model.Source{}).Where("tenant_id = ? AND deleted_at IS NULL", tenantID) + if status != "" { + q = q.Where("status = ?", status) + } + if afterID > 0 { + q = q.Where("id > ?", afterID) + } + var rows []*model.Source + if err := q.Order("id ASC").Limit(clampSourceLimit(limit)).Find(&rows).Error; err != nil { + return nil, fmt.Errorf("page wiki sources: %w", err) + } + return rows, nil +} + +func (r *Repo) GetSource(ctx context.Context, tenantID, id uint64) (*model.Source, error) { + var row model.Source + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ? AND deleted_at IS NULL", tenantID, id).First(&row).Error; err != nil { + return nil, mapNotFound(err) + } + return &row, nil +} + +// DeleteSource deletes a source and its durable versions/chunks. The business +// layer invalidates the active immutable build after this transaction commits. +func (r *Repo) DeleteSource(ctx context.Context, tenantID, id uint64) error { + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var source model.Source + if err := tx.Where("tenant_id = ? AND id = ? AND deleted_at IS NULL", tenantID, id).First(&source).Error; err != nil { + return mapNotFound(err) + } + if err := tx.Where("tenant_id = ? AND source_id = ?", tenantID, id).Delete(&model.SourceVersion{}).Error; err != nil { + return fmt.Errorf("delete wiki source versions: %w", err) + } + if err := tx.Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&model.Source{}).Error; err != nil { + return fmt.Errorf("delete wiki source: %w", err) + } + return nil + }) + if err != nil { + return fmt.Errorf("delete wiki source %d: %w", id, err) + } + return nil +} + +func (r *Repo) GetVersion(ctx context.Context, tenantID, id uint64) (*model.SourceVersion, error) { + var row model.SourceVersion + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ? AND deleted_at IS NULL", tenantID, id).First(&row).Error; err != nil { + return nil, mapNotFound(err) + } + return &row, nil +} + +func (r *Repo) MarkSourceStatus(ctx context.Context, tenantID, id uint64, status string) error { + res := r.db.WithContext(ctx).Model(&model.Source{}).Where("tenant_id = ? AND id = ?", tenantID, id).Update("status", status) + if res.Error != nil { + return fmt.Errorf("mark wiki source: %w", res.Error) + } + if res.RowsAffected == 0 { + return errs.ErrNotFound + } + return nil +} diff --git a/internal/manager/data/knowledge/llm_wiki/store/tree_integration_test.go b/internal/manager/data/knowledge/llm_wiki/store/tree_integration_test.go new file mode 100644 index 000000000..27a9c826a --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store/tree_integration_test.go @@ -0,0 +1,162 @@ +package store + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" +) + +func TestTreeReadsPublishedBuildPages(t *testing.T) { + repo, _ := testRepo(t) + ctx := context.Background() + root := t.TempDir() + files, err := biz.NewFileStore(root) + if err != nil { + t.Fatal(err) + } + if err := files.Ensure(ctx); err != nil { + t.Fatal(err) + } + build, err := repo.CreateBuild(ctx, biz.DefaultTenantID) + if err != nil { + t.Fatal(err) + } + bodyPath := filepath.ToSlash(filepath.Join("builds", fmt.Sprintf("%d", build.ID), "pages", "generated.md")) + absPath := filepath.Join(root, "wiki", filepath.FromSlash(bodyPath)) + if err := os.MkdirAll(filepath.Dir(absPath), 0o750); err != nil { + t.Fatal(err) + } + generatedBody := "# Generated Wiki\n" + if err := os.WriteFile(absPath, []byte(generatedBody), 0o640); err != nil { + t.Fatal(err) + } + page := &model.WikiBuildPage{BuildID: build.ID, TenantID: biz.DefaultTenantID, PageID: "generated", PageType: "generated", Title: "Generated Wiki", BodyPath: bodyPath, BodySHA256: "hash", SourceRefsJSON: "[]"} + if err := repo.CreatePage(ctx, page); err != nil { + t.Fatal(err) + } + if err := repo.UpdateBuildStatus(ctx, biz.DefaultTenantID, build.ID, model.BuildValidated, ""); err != nil { + t.Fatal(err) + } + if err := repo.ActivateBuild(ctx, biz.DefaultTenantID, build.ID); err != nil { + t.Fatal(err) + } + uc, err := biz.NewWithUsageRecorder(ctx, repo, files, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + nodes, err := uc.ListTree(ctx, "wiki", "") + if err != nil { + t.Fatal(err) + } + for _, node := range nodes { + if node.PageID != "generated" { + continue + } + if node.RelativePath != "generated.md" || node.Name != "Generated Wiki" { + t.Fatalf("node = %+v", node) + } + detail, err := uc.GetNode(ctx, node.ID) + if err != nil { + t.Fatal(err) + } + if detail.Content != generatedBody { + t.Fatalf("content = %q", detail.Content) + } + return + } + t.Fatalf("generated page missing from tree: %+v", nodes) +} + +func TestTreePlacesGeneratedPageUnderPrimarySourceDirectory(t *testing.T) { + repo, _ := testRepo(t) + ctx := context.Background() + root := t.TempDir() + files, err := biz.NewFileStore(root) + if err != nil { + t.Fatal(err) + } + if err := files.Ensure(ctx); err != nil { + t.Fatal(err) + } + build, err := repo.CreateBuild(ctx, biz.DefaultTenantID) + if err != nil { + t.Fatal(err) + } + bodyPath := filepath.ToSlash(filepath.Join("builds", fmt.Sprintf("%d", build.ID), "pages", "dns-diagnosis.md")) + absPath := filepath.Join(root, "wiki", filepath.FromSlash(bodyPath)) + if err := os.MkdirAll(filepath.Dir(absPath), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(absPath, []byte("# DNS diagnosis\n"), 0o640); err != nil { + t.Fatal(err) + } + page := &model.WikiBuildPage{ + BuildID: build.ID, + TenantID: biz.DefaultTenantID, + PageID: "dns-diagnosis", + PageType: "generated", + Title: "DNS diagnosis", + BodyPath: bodyPath, + BodySHA256: "hash", + SourceRefsJSON: `[{"source_id":7,"source_version_id":8,"chunk_ordinal":0,"content_hash":"source-hash","source_path":"network/dns/troubleshooting.md"}]`, + } + if err := repo.CreatePage(ctx, page); err != nil { + t.Fatal(err) + } + if err := repo.UpdateBuildStatus(ctx, biz.DefaultTenantID, build.ID, model.BuildValidated, ""); err != nil { + t.Fatal(err) + } + if err := repo.ActivateBuild(ctx, biz.DefaultTenantID, build.ID); err != nil { + t.Fatal(err) + } + uc, err := biz.NewWithUsageRecorder(ctx, repo, files, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + + rootNodes, err := uc.ListTree(ctx, "wiki", "") + if err != nil { + t.Fatal(err) + } + var network *biz.TreeNode + for i := range rootNodes { + if rootNodes[i].Kind == "folder" && rootNodes[i].RelativePath == "network" { + network = &rootNodes[i] + break + } + } + if network == nil { + t.Fatalf("network folder missing from tree: %+v", rootNodes) + } + + networkNodes, err := uc.ListTree(ctx, "wiki", network.ID) + if err != nil { + t.Fatal(err) + } + var dns *biz.TreeNode + for i := range networkNodes { + if networkNodes[i].Kind == "folder" && networkNodes[i].RelativePath == "network/dns" { + dns = &networkNodes[i] + break + } + } + if dns == nil { + t.Fatalf("dns folder missing from tree: %+v", networkNodes) + } + + dnsNodes, err := uc.ListTree(ctx, "wiki", dns.ID) + if err != nil { + t.Fatal(err) + } + for _, node := range dnsNodes { + if node.PageID == "dns-diagnosis" && node.RelativePath == "network/dns/dns-diagnosis.md" { + return + } + } + t.Fatalf("generated page missing from source directory: %+v", dnsNodes) +} diff --git a/internal/manager/data/knowledge/llm_wiki/store/usage_integration_test.go b/internal/manager/data/knowledge/llm_wiki/store/usage_integration_test.go new file mode 100644 index 000000000..dd96a80c9 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store/usage_integration_test.go @@ -0,0 +1,98 @@ +package store + +import ( + "context" + "io" + "log/slog" + "strings" + "testing" + "time" + + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type compileUsageLLM struct { + usage llm.Usage +} + +func (f *compileUsageLLM) Complete(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return &llm.ChatResp{ + Assistant: llm.Message{ + Role: "assistant", + Content: `{"pages":[{"page_id":"overview","title":"Overview","sections":[{"heading":"Overview","content":"line 1\nline 2\nline 3\nline 4","source_ids":[0]}]}]}`, + }, + Usage: f.usage, + }, nil +} + +func (f *compileUsageLLM) ModelVersion() string { + return "fake-wiki-model" +} + +type compileUsageRecorder struct { + usages []llm.Usage + closed bool +} + +func (r *compileUsageRecorder) Start(context.Context, uint64) (biz.TokenUsageSink, error) { + return &compileUsageSink{recorder: r}, nil +} + +type compileUsageSink struct { + recorder *compileUsageRecorder +} + +func (s *compileUsageSink) Record(_ context.Context, usage llm.Usage) error { + s.recorder.usages = append(s.recorder.usages, usage) + return nil +} + +func (s *compileUsageSink) Close(context.Context) error { + s.recorder.closed = true + return nil +} + +func TestCompileJob_RecordsSuccessfulLLMUsage(t *testing.T) { + repo, _ := testRepo(t) + + ctx := context.Background() + files, err := biz.NewFileStore(t.TempDir()) + require.NoError(t, err) + require.NoError(t, files.Ensure(ctx)) + + usageRecorder := &compileUsageRecorder{} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + uc, err := biz.NewWithUsageRecorder( + ctx, + repo, + files, + &compileUsageLLM{usage: llm.Usage{PromptTokens: 120, CompletionTokens: 30, TotalTokens: 150}}, + nil, + log, + usageRecorder, + ) + require.NoError(t, err) + + _, err = uc.SyncOrganizationSources(ctx, []biz.OrganizationSource{{ + ID: 1, + Title: "Operations", + Path: "ops", + Content: strings.Repeat("Operational knowledge for the fake Wiki compile. ", 8), + }}) + require.NoError(t, err) + + job, err := uc.CreateCompileJob(ctx, biz.DefaultTenantID, false, nil) + require.NoError(t, err) + require.NoError(t, uc.RunOnce(ctx, biz.DefaultTenantID, "usage-integration-test", time.Minute)) + + storedJob, err := repo.GetJob(ctx, biz.DefaultTenantID, job.ID) + require.NoError(t, err) + assert.Equal(t, model.JobSucceeded, storedJob.Status) + + assert.Equal(t, []llm.Usage{{PromptTokens: 120, CompletionTokens: 30, TotalTokens: 150}}, usageRecorder.usages) + assert.True(t, usageRecorder.closed) +} diff --git a/internal/manager/data/knowledge/llm_wiki/store_test.go b/internal/manager/data/knowledge/llm_wiki/store_test.go new file mode 100644 index 000000000..43130d187 --- /dev/null +++ b/internal/manager/data/knowledge/llm_wiki/store_test.go @@ -0,0 +1,43 @@ +package llm_wiki + +import ( + "context" + "testing" + + "github.com/glebarez/sqlite" + store "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki/store" + "gorm.io/gorm" +) + +func TestOpen_ProvidesRepositoryAndSearchIndexOnSharedDatabase(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := store.Migrate(db); err != nil { + t.Fatal(err) + } + opened, err := Open(context.Background(), db, nil, nil, 0, nil) + if err != nil { + t.Fatal(err) + } + if opened.Repository() == nil { + t.Fatal("repository is nil") + } + if opened.SearchIndex == nil { + t.Fatal("search index is nil") + } + _, total, err := opened.Repository().ListSources(context.Background(), 1, "", 10) + if err != nil { + t.Fatal(err) + } + if total != 0 { + t.Fatalf("source total = %d", total) + } +} + +func TestOpen_RejectsMissingDatabase(t *testing.T) { + if _, err := Open(context.Background(), nil, nil, nil, 0, nil); err == nil { + t.Fatal("nil database was accepted") + } +} diff --git a/internal/manager/model/knowledge/llm_wiki/build.go b/internal/manager/model/knowledge/llm_wiki/build.go new file mode 100644 index 000000000..1e2804b70 --- /dev/null +++ b/internal/manager/model/knowledge/llm_wiki/build.go @@ -0,0 +1,52 @@ +package llm_wiki + +import "time" + +// Build status constants for the staged build lifecycle. +const ( + BuildStaging = "staging" // pages being written, not yet validated + BuildValidated = "validated" // all pages present and hash-verified + BuildActive = "active" // the current published build + BuildFailed = "failed" // validation or activation failed +) + +// WikiBuild represents an isolated snapshot of the entire Wiki corpus. +// Builds are created in staging state, validated, then atomically promoted +// to active. Only one build can be active at a time per tenant. +type WikiBuild struct { + ID uint64 `gorm:"column:id;primaryKey;autoIncrement"` + TenantID uint64 `gorm:"column:tenant_id;not null;default:0;index:idx_wiki_build_tenant_status,priority:1"` + Status string `gorm:"column:status;size:24;not null;default:'staging';index:idx_wiki_build_tenant_status,priority:2"` + PageCount int `gorm:"column:page_count;not null;default:0"` + ErrorMsg string `gorm:"column:error_msg;type:text;not null"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` + ActivatedAt *time.Time `gorm:"column:activated_at"` +} + +func (WikiBuild) TableName() string { return "wiki_builds" } + +// WikiBuildPage is a single page within a build. Pages are keyed by +// (build_id, page_id), so multiple immutable builds can coexist. +type WikiBuildPage struct { + ID uint64 `gorm:"column:id;primaryKey;autoIncrement"` + BuildID uint64 `gorm:"column:build_id;not null;uniqueIndex:uk_wiki_build_page,priority:1"` + TenantID uint64 `gorm:"column:tenant_id;not null;default:0;index:idx_wiki_build_page_tenant"` + PageID string `gorm:"column:page_id;size:64;not null;default:'';uniqueIndex:uk_wiki_build_page,priority:2"` + PageType string `gorm:"column:page_type;size:24;not null;default:'generated'"` + Title string `gorm:"column:title;size:512;not null;default:''"` + BodyPath string `gorm:"column:body_path;size:1024;not null;default:''"` + BodySHA256 string `gorm:"column:body_sha256;size:64;not null;default:''"` + SourceRefsJSON string `gorm:"column:source_refs_json;type:json;not null"` // JSON array of source references + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` +} + +func (WikiBuildPage) TableName() string { return "wiki_build_pages" } + +// SourceRef records the provenance of a page back to the raw knowledge chunk. +type SourceRef struct { + SourceID uint64 `json:"source_id"` + SourceVersionID uint64 `json:"source_version_id"` + ChunkOrdinal int `json:"chunk_ordinal"` + ContentHash string `json:"content_hash"` + SourcePath string `json:"source_path,omitempty"` +} diff --git a/internal/manager/model/knowledge/llm_wiki/index.go b/internal/manager/model/knowledge/llm_wiki/index.go new file mode 100644 index 000000000..2c3de6ff0 --- /dev/null +++ b/internal/manager/model/knowledge/llm_wiki/index.go @@ -0,0 +1,16 @@ +package llm_wiki + +// WikiLexical stores the portable lexical search row for one page. It is a +// regular table with LIKE-based substring matching so SQLite and MySQL share +// the same query semantics. Rebuildable from wiki_build_pages and artifacts. +type WikiLexical struct { + PageKey string `gorm:"column:page_key;size:96;primaryKey"` + TenantID uint64 `gorm:"column:tenant_id;not null;index:idx_wiki_lexical_tenant"` + PageID string `gorm:"column:page_id;size:64;not null"` + PageType string `gorm:"column:page_type;size:24;not null;default:''"` + Title string `gorm:"column:title;size:512;not null;default:''"` + Aliases string `gorm:"column:aliases;size:512;not null;default:''"` + Content string `gorm:"column:content;type:longtext;not null"` +} + +func (WikiLexical) TableName() string { return "wiki_lexical" } diff --git a/internal/manager/model/knowledge/llm_wiki/model.go b/internal/manager/model/knowledge/llm_wiki/model.go new file mode 100644 index 000000000..1fa669c98 --- /dev/null +++ b/internal/manager/model/knowledge/llm_wiki/model.go @@ -0,0 +1,84 @@ +// Package llm_wiki contains persistence entities for the LLM Wiki bounded context. +package llm_wiki + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "time" +) + +const ( + SourcePending = "pending" + SourceSucceeded = "succeeded" + SourceStale = "stale" + SourceFailed = "failed" + + JobPending = "pending" + JobRunning = "running" + JobSucceeded = "succeeded" + JobSkipped = "skipped" + JobFailed = "failed" + JobCancelled = "cancelled" +) + +// ActiveJobKey is the active_key an in-flight compile job holds, and the value +// `uk_wiki_job_active` makes unique. It is scoped to the tenant rather than to a +// source set so the database itself enforces at most one in-flight job per +// tenant: two concurrent creates that both read "no active job" cannot both +// insert. A job releases the key when it leaves `running`. +func ActiveJobKey(tenantID uint64) string { + sum := sha256.Sum256(fmt.Appendf(nil, "tenant:%d:active-compile", tenantID)) + return hex.EncodeToString(sum[:]) +} + +type Source struct { + ID uint64 `gorm:"column:id;primaryKey;autoIncrement"` + TenantID uint64 `gorm:"column:tenant_id;not null;default:0;uniqueIndex:uk_wiki_source,priority:1;index:idx_wiki_source_status,priority:1"` + SourceKey string `gorm:"column:source_key;size:512;not null;default:'';uniqueIndex:uk_wiki_source,priority:2"` + SourceType string `gorm:"column:source_type;size:32;not null;default:''"` + RawPath string `gorm:"column:raw_path;size:1024;not null;default:''"` + CurrentVersionID *uint64 `gorm:"column:current_version_id"` + ContentSHA256 string `gorm:"column:content_sha256;size:64;not null;default:''"` + Status string `gorm:"column:status;size:24;not null;default:'pending';index:idx_wiki_source_status,priority:2"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime"` + DeletedAt *time.Time `gorm:"column:deleted_at;index:idx_wiki_source_deleted"` +} + +func (Source) TableName() string { return "wiki_sources" } + +type SourceVersion struct { + ID uint64 `gorm:"column:id;primaryKey;autoIncrement"` + TenantID uint64 `gorm:"column:tenant_id;not null;default:0;uniqueIndex:uk_wiki_source_version,priority:1;index:idx_wiki_version_source,priority:1"` + SourceID uint64 `gorm:"column:source_id;not null;uniqueIndex:uk_wiki_source_version,priority:2;index:idx_wiki_version_source,priority:2"` + SHA256 string `gorm:"column:sha256;size:64;not null;default:'';uniqueIndex:uk_wiki_source_version,priority:3"` + SizeBytes uint64 `gorm:"column:size_bytes;not null;default:0"` + SnapshotPath string `gorm:"column:snapshot_path;size:1024;not null;default:''"` + SchemaVersion string `gorm:"column:schema_version;size:32;not null;default:'v1'"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime"` + DeletedAt *time.Time `gorm:"column:deleted_at;index:idx_wiki_version_deleted"` +} + +func (SourceVersion) TableName() string { return "wiki_source_versions" } + +type CompileJob struct { + ID uint64 `gorm:"column:id;primaryKey;autoIncrement"` + TenantID uint64 `gorm:"column:tenant_id;not null;default:0;index:idx_wiki_job_claim,priority:1"` + ActiveKey *string `gorm:"column:active_key;size:64;uniqueIndex:uk_wiki_job_active"` + Status string `gorm:"column:status;size:24;not null;default:'pending';index:idx_wiki_job_claim,priority:2"` + Stage string `gorm:"column:stage;size:32;not null;default:'queued'"` + ForceCompile bool `gorm:"column:force_compile;not null;default:false"` + SourceIDs *string `gorm:"column:source_ids;size:2048;not null;default:''"` + LeaseOwner string `gorm:"column:lease_owner;size:128;not null;default:''"` + LeaseExpiresAt *time.Time `gorm:"column:lease_expires_at;index:idx_wiki_job_claim,priority:3"` + Attempt uint32 `gorm:"column:attempt;not null;default:0"` + CancelRequested bool `gorm:"column:cancel_requested;not null;default:false"` + ErrorMessage string `gorm:"column:error_message;size:2048;not null;default:''"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime"` + DeletedAt *time.Time `gorm:"column:deleted_at;index:idx_wiki_job_deleted"` +} + +func (CompileJob) TableName() string { return "wiki_compile_jobs" } diff --git a/internal/manager/model/knowledge/model.go b/internal/manager/model/knowledge/model.go index da877b936..aee96cf02 100644 --- a/internal/manager/model/knowledge/model.go +++ b/internal/manager/model/knowledge/model.go @@ -75,20 +75,20 @@ func (Repository) TableName() string { return "knowledge_repos" } // per identity the simplest model. private_key + passphrase are // AES-encrypted at the data layer before insertion; never logged. type SSHIdentity struct { - ID uint64 `gorm:"primaryKey;autoIncrement"` - Name string `gorm:"size:128;not null;uniqueIndex:uk_ssh_name"` - PrivateKey string `gorm:"type:text;not null;column:private_key"` - PublicKey string `gorm:"type:text;not null;column:public_key"` - Fingerprint string `gorm:"size:128;not null"` // SHA256:xxx derived from PublicKey - Passphrase string `gorm:"type:text;column:passphrase"` // nullable; MVP rejects non-empty - HostsJSON string `gorm:"type:text;not null;column:hosts"` // JSON array of host glob patterns + ID uint64 `gorm:"primaryKey;autoIncrement"` + Name string `gorm:"size:128;not null;uniqueIndex:uk_ssh_name"` + PrivateKey string `gorm:"type:text;not null;column:private_key"` + PublicKey string `gorm:"type:text;not null;column:public_key"` + Fingerprint string `gorm:"size:128;not null"` // SHA256:xxx derived from PublicKey + Passphrase string `gorm:"type:text;column:passphrase"` // nullable; MVP rejects non-empty + HostsJSON string `gorm:"type:text;not null;column:hosts"` // JSON array of host glob patterns // MySQL TEXT columns cannot carry a DEFAULT clause (Error 1101) — // so this is NOT NULL but no DB-level default; biz layer always // supplies at least the empty string on insert. - KnownHosts string `gorm:"type:text;not null;column:known_hosts"` - LastUsedAt *time.Time `gorm:"column:last_used_at"` - CreatedAt time.Time - UpdatedAt time.Time + KnownHosts string `gorm:"type:text;not null;column:known_hosts"` + LastUsedAt *time.Time `gorm:"column:last_used_at"` + CreatedAt time.Time + UpdatedAt time.Time } // TableName pins the table name. @@ -121,18 +121,18 @@ type Doc struct { // the original was — Chinese blog, English RFC, etc.). It's also // the natural-key input to manualDocID, so changing it changes // the doc id. - Title string + Title string // TitleEN is an optional English overlay shown when the operator's // locale is en-US. Empty = no override; the UI falls back to // Title (original). Lets a Chinese-language vault stay readable // for non-Chinese operators without lossy auto-translation. Stored // alongside Title in the qdrant payload as `title_en`. - TitleEN string - Content string - Path string // "/"-separated breadcrumb; empty = root - Tags []string // free-form labels; nil = none - CreatedAt time.Time - UpdatedAt time.Time + TitleEN string + Content string + Path string // "/"-separated breadcrumb; empty = root + Tags []string // free-form labels; nil = none + CreatedAt time.Time + UpdatedAt time.Time } // ListDocsFilter narrows /knowledge/docs and the biz Search. SourceType @@ -147,6 +147,10 @@ type ListDocsFilter struct { PathPrefix string Tag string Limit int + // All walks every Qdrant scroll page before deduplicating logical + // documents. It is reserved for internal snapshot consumers such as LLM + // Wiki sync; interactive list endpoints should keep their bounded Limit. + All bool } // SearchOptions narrows the vector search. Path / PathPrefix / Tags @@ -155,6 +159,7 @@ type ListDocsFilter struct { // goes way up when the caller already knows the domain (e.g. LLM // with `path_prefix=网络/`). type SearchOptions struct { + Mode string Path string PathPrefix string Tags []string // any-match (filter passes if doc has any one) @@ -163,6 +168,11 @@ type SearchOptions struct { // SearchHit is the shared search result shape (Doc + cosine score). type SearchHit struct { - Doc *Doc - Score float64 + Doc *Doc + Score float64 + Layer string + PageType string + PageID string + SourceVersionID string + MatchedNode string } diff --git a/internal/manager/server/knowledge/guards_test.go b/internal/manager/server/knowledge/guards_test.go index f1d0a780f..666afd271 100644 --- a/internal/manager/server/knowledge/guards_test.go +++ b/internal/manager/server/knowledge/guards_test.go @@ -1,6 +1,8 @@ package knowledge import ( + "bytes" + "fmt" "net/http" "strings" "testing" @@ -104,7 +106,7 @@ func TestE2E_MoveDoc(t *testing.T) { // TestE2E_Validation covers the input-rejection paths that surface as 400/404. func TestE2E_Validation(t *testing.T) { - router, _ := newE2E(t) + router, store := newE2E(t) cases := []struct { name string @@ -130,6 +132,20 @@ func TestE2E_Validation(t *testing.T) { } }) + t.Run("image-only pdf rejected", func(t *testing.T) { + ct, body := buildUpload(t, "scan.pdf", string(emptyPDF(t)), "", "") + rec := req(t, router, http.MethodPost, "/v1/knowledge/upload", ct, body) + if rec.Code != http.StatusBadRequest { + t.Fatalf("want 400, got %d (%s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "no reliably extractable body text") { + t.Fatalf("unexpected error: %s", rec.Body.String()) + } + if store.count() != 0 { + t.Fatalf("image-only PDF created %d knowledge points", store.count()) + } + }) + t.Run("get missing doc 404", func(t *testing.T) { rec := req(t, router, http.MethodGet, "/v1/knowledge/docs/999999", "", nil) if rec.Code != http.StatusNotFound { @@ -138,6 +154,28 @@ func TestE2E_Validation(t *testing.T) { }) } +// emptyPDF builds a valid one-page PDF with no text layer. +func emptyPDF(t *testing.T) []byte { + t.Helper() + var b bytes.Buffer + b.WriteString("%PDF-1.4\n") + offsets := make([]int, 1, 6) + writeObject := func(n int, body string) { + offsets = append(offsets, b.Len()) + fmt.Fprintf(&b, "%d 0 obj\n%s\nendobj\n", n, body) + } + writeObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + writeObject(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + writeObject(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>") + xref := b.Len() + fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)) + for _, offset := range offsets[1:] { + fmt.Fprintf(&b, "%010d 00000 n \n", offset) + } + fmt.Fprintf(&b, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets), xref) + return b.Bytes() +} + // TestE2E_ListFiltersAndPaths exercises the source_type / tag filters and the // /paths folder-count endpoint over a small mixed corpus. func TestE2E_ListFiltersAndPaths(t *testing.T) { diff --git a/internal/manager/server/knowledge/http.go b/internal/manager/server/knowledge/http.go index 5c4abcaef..c1eb88047 100644 --- a/internal/manager/server/knowledge/http.go +++ b/internal/manager/server/knowledge/http.go @@ -8,7 +8,7 @@ // POST /v1/knowledge/docs create manual doc // PATCH /v1/knowledge/docs/{id} update manual doc title/content // DELETE /v1/knowledge/docs/{id} delete manual doc -// GET /v1/knowledge/search?q=...&limit=N keyword search across all docs +// GET /v1/knowledge/search?q=...&limit=N&mode=hybrid mixed vector search across RAG and LLM Wiki docs // // GET /v1/knowledge/repos list registered git repos // POST /v1/knowledge/repos register a git repo @@ -72,6 +72,10 @@ type Service interface { DeleteSSHIdentity(ctx context.Context, id uint64) error } +type SearchService interface { + Search(ctx context.Context, q string, opts biz.SearchOptions) ([]biz.SearchHit, error) +} + // AuthzMW is the narrow casbin middleware contract. Optional — when // nil mutating routes fall through to the legacy passthrough (any // authenticated caller, since auth middleware already gates). @@ -81,12 +85,23 @@ type AuthzMW interface { // Handler bundles the service. type Handler struct { - svc Service - authz AuthzMW + svc Service + searcher SearchService + authz AuthzMW + llmWikiSvc llmWikiService } // NewHandler builds the handler. -func NewHandler(s Service) *Handler { return &Handler{svc: s} } +func NewHandler(s Service) *Handler { return &Handler{svc: s, searcher: s} } + +// SetSearchService replaces the default RAG searcher with an optional hybrid +// searcher. The composition root uses this to add LLM Wiki results without +// coupling the knowledge HTTP layer to the Wiki bounded context. +func (h *Handler) SetSearchService(search SearchService) { + if search != nil { + h.searcher = search + } +} // SetAuthz wires the casbin middleware post-construction. func (h *Handler) SetAuthz(a AuthzMW) { h.authz = a } @@ -109,6 +124,12 @@ func passthrough(next http.Handler) http.Handler { return next } // Register wires the routes on r. func (h *Handler) Register(r chi.Router) { + if h.svc == nil { + if h.llmWikiSvc != nil { + h.registerLLMWiki(r) + } + return + } r.Get("/v1/knowledge/docs", h.listDocs) r.Get("/v1/knowledge/docs/{id}", h.getDoc) r.With(h.writeMW("knowledge:doc")).Post("/v1/knowledge/docs", h.createDoc) @@ -136,6 +157,9 @@ func (h *Handler) Register(r chi.Router) { r.With(h.writeMW("knowledge:repo")).Post("/v1/knowledge/ssh-identities/generate", h.generateSSHIdentity) r.With(h.writeMW("knowledge:repo")).Patch("/v1/knowledge/ssh-identities/{id}", h.updateSSHIdentity) r.With(h.deleteMW("knowledge:repo")).Delete("/v1/knowledge/ssh-identities/{id}", h.deleteSSHIdentity) + if h.llmWikiSvc != nil { + h.registerLLMWiki(r) + } } // --- DTOs --- @@ -445,6 +469,7 @@ func (h *Handler) search(w http.ResponseWriter, r *http.Request) { } } opts := biz.SearchOptions{ + Mode: r.URL.Query().Get("mode"), Limit: limit, Path: r.URL.Query().Get("path"), PathPrefix: r.URL.Query().Get("path_prefix"), @@ -453,18 +478,31 @@ func (h *Handler) search(w http.ResponseWriter, r *http.Request) { // repeat ?tag=a&tag=b opts.Tags = tags } - hits, err := h.svc.Search(r.Context(), q, opts) + hits, err := h.searcher.Search(r.Context(), q, opts) if err != nil { writeErr(w, err) return } type hitDTO struct { - Doc docDTO `json:"doc"` - Score float64 `json:"score"` + Doc docDTO `json:"doc"` + Score float64 `json:"score"` + Layer string `json:"layer,omitempty"` + PageType string `json:"page_type,omitempty"` + PageID string `json:"page_id,omitempty"` + SourceVersionID string `json:"source_version_id,omitempty"` + MatchedNode string `json:"matched_node,omitempty"` } out := make([]hitDTO, 0, len(hits)) for _, h := range hits { - out = append(out, hitDTO{Doc: toDocDTO(h.Doc, true), Score: h.Score}) + out = append(out, hitDTO{ + Doc: toDocDTO(h.Doc, true), + Score: h.Score, + Layer: h.Layer, + PageType: h.PageType, + PageID: h.PageID, + SourceVersionID: h.SourceVersionID, + MatchedNode: h.MatchedNode, + }) } writeJSON(w, http.StatusOK, map[string]any{"items": out, "total": len(out)}) } diff --git a/internal/manager/server/knowledge/http_test.go b/internal/manager/server/knowledge/http_test.go index 78c940225..bf9ba660b 100644 --- a/internal/manager/server/knowledge/http_test.go +++ b/internal/manager/server/knowledge/http_test.go @@ -37,7 +37,7 @@ func newMemVec() *memVec { return &memVec{points: map[uint64]qdrantx.SearchHit{} func (m *memVec) count() int { return len(m.points) } -func (m *memVec) EnsureCollection(context.Context, string, int) error { return nil } +func (m *memVec) EnsureCollection(context.Context, string, int) error { return nil } func (m *memVec) EnsurePayloadIndex(context.Context, string, string, string) error { return nil } func (m *memVec) Upsert(_ context.Context, _ string, pts []qdrantx.Point) error { @@ -208,6 +208,55 @@ func newE2E(t *testing.T) (http.Handler, *memVec) { return r, store } +type mixedSearchStub struct { + got biz.SearchOptions +} + +func (s *mixedSearchStub) Search(_ context.Context, _ string, opts biz.SearchOptions) ([]biz.SearchHit, error) { + s.got = opts + return []biz.SearchHit{ + {Doc: &model.Doc{ID: 1, SourceType: "wiki", Title: "RAG Wiki", Content: "wiki preview"}, Score: 0.9, Layer: "wiki", PageType: "topic", PageID: "topic-rag", SourceVersionID: "7", MatchedNode: "HNSW"}, + {Doc: &model.Doc{ID: 2, SourceType: "upload", Title: "RAG Source", Content: "raw preview"}, Score: 0.8, Layer: "raw"}, + }, nil +} + +func TestSearch_ReturnsMixedHitMetadata(t *testing.T) { + store := newMemVec() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + uc, err := biz.New(context.Background(), nil, store, idEmbed{}, t.TempDir(), log) + if err != nil { + t.Fatalf("biz.New: %v", err) + } + searcher := &mixedSearchStub{} + handler := NewHandler(uc) + handler.SetSearchService(searcher) + router := chi.NewRouter() + handler.Register(router) + + rec := req(t, router, http.MethodGet, "/v1/knowledge/search?q=rag&limit=2&mode=hybrid", "", nil) + if rec.Code != http.StatusOK { + t.Fatalf("search: want 200, got %d (%s)", rec.Code, rec.Body.String()) + } + var body struct { + Items []struct { + Doc docDTO `json:"doc"` + Layer string `json:"layer"` + PageID string `json:"page_id"` + SourceVersionID string `json:"source_version_id"` + MatchedNode string `json:"matched_node"` + } `json:"items"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if len(body.Items) != 2 || body.Items[0].Layer != "wiki" || body.Items[0].PageID != "topic-rag" || body.Items[0].SourceVersionID != "7" || body.Items[0].MatchedNode != "HNSW" || body.Items[1].Layer != "raw" { + t.Fatalf("mixed search response = %+v", body.Items) + } + if searcher.got.Mode != "hybrid" || searcher.got.Limit != 2 { + t.Fatalf("search options = %+v", searcher.got) + } +} + // req fires one request and returns the recorder. body==nil sends no body; // a string body is sent verbatim with the given content type. func req(t *testing.T, router http.Handler, method, path, contentType string, body io.Reader) *httptest.ResponseRecorder { diff --git a/internal/manager/server/knowledge/llmwiki_http.go b/internal/manager/server/knowledge/llmwiki_http.go new file mode 100644 index 000000000..855c352b5 --- /dev/null +++ b/internal/manager/server/knowledge/llmwiki_http.go @@ -0,0 +1,347 @@ +// LLM Wiki HTTP routes and response types. +package knowledge + +import ( + "context" + "encoding/json" + "errors" + "mime" + "net/http" + "strconv" + "time" + + "github.com/go-chi/chi/v5" + orgbiz "github.com/ongridio/ongrid/internal/manager/biz/knowledge" + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + orgmodel "github.com/ongridio/ongrid/internal/manager/model/knowledge" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/ongridio/ongrid/internal/pkg/errs" +) + +type llmWikiService interface { + ListTree(ctx context.Context, layer, parentID string) ([]biz.TreeNode, error) + GetNode(ctx context.Context, id string) (*biz.NodeDetail, error) + PreviewNode(ctx context.Context, id string) (*biz.NodePreview, error) + DeleteNode(ctx context.Context, id string) error + ListSources(ctx context.Context, tenantID uint64, status string, limit int) ([]*model.Source, int64, error) + ListJobs(ctx context.Context, tenantID uint64, limit int) ([]*model.CompileJob, int64, error) + CreateCompileJob(ctx context.Context, tenantID uint64, force bool, sourceIDs []uint64) (*model.CompileJob, error) + RetryJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) + CancelJob(ctx context.Context, tenantID, id uint64) (*model.CompileJob, error) + Search(ctx context.Context, tenantID uint64, query string, limit int) ([]biz.SearchHit, error) + SyncOrganizationSources(ctx context.Context, docs []biz.OrganizationSource) (*biz.SyncResult, error) +} + +func (h *Handler) SetLLMWikiService(svc llmWikiService) { h.llmWikiSvc = svc } + +func (h *Handler) wikiWriteMW(next http.Handler) http.Handler { + if h.authz == nil { + return next + } + return h.authz.Require("knowledge:doc", "write")(next) +} + +func (h *Handler) registerLLMWiki(r chi.Router) { + r.Get("/v1/knowledge/llm-wiki/tree", h.listTree) + r.Get("/v1/knowledge/llm-wiki/nodes/{id}", h.getNode) + r.Get("/v1/knowledge/llm-wiki/nodes/{id}/preview", h.preview) + r.With(h.deleteMW("knowledge:doc")).Delete("/v1/knowledge/llm-wiki/nodes/{id}", h.deleteNode) + r.Get("/v1/knowledge/llm-wiki/sources", h.listSources) + r.Get("/v1/knowledge/llm-wiki/search", h.searchLLMWiki) + r.Get("/v1/knowledge/llm-wiki/jobs", h.listJobs) + r.With(h.wikiWriteMW).Post("/v1/knowledge/llm-wiki/sync", h.syncOrganizationSources) + r.With(h.wikiWriteMW).Post("/v1/knowledge/llm-wiki/compile", h.compile) + r.With(h.wikiWriteMW).Post("/v1/knowledge/llm-wiki/jobs/{id}/retry", h.retryJob) + r.With(h.wikiWriteMW).Post("/v1/knowledge/llm-wiki/jobs/{id}/cancel", h.cancelJob) +} + +// syncOrganizationSources godoc +// @Summary 将组织知识库同步到 LLM Wiki 原始来源 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/sync [post] +func (h *Handler) syncOrganizationSources(w http.ResponseWriter, r *http.Request) { + if h.svc == nil { + writeError(w, errors.New("knowledge service unavailable")) + return + } + docs := make([]biz.OrganizationSource, 0) + for _, sourceType := range []string{orgmodel.SourceManual, orgmodel.SourceUpload, orgmodel.SourceRepo} { + rows, err := h.svc.ListDocs(r.Context(), orgbiz.ListDocsFilter{SourceType: sourceType, All: true}) + if err != nil { + writeError(w, err) + return + } + for _, doc := range rows { + docs = append(docs, biz.OrganizationSource{ID: doc.ID, Title: doc.Title, Path: doc.Path, Content: doc.Content}) + } + } + result, err := h.llmWikiSvc.SyncOrganizationSources(r.Context(), docs) + if err != nil { + writeError(w, err) + return + } + writeData(w, http.StatusOK, result) +} + +// listTree godoc +// @Summary 列出 LLM Wiki 文件树 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/tree [get] +func (h *Handler) listTree(w http.ResponseWriter, r *http.Request) { + items, err := h.llmWikiSvc.ListTree(r.Context(), r.URL.Query().Get("layer"), r.URL.Query().Get("parent_id")) + if err != nil { + writeError(w, err) + return + } + documentCount := 0 + for _, item := range items { + if item.Kind == "file" { + documentCount++ + } + } + writeData(w, http.StatusOK, map[string]any{"items": items, "total": len(items), "document_count": documentCount}) +} + +// getNode godoc +// @Summary 读取 LLM Wiki 节点 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/nodes/{id} [get] +func (h *Handler) getNode(w http.ResponseWriter, r *http.Request) { + item, err := h.llmWikiSvc.GetNode(r.Context(), chi.URLParam(r, "id")) + if err != nil { + writeError(w, err) + return + } + writeData(w, http.StatusOK, item) +} + +// preview godoc +// @Summary 预览 LLM Wiki Raw PDF/DOCX 文件 +// @Success 200 {file} binary +// @Router /v1/knowledge/llm-wiki/nodes/{id}/preview [get] +func (h *Handler) preview(w http.ResponseWriter, r *http.Request) { + item, err := h.llmWikiSvc.PreviewNode(r.Context(), chi.URLParam(r, "id")) + if err != nil { + writeError(w, err) + return + } + w.Header().Set("Content-Type", item.ContentType) + w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": item.Name})) + w.WriteHeader(http.StatusOK) + if _, err := w.Write(item.Content); err != nil { + return + } +} + +// deleteNode godoc +// @Summary 删除 LLM Wiki Raw/Wiki 文件 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/nodes/{id} [delete] +func (h *Handler) deleteNode(w http.ResponseWriter, r *http.Request) { + if err := h.llmWikiSvc.DeleteNode(r.Context(), chi.URLParam(r, "id")); err != nil { + writeError(w, err) + return + } + writeData(w, http.StatusOK, map[string]bool{"deleted": true}) +} + +// listSources godoc +// @Summary 列出 LLM Wiki 来源 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/sources [get] +func (h *Handler) listSources(w http.ResponseWriter, r *http.Request) { + rows, total, err := h.llmWikiSvc.ListSources(r.Context(), biz.DefaultTenantID, r.URL.Query().Get("status"), parseLimit(r, 200)) + if err != nil { + writeError(w, err) + return + } + items := make([]sourceDTO, 0, len(rows)) + for _, row := range rows { + items = append(items, toSourceDTO(row)) + } + writeData(w, http.StatusOK, map[string]any{"items": items, "total": total}) +} + +// search godoc +// @Summary 搜索已发布的 LLM Wiki 页面 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/search [get] +func (h *Handler) searchLLMWiki(w http.ResponseWriter, r *http.Request) { + items, err := h.llmWikiSvc.Search(r.Context(), biz.DefaultTenantID, r.URL.Query().Get("q"), parseLimit(r, 10)) + if err != nil { + writeError(w, err) + return + } + writeData(w, http.StatusOK, map[string]any{"items": items, "total": len(items)}) +} + +// listJobs godoc +// @Summary 列出 LLM Wiki 编译任务 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/jobs [get] +func (h *Handler) listJobs(w http.ResponseWriter, r *http.Request) { + rows, total, err := h.llmWikiSvc.ListJobs(r.Context(), biz.DefaultTenantID, parseLimit(r, 50)) + if err != nil { + writeError(w, err) + return + } + items := make([]jobDTO, 0, len(rows)) + for _, row := range rows { + items = append(items, toJobDTO(row)) + } + writeData(w, http.StatusOK, map[string]any{"items": items, "total": total}) +} + +// compile godoc +// @Summary 创建 LLM Wiki 编译任务 +// @Success 202 {object} response +// @Router /v1/knowledge/llm-wiki/compile [post] +func (h *Handler) compile(w http.ResponseWriter, r *http.Request) { + var req struct { + Force bool `json:"force"` + SourceIDs []string `json:"source_ids,omitempty"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, errors.Join(errs.ErrInvalid, err)) + return + } + parsedIDs, err := parseStringIDs(req.SourceIDs) + if err != nil { + writeError(w, err) + return + } + job, err := h.llmWikiSvc.CreateCompileJob(r.Context(), biz.DefaultTenantID, req.Force, parsedIDs) + if err != nil { + writeError(w, err) + return + } + writeData(w, http.StatusAccepted, toJobDTO(job)) +} + +func parseStringIDs(raw []string) ([]uint64, error) { + if len(raw) == 0 { + return nil, nil + } + ids := make([]uint64, 0, len(raw)) + for _, s := range raw { + id, err := biz.ParseID(s) + if err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, nil +} + +// retryJob godoc +// @Summary 重试 LLM Wiki 编译任务 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/jobs/{id}/retry [post] +func (h *Handler) retryJob(w http.ResponseWriter, r *http.Request) { + id, err := biz.ParseID(chi.URLParam(r, "id")) + if err != nil { + writeError(w, err) + return + } + job, err := h.llmWikiSvc.RetryJob(r.Context(), biz.DefaultTenantID, id) + if err != nil { + writeError(w, err) + return + } + writeData(w, http.StatusOK, toJobDTO(job)) +} + +// cancelJob godoc +// @Summary 取消 LLM Wiki 编译任务 +// @Success 200 {object} response +// @Router /v1/knowledge/llm-wiki/jobs/{id}/cancel [post] +func (h *Handler) cancelJob(w http.ResponseWriter, r *http.Request) { + id, err := biz.ParseID(chi.URLParam(r, "id")) + if err != nil { + writeError(w, err) + return + } + job, err := h.llmWikiSvc.CancelJob(r.Context(), biz.DefaultTenantID, id) + if err != nil { + writeError(w, err) + return + } + writeData(w, http.StatusOK, toJobDTO(job)) +} + +type response struct { + Code string `json:"code"` + Message string `json:"message"` + Data any `json:"data,omitempty"` +} + +type sourceDTO struct { + ID string `json:"id"` + SourceKey string `json:"source_key"` + SourceType string `json:"source_type"` + RawPath string `json:"raw_path"` + CurrentVersionID string `json:"current_version_id,omitempty"` + Status string `json:"status"` + UpdatedAt time.Time `json:"updated_at"` +} + +type jobDTO struct { + ID string `json:"id"` + Status string `json:"status"` + Stage string `json:"stage"` + Error string `json:"error,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func toSourceDTO(row *model.Source) sourceDTO { + out := sourceDTO{ID: strconv.FormatUint(row.ID, 10), SourceKey: row.SourceKey, SourceType: row.SourceType, RawPath: row.RawPath, Status: row.Status, UpdatedAt: row.UpdatedAt} + if row.CurrentVersionID != nil { + out.CurrentVersionID = strconv.FormatUint(*row.CurrentVersionID, 10) + } + return out +} + +func toJobDTO(row *model.CompileJob) jobDTO { + return jobDTO{ID: strconv.FormatUint(row.ID, 10), Status: row.Status, Stage: row.Stage, Error: row.ErrorMessage, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt} +} + +func parseLimit(r *http.Request, fallback int) int { + value, err := strconv.Atoi(r.URL.Query().Get("limit")) + if err != nil || value <= 0 { + return fallback + } + return value +} + +func writeData(w http.ResponseWriter, status int, data any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(response{Code: "ok", Message: "ok", Data: data}); err != nil { + return + } +} + +func writeError(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + code := "internal" + message := "internal error" + if errors.Is(err, errs.ErrInvalid) { + status = http.StatusBadRequest + code = "invalid_argument" + message = err.Error() + } else if errors.Is(err, errs.ErrNotFound) { + status = http.StatusNotFound + code = "not_found" + message = "not found" + } else if errors.Is(err, errs.ErrConflict) { + status = http.StatusConflict + code = "conflict" + message = err.Error() + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if encodeErr := json.NewEncoder(w).Encode(response{Code: code, Message: message}); encodeErr != nil { + return + } +} diff --git a/internal/manager/server/knowledge/llmwiki_http_test.go b/internal/manager/server/knowledge/llmwiki_http_test.go new file mode 100644 index 000000000..4c0b54e04 --- /dev/null +++ b/internal/manager/server/knowledge/llmwiki_http_test.go @@ -0,0 +1,249 @@ +package knowledge + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + orgbiz "github.com/ongridio/ongrid/internal/manager/biz/knowledge" + biz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + orgmodel "github.com/ongridio/ongrid/internal/manager/model/knowledge" + model "github.com/ongridio/ongrid/internal/manager/model/knowledge/llm_wiki" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type serviceStub struct { + compiled bool + tenantID uint64 + force bool + deleted string + syncDocs []biz.OrganizationSource + syncRuns int +} + +func (*serviceStub) ListTree(context.Context, string, string) ([]biz.TreeNode, error) { + return nil, nil +} + +func (*serviceStub) GetNode(context.Context, string) (*biz.NodeDetail, error) { return nil, nil } + +func (*serviceStub) PreviewNode(context.Context, string) (*biz.NodePreview, error) { + return &biz.NodePreview{Name: "guide.pdf", ContentType: "application/pdf", Content: []byte("%PDF")}, nil +} + +func (s *serviceStub) DeleteNode(_ context.Context, id string) error { + s.deleted = id + return nil +} + +func (*serviceStub) ListSources(context.Context, uint64, string, int) ([]*model.Source, int64, error) { + return nil, 0, nil +} + +func (*serviceStub) ListJobs(context.Context, uint64, int) ([]*model.CompileJob, int64, error) { + return nil, 0, nil +} + +func (s *serviceStub) CreateCompileJob(_ context.Context, tenantID uint64, force bool, sourceIDs []uint64) (*model.CompileJob, error) { + s.compiled = true + s.tenantID = tenantID + s.force = force + return &model.CompileJob{ID: 9007199254740993, ForceCompile: force, Status: model.JobPending, Stage: "queued"}, nil +} + +func (*serviceStub) RetryJob(context.Context, uint64, uint64) (*model.CompileJob, error) { + return nil, nil +} + +func (*serviceStub) CancelJob(context.Context, uint64, uint64) (*model.CompileJob, error) { + return nil, nil +} + +func (*serviceStub) Search(context.Context, uint64, string, int) ([]biz.SearchHit, error) { + return nil, nil +} + +func (s *serviceStub) SyncOrganizationSources(_ context.Context, docs []biz.OrganizationSource) (*biz.SyncResult, error) { + s.syncRuns++ + s.syncDocs = append([]biz.OrganizationSource(nil), docs...) + return &biz.SyncResult{Total: len(docs)}, nil +} + +type organizationDocsStub struct { + Service + docs map[string][]*orgmodel.Doc + filters []orgbiz.ListDocsFilter + errType string +} + +func (s *organizationDocsStub) ListDocs(_ context.Context, filter orgbiz.ListDocsFilter) ([]*orgmodel.Doc, error) { + s.filters = append(s.filters, filter) + if filter.SourceType == s.errType { + return nil, errors.New("list docs failed") + } + return s.docs[filter.SourceType], nil +} + +func newLLMWikiHandler(svc *serviceStub) *Handler { + h := NewHandler(nil) + h.SetLLMWikiService(svc) + return h +} + +func TestSyncOrganizationSources_IncludesEveryOrganizationSourceType(t *testing.T) { + orgSvc := &organizationDocsStub{docs: map[string][]*orgmodel.Doc{ + "manual": {{ID: 1, Title: "Manual", Path: "ops", Content: "manual body"}}, + "upload": {{ID: 2, Title: "Upload", Path: "docs", Content: "upload body"}}, + "repo": {{ID: 3, SourceType: "repo", RepoID: ptrUint64(9), URL: "cmd/main.go", Title: "main", Path: "cmd", Content: "package main"}}, + }} + wikiSvc := &serviceStub{} + handler := NewHandler(orgSvc) + handler.SetLLMWikiService(wikiSvc) + router := chi.NewRouter() + handler.Register(router) + + request := httptest.NewRequest(http.MethodPost, "/v1/knowledge/llm-wiki/sync", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code, response.Body.String()) + require.Len(t, orgSvc.filters, 3) + assert.Equal(t, []string{"manual", "upload", "repo"}, []string{ + orgSvc.filters[0].SourceType, + orgSvc.filters[1].SourceType, + orgSvc.filters[2].SourceType, + }) + for _, filter := range orgSvc.filters { + assert.True(t, filter.All) + } + require.Equal(t, 1, wikiSvc.syncRuns) + require.Len(t, wikiSvc.syncDocs, 3) + assert.Equal(t, biz.OrganizationSource{ID: 3, Title: "main", Path: "cmd", Content: "package main"}, wikiSvc.syncDocs[2]) +} + +func TestSyncOrganizationSources_DoesNotMirrorPartialListing(t *testing.T) { + orgSvc := &organizationDocsStub{ + docs: map[string][]*orgmodel.Doc{"manual": {{ID: 1, Title: "Manual", Content: "body"}}}, + errType: "upload", + } + wikiSvc := &serviceStub{} + handler := NewHandler(orgSvc) + handler.SetLLMWikiService(wikiSvc) + router := chi.NewRouter() + handler.Register(router) + + request := httptest.NewRequest(http.MethodPost, "/v1/knowledge/llm-wiki/sync", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + + assert.Equal(t, http.StatusInternalServerError, response.Code) + assert.Zero(t, wikiSvc.syncRuns) + assert.Len(t, orgSvc.filters, 2) +} + +func ptrUint64(value uint64) *uint64 { return &value } + +func TestCompile_ReturnsEnvelopeAndStringIDs(t *testing.T) { + svc := &serviceStub{} + router := chi.NewRouter() + newLLMWikiHandler(svc).Register(router) + req := httptest.NewRequest(http.MethodPost, "/v1/knowledge/llm-wiki/compile", strings.NewReader(`{"force":true}`)) + req.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + if response.Code != http.StatusAccepted { + t.Fatalf("status = %d, body=%s", response.Code, response.Body.String()) + } + if !svc.compiled || svc.tenantID != biz.DefaultTenantID || !svc.force { + t.Fatalf("compile request = %+v", svc) + } + body := response.Body.String() + if !strings.Contains(body, `"code":"ok"`) || !strings.Contains(body, `"id":"9007199254740993"`) { + t.Fatalf("response = %s", body) + } +} + +func TestPreview_ReturnsInlineBinaryContent(t *testing.T) { + router := chi.NewRouter() + newLLMWikiHandler(&serviceStub{}).Register(router) + req := httptest.NewRequest(http.MethodGet, "/v1/knowledge/llm-wiki/nodes/raw-guide/preview", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", response.Code, response.Body.String()) + } + if got := response.Header().Get("Content-Type"); got != "application/pdf" { + t.Fatalf("content type = %q", got) + } + if got := response.Header().Get("Content-Disposition"); !strings.Contains(got, `inline`) || !strings.Contains(got, `guide.pdf`) { + t.Fatalf("content disposition = %q", got) + } + if got := response.Body.String(); got != "%PDF" { + t.Fatalf("body = %q", got) + } +} + +func TestDeleteNode_ReturnsSuccessEnvelope(t *testing.T) { + router := chi.NewRouter() + newLLMWikiHandler(&serviceStub{}).Register(router) + req := httptest.NewRequest(http.MethodDelete, "/v1/knowledge/llm-wiki/nodes/raw-guide", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), `"deleted":true`) { + t.Fatalf("response = %s", response.Body.String()) + } +} + +// denyAuthz records which permission each route asks for and rejects the +// request, so a mutating route that lost its middleware shows up as a 200. +type denyAuthz struct{ required []string } + +func (a *denyAuthz) Require(obj, act string) func(http.Handler) http.Handler { + a.required = append(a.required, obj+":"+act) + return func(http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + }) + } +} + +func (a *denyAuthz) grants(permission string) bool { + return slices.Contains(a.required, permission) +} + +// TestDeleteNode_RequiresDeletePermission — the Wiki delete route removes a +// source, its versions and the published build, so it must be guarded like the +// other mutating Wiki routes. +func TestDeleteNode_RequiresDeletePermission(t *testing.T) { + authz := &denyAuthz{} + svc := &serviceStub{} + handler := newLLMWikiHandler(svc) + handler.SetAuthz(authz) + router := chi.NewRouter() + handler.Register(router) + + req := httptest.NewRequest(http.MethodDelete, "/v1/knowledge/llm-wiki/nodes/raw-guide", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, req) + + if response.Code != http.StatusForbidden { + t.Fatalf("status = %d, body=%s", response.Code, response.Body.String()) + } + if !authz.grants("knowledge:doc:delete") { + t.Fatalf("required permissions = %v", authz.required) + } + if svc.deleted != "" { + t.Fatalf("delete reached the service with id %q", svc.deleted) + } +} diff --git a/internal/manager/service/knowledge/hybrid_search.go b/internal/manager/service/knowledge/hybrid_search.go new file mode 100644 index 000000000..c2dcd0b55 --- /dev/null +++ b/internal/manager/service/knowledge/hybrid_search.go @@ -0,0 +1,129 @@ +// Package knowledge contains application services that compose the +// operator knowledge base with the compiled LLM Wiki. +package knowledge + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "errors" + "strings" + + knowledgebiz "github.com/ongridio/ongrid/internal/manager/biz/knowledge" + llmwikibiz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + knowledgemodel "github.com/ongridio/ongrid/internal/manager/model/knowledge" +) + +// RawSearcher is the part of the operator knowledge base needed by the +// hybrid search service. +type RawSearcher interface { + Search(ctx context.Context, query string, opts knowledgebiz.SearchOptions) ([]knowledgebiz.SearchHit, error) +} + +// HybridSearcher merges compiled Wiki hits with the existing Raw knowledge +// base. It is shared by the Agent tool and the Knowledge HTTP API. +type HybridSearcher struct { + raw RawSearcher + wiki *llmwikibiz.Usecase +} + +// NewHybridSearcher creates a searcher. wiki may be nil when the LLM Wiki +// feature is disabled; Raw search remains available in that case. +func NewHybridSearcher(raw RawSearcher, wiki *llmwikibiz.Usecase) *HybridSearcher { + return &HybridSearcher{raw: raw, wiki: wiki} +} + +// hasSearchFilters reports whether the caller restricted the search to a +// knowledge-base path or to a set of tags. +func hasSearchFilters(opts knowledgebiz.SearchOptions) bool { + return strings.TrimSpace(opts.Path) != "" || + strings.TrimSpace(opts.PathPrefix) != "" || + len(opts.Tags) > 0 +} + +// Search implements the application-level hybrid retrieval policy. +func (s *HybridSearcher) Search(ctx context.Context, query string, opts knowledgebiz.SearchOptions) ([]knowledgebiz.SearchHit, error) { + if opts.Limit <= 0 { + opts.Limit = 10 + } + mode := strings.ToLower(strings.TrimSpace(opts.Mode)) + if mode == "" { + mode = "hybrid" + } + if mode == "rag" || s.wiki == nil { + opts.Mode = "" + return s.raw.Search(ctx, query, opts) + } + + // A Wiki page is generated from sources, not stored under a knowledge-base + // path, and carries no tags — so a Wiki hit cannot be checked against Path, + // PathPrefix or Tags. Returning one anyway would smuggle in a page the caller + // explicitly excluded, so a filtered query never merges Wiki hits: hybrid and + // rag answer from the raw knowledge base, and a wiki-only request reports no + // hits rather than unfiltered ones. + if hasSearchFilters(opts) { + if mode == "wiki" { + return nil, nil + } + opts.Mode = "" + return s.raw.Search(ctx, query, opts) + } + + wikiHits, wikiErr := s.wiki.Search(ctx, llmwikibiz.DefaultTenantID, query, opts.Limit) + converted := make([]knowledgebiz.SearchHit, 0, len(wikiHits)) + for _, hit := range wikiHits { + digest := sha256.Sum256([]byte(hit.PageID)) + converted = append(converted, knowledgebiz.SearchHit{ + Doc: &knowledgemodel.Doc{ + ID: binary.BigEndian.Uint64(digest[:8]), + SourceType: "wiki", + Title: hit.Title, + Content: hit.Preview, + }, + Score: hit.Score, + Layer: hit.Layer, + PageType: hit.PageType, + PageID: hit.PageID, + SourceVersionID: hit.SourceVersionID, + MatchedNode: hit.MatchedNode, + }) + } + if mode == "wiki" { + return converted, wikiErr + } + + opts.Mode = "" + rawHits, rawErr := s.raw.Search(ctx, query, opts) + if wikiErr != nil && rawErr != nil { + return nil, errors.Join(wikiErr, rawErr) + } + if wikiErr != nil { + return rawHits, nil + } + if rawErr != nil { + return converted, nil + } + for rank := range rawHits { + if rawHits[rank].Layer == "" { + rawHits[rank].Layer = "raw" + } + } + + // The two layers interleave by rank, Wiki first. A weighted rank merge cannot + // do that: any Wiki weight above 1.0 puts every Wiki hit ahead of every raw + // hit (1.5/61 > 1.0/61), so a limit of ten Wiki hits crowded the operator + // playbooks out of every result — the raw layer is the evidence the Wiki is + // compiled from and the one the knowledge prologue looks for, so it must stay + // reachable. Whichever layer runs out of hits first hands its remaining slots + // to the other, and each hit keeps the relevance its own layer measured. + merged := make([]knowledgebiz.SearchHit, 0, len(converted)+len(rawHits)) + for rank := 0; len(merged) < opts.Limit && (rank < len(converted) || rank < len(rawHits)); rank++ { + if rank < len(converted) { + merged = append(merged, converted[rank]) + } + if len(merged) < opts.Limit && rank < len(rawHits) { + merged = append(merged, rawHits[rank]) + } + } + return merged, nil +} diff --git a/internal/manager/service/knowledge/hybrid_search_test.go b/internal/manager/service/knowledge/hybrid_search_test.go new file mode 100644 index 000000000..aa97d4476 --- /dev/null +++ b/internal/manager/service/knowledge/hybrid_search_test.go @@ -0,0 +1,253 @@ +package knowledge + +import ( + "context" + "fmt" + "io" + "log/slog" + "testing" + + "github.com/glebarez/sqlite" + knowledgebiz "github.com/ongridio/ongrid/internal/manager/biz/knowledge" + llmwikibiz "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + llmwikistore "github.com/ongridio/ongrid/internal/manager/data/knowledge/llm_wiki/store" + knowledgemodel "github.com/ongridio/ongrid/internal/manager/model/knowledge" + "github.com/ongridio/ongrid/internal/pkg/llm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// countingIndex stands in for the Wiki search index and records whether a query +// reached it at all. hits overrides the single default result when set. +type countingIndex struct { + queries int + hits []llmwikibiz.SearchHit +} + +func (i *countingIndex) IndexPage(context.Context, llmwikibiz.IndexDocument) error { return nil } +func (i *countingIndex) Clear(context.Context, uint64) error { return nil } + +func (i *countingIndex) Search(context.Context, uint64, string, int) ([]llmwikibiz.SearchHit, error) { + i.queries++ + if i.hits != nil { + return i.hits, nil + } + return []llmwikibiz.SearchHit{{Layer: "wiki", PageID: "dns-overview", Title: "DNS", Preview: "body", Score: 1}}, nil +} + +// recordingRaw records the raw search call it answered. hits overrides the +// single default result when set. +type recordingRaw struct { + calls int + opts knowledgebiz.SearchOptions + hits []knowledgebiz.SearchHit +} + +func (r *recordingRaw) Search(_ context.Context, _ string, opts knowledgebiz.SearchOptions) ([]knowledgebiz.SearchHit, error) { + r.calls++ + r.opts = opts + if r.hits != nil { + return r.hits, nil + } + return []knowledgebiz.SearchHit{{Doc: &knowledgemodel.Doc{ID: 1, Title: "raw hit"}, Layer: "raw", Score: 1}}, nil +} + +// wikiHitList builds n Wiki hits, already in the order the index ranked them. +func wikiHitList(n int) []llmwikibiz.SearchHit { + hits := make([]llmwikibiz.SearchHit, 0, n) + for i := 0; i < n; i++ { + hits = append(hits, llmwikibiz.SearchHit{ + Layer: "wiki", + PageID: fmt.Sprintf("wiki-%d", i), + Title: fmt.Sprintf("Wiki %d", i), + Preview: "body", + Score: 0.9, + }) + } + return hits +} + +// rawHitList builds n raw hits, already in the order the vector search ranked them. +func rawHitList(n int) []knowledgebiz.SearchHit { + hits := make([]knowledgebiz.SearchHit, 0, n) + for i := 0; i < n; i++ { + hits = append(hits, knowledgebiz.SearchHit{ + Doc: &knowledgemodel.Doc{ID: uint64(i + 1), Title: fmt.Sprintf("Raw %d", i)}, + Layer: "raw", + Score: 0.8, + }) + } + return hits +} + +// newHybridSearcher wires a hybrid searcher over a real Wiki usecase backed by an +// in-memory store, so the Wiki branch behaves as it does in production. +func newHybridSearcher(t *testing.T, index *countingIndex) (*HybridSearcher, *recordingRaw) { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + require.NoError(t, err) + repo := llmwikistore.New(db) + require.NoError(t, llmwikistore.Migrate(db)) + files, err := llmwikibiz.NewFileStore(t.TempDir()) + require.NoError(t, err) + wiki, err := llmwikibiz.NewWithUsageRecorder(context.Background(), repo, files, stubCompilerLLM{}, index, slog.New(slog.NewTextHandler(io.Discard, nil)), nil) + require.NoError(t, err) + + raw := &recordingRaw{} + return NewHybridSearcher(raw, wiki), raw +} + +type stubCompilerLLM struct{} + +func (stubCompilerLLM) Complete(context.Context, llm.ChatReq) (*llm.ChatResp, error) { + return nil, nil +} +func (stubCompilerLLM) ModelVersion() string { return "stub" } + +// TestHybridSearch_WithoutFiltersMergesWikiHits is the baseline the filtered case +// is contrasted against. +func TestHybridSearch_WithoutFiltersMergesWikiHits(t *testing.T) { + index := &countingIndex{} + searcher, raw := newHybridSearcher(t, index) + + hits, err := searcher.Search(context.Background(), "dns", knowledgebiz.SearchOptions{Limit: 10}) + + require.NoError(t, err) + assert.Equal(t, 1, index.queries) + assert.Equal(t, 1, raw.calls) + layers := make([]string, 0, len(hits)) + for _, hit := range hits { + layers = append(layers, hit.Layer) + } + assert.Contains(t, layers, "wiki") + assert.Contains(t, layers, "raw") +} + +// TestHybridSearch_KeepsLayerScoresAndOrdersByFusion — the searcher orders the +// merged list by RRF, but it must not publish the rank value as the score. Rank +// values span 1.5/(60+rank) down to 1.0/(60+rank), so overwriting the layers' +// own relevance with one put every result inside a hundredth of the others: the +// Knowledge page rendered the whole top ten as "0.02", and the agent's knowledge +// prologue, which injects a playbook only above 0.6, stopped firing entirely. +func TestHybridSearch_KeepsLayerScoresAndOrdersByFusion(t *testing.T) { + index := &countingIndex{} + searcher, _ := newHybridSearcher(t, index) + + hits, err := searcher.Search(context.Background(), "dns", knowledgebiz.SearchOptions{Limit: 10}) + + require.NoError(t, err) + require.Len(t, hits, 2) + // The interleave puts Wiki first at equal rank... + assert.Equal(t, "wiki", hits[0].Layer) + assert.Equal(t, "raw", hits[1].Layer) + // ...and each hit still reports the relevance its own layer measured. + for _, hit := range hits { + assert.Equal(t, 1.0, hit.Score, "hybrid must not replace the layer's own score") + } +} + +// TestHybridSearch_InterleavesSoNeitherLayerIsCrowdedOut — a page of Wiki hits +// used to hide the raw knowledge base completely. The rank weights put every +// Wiki hit above every raw hit (1.5/61 > 1.0/61), so the operator playbooks — +// the evidence the Wiki is compiled from — were unreachable whenever the Wiki +// had enough pages to fill the result. +func TestHybridSearch_InterleavesSoNeitherLayerIsCrowdedOut(t *testing.T) { + index := &countingIndex{hits: wikiHitList(10)} + searcher, raw := newHybridSearcher(t, index) + raw.hits = rawHitList(10) + + hits, err := searcher.Search(context.Background(), "dns", knowledgebiz.SearchOptions{Limit: 10}) + + require.NoError(t, err) + require.Len(t, hits, 10) + for position, want := range []string{"wiki", "raw", "wiki", "raw", "wiki", "raw"} { + assert.Equal(t, want, hits[position].Layer, "position %d", position) + } + layers := map[string]int{} + for _, hit := range hits { + layers[hit.Layer]++ + } + assert.Equal(t, 5, layers["wiki"]) + assert.Equal(t, 5, layers["raw"]) +} + +// TestHybridSearch_ShortLayerHandsItsSlotsToTheOther — the interleave must not +// waste a slot on a layer that has no hit left for that rank. +func TestHybridSearch_ShortLayerHandsItsSlotsToTheOther(t *testing.T) { + index := &countingIndex{hits: wikiHitList(2)} + searcher, raw := newHybridSearcher(t, index) + raw.hits = rawHitList(10) + + hits, err := searcher.Search(context.Background(), "dns", knowledgebiz.SearchOptions{Limit: 10}) + + require.NoError(t, err) + require.Len(t, hits, 10) + layers := map[string]int{} + for _, hit := range hits { + layers[hit.Layer]++ + } + assert.Equal(t, 2, layers["wiki"]) + assert.Equal(t, 8, layers["raw"]) +} + +// TestHybridSearch_WikiOnlySearchIsUnchanged — the interleave only shapes the +// merged hybrid result, not a caller that asked for one layer. +func TestHybridSearch_WikiOnlySearchIsUnchanged(t *testing.T) { + index := &countingIndex{hits: wikiHitList(4)} + searcher, raw := newHybridSearcher(t, index) + + hits, err := searcher.Search(context.Background(), "dns", knowledgebiz.SearchOptions{Limit: 10, Mode: "wiki"}) + + require.NoError(t, err) + require.Len(t, hits, 4) + assert.Zero(t, raw.calls) + for _, hit := range hits { + assert.Equal(t, "wiki", hit.Layer) + } +} + +// TestHybridSearch_WithFiltersDropsWikiHits — a Wiki page is generated from +// sources rather than stored under a knowledge-base path and carries no tags, so +// a filtered query cannot be answered from the Wiki without returning pages the +// caller excluded. The Knowledge page sends path_prefix when a directory is +// selected, which used to mix in Wiki pages from anywhere. +func TestHybridSearch_WithFiltersDropsWikiHits(t *testing.T) { + filters := map[string]knowledgebiz.SearchOptions{ + "path": {Path: "docs/runbook.md"}, + "path prefix": {PathPrefix: "docs"}, + "tags": {Tags: []string{"ops"}}, + } + for name, opts := range filters { + t.Run(name, func(t *testing.T) { + index := &countingIndex{} + searcher, raw := newHybridSearcher(t, index) + opts.Limit = 10 + opts.Mode = "hybrid" + + hits, err := searcher.Search(context.Background(), "dns", opts) + + require.NoError(t, err) + assert.Zero(t, index.queries, "a filtered query must not consult the Wiki index") + assert.Equal(t, 1, raw.calls) + require.Len(t, hits, 1) + require.NotNil(t, hits[0].Doc) + assert.Equal(t, "raw hit", hits[0].Doc.Title) + }) + } +} + +// TestHybridSearch_WikiOnlyWithFiltersReportsNoHits — the caller asked for Wiki +// pages only, so unfiltered Wiki hits would be wrong and raw hits would ignore +// the mode: an empty result is the honest answer. +func TestHybridSearch_WikiOnlyWithFiltersReportsNoHits(t *testing.T) { + index := &countingIndex{} + searcher, raw := newHybridSearcher(t, index) + + hits, err := searcher.Search(context.Background(), "dns", knowledgebiz.SearchOptions{Limit: 10, Mode: "wiki", PathPrefix: "docs"}) + + require.NoError(t, err) + assert.Empty(t, hits) + assert.Zero(t, index.queries) + assert.Zero(t, raw.calls) +} diff --git a/internal/manager/service/knowledge/llmwiki_usage.go b/internal/manager/service/knowledge/llmwiki_usage.go new file mode 100644 index 000000000..59a09a594 --- /dev/null +++ b/internal/manager/service/knowledge/llmwiki_usage.go @@ -0,0 +1,76 @@ +package knowledge + +import ( + "context" + "fmt" + "time" + + managerbizaiops "github.com/ongridio/ongrid/internal/manager/biz/aiops" + managerbizllmwiki "github.com/ongridio/ongrid/internal/manager/biz/knowledge/llm_wiki" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/llm" +) + +// LLMWikiUsageRecorder adapts Wiki compilation to the existing chat +// transcript usage model. Internal work sessions are excluded from the user +// chat list but remain included in the established global token aggregates. +type LLMWikiUsageRecorder struct { + sessions managerbizaiops.SessionRepo +} + +var _ managerbizllmwiki.TokenUsageRecorder = (*LLMWikiUsageRecorder)(nil) + +// NewLLMWikiUsageRecorder creates the adapter used by the manager composition +// root. It deliberately writes through the existing AIOps session repository. +func NewLLMWikiUsageRecorder(sessions managerbizaiops.SessionRepo) managerbizllmwiki.TokenUsageRecorder { + return &LLMWikiUsageRecorder{sessions: sessions} +} + +func (r *LLMWikiUsageRecorder) Start(ctx context.Context, jobID uint64) (managerbizllmwiki.TokenUsageSink, error) { + if r == nil || r.sessions == nil { + return nil, fmt.Errorf("llmwiki usage: AIOps session repository is not configured") + } + session := &aiopsmodel.Session{ + UserID: 0, + Title: fmt.Sprintf("LLM Wiki compile job %d", jobID), + Kind: aiopsmodel.SessionKindWork, + Initiator: aiopsmodel.SessionInitiatorScheduler, + Audience: aiopsmodel.SessionAudienceInternal, + } + if err := r.sessions.CreateSession(ctx, session); err != nil { + return nil, fmt.Errorf("llmwiki usage: create session: %w", err) + } + return &llmWikiUsageSink{sessions: r.sessions, sessionID: session.ID}, nil +} + +type llmWikiUsageSink struct { + sessions managerbizaiops.SessionRepo + sessionID string +} + +var _ managerbizllmwiki.TokenUsageSink = (*llmWikiUsageSink)(nil) + +func (s *llmWikiUsageSink) Record(ctx context.Context, usage llm.Usage) error { + if s == nil || s.sessions == nil || s.sessionID == "" { + return fmt.Errorf("llmwiki usage: session sink is not configured") + } + promptTokens := usage.PromptTokens + completionTokens := usage.CompletionTokens + return s.sessions.AppendMessage(ctx, &aiopsmodel.Message{ + SessionID: s.sessionID, + Role: "assistant", + PromptTokens: &promptTokens, + CompletionTokens: &completionTokens, + CreatedAt: time.Now().UTC(), + }) +} + +func (s *llmWikiUsageSink) Close(ctx context.Context) error { + if s == nil || s.sessions == nil || s.sessionID == "" { + return fmt.Errorf("llmwiki usage: session sink is not configured") + } + if err := s.sessions.CloseSession(ctx, s.sessionID); err != nil { + return fmt.Errorf("llmwiki usage: close session: %w", err) + } + return nil +} diff --git a/internal/manager/service/knowledge/llmwiki_usage_test.go b/internal/manager/service/knowledge/llmwiki_usage_test.go new file mode 100644 index 000000000..e9b2cdd31 --- /dev/null +++ b/internal/manager/service/knowledge/llmwiki_usage_test.go @@ -0,0 +1,55 @@ +package knowledge + +import ( + "context" + "testing" + "time" + + "github.com/glebarez/sqlite" + manageraiopsdata "github.com/ongridio/ongrid/internal/manager/data/aiops/store" + aiopsmodel "github.com/ongridio/ongrid/internal/manager/model/aiops" + "github.com/ongridio/ongrid/internal/pkg/llm" + "gorm.io/gorm" +) + +func TestLLMWikiUsageRecorder_ReusesChatTranscriptUsage(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&aiopsmodel.Session{}, &aiopsmodel.Message{}); err != nil { + t.Fatal(err) + } + + repo := manageraiopsdata.NewSessionRepo(db) + recorder := NewLLMWikiUsageRecorder(repo) + sink, err := recorder.Start(context.Background(), 42) + if err != nil { + t.Fatal(err) + } + if err := sink.Record(context.Background(), llm.Usage{PromptTokens: 120, CompletionTokens: 30, TotalTokens: 150}); err != nil { + t.Fatal(err) + } + if err := sink.Record(context.Background(), llm.Usage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}); err != nil { + t.Fatal(err) + } + if err := sink.Close(context.Background()); err != nil { + t.Fatal(err) + } + + sums, err := repo.SumTokensSince(context.Background(), time.Now().UTC().Add(-time.Minute)) + if err != nil { + t.Fatal(err) + } + if sums.PromptTokens != 130 || sums.CompletionTokens != 35 || sums.Requests != 2 { + t.Fatalf("usage sums = %+v, want prompt=130 completion=35 requests=2", sums) + } + + var session aiopsmodel.Session + if err := db.First(&session).Error; err != nil { + t.Fatal(err) + } + if session.Kind != aiopsmodel.SessionKindWork || session.Audience != aiopsmodel.SessionAudienceInternal || session.Initiator != aiopsmodel.SessionInitiatorScheduler || session.ClosedAt == nil { + t.Fatalf("session metadata = kind=%q audience=%q initiator=%q closed_at=%v", session.Kind, session.Audience, session.Initiator, session.ClosedAt) + } +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 2e8766771..66d3b3ec0 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -61,6 +61,7 @@ type Config struct { Profiles ProfilesConfig PacketCapture PacketCaptureConfig Skills SkillsConfig + LLMWiki LLMWikiConfig } // SkillsConfig wires the manager-side subprocess skill loader. The @@ -450,6 +451,12 @@ type EdgeConfig struct { SecretsFile string } +type LLMWikiConfig struct { + Enabled bool + Dir string + TimeoutSeconds int +} + // Load reads env vars and returns a Config with defaults applied. // It never returns a non-nil error in MVP; the signature leaves room // for future validation (e.g. required fields). @@ -597,6 +604,10 @@ func Load() (*Config, error) { c.Skills.ExternalDirs = getEnvCSV("ONGRID_SKILLS_EXTERNAL_DIRS", nil) + c.LLMWiki.Enabled = getEnvBool("ONGRID_LLM_WIKI_ENABLED", false) + c.LLMWiki.Dir = getEnv("ONGRID_LLM_WIKI_DIR", "/var/lib/ongrid/llm-wiki") + c.LLMWiki.TimeoutSeconds = getEnvInt("ONGRID_LLM_WIKI_TIMEOUT_SECONDS", 600) + return c, nil } diff --git a/internal/pkg/docextract/docx2md_test.go b/internal/pkg/docextract/docx2md_test.go index 8ba38c518..771d6b5d6 100644 --- a/internal/pkg/docextract/docx2md_test.go +++ b/internal/pkg/docextract/docx2md_test.go @@ -69,6 +69,24 @@ func TestDOCX2MD_UsesStyleOutlineLevelForHeading(t *testing.T) { } } +func TestExtractPlainText_DoesNotEmitMarkdown(t *testing.T) { + document := ` +Release notes +Plain paragraphwith tab +` + + got, err := ExtractPlainText("guide.docx", testDOCX(t, map[string]string{"word/document.xml": document})) + if err != nil { + t.Fatalf("extract plain text: %v", err) + } + if strings.Contains(got, "# Release notes") || strings.Contains(got, "|") { + t.Fatalf("plain preview contains Markdown syntax: %q", got) + } + if !strings.Contains(got, "Release notes\nPlain paragraph\twith tab") { + t.Fatalf("plain preview lost document text: %q", got) + } +} + func TestDOCX2MD_TableFirstRowBecomesHeader(t *testing.T) { document := ` NameVersion diff --git a/internal/pkg/docextract/extract.go b/internal/pkg/docextract/extract.go index dc74e2c0a..d212d95ba 100644 --- a/internal/pkg/docextract/extract.go +++ b/internal/pkg/docextract/extract.go @@ -1,8 +1,8 @@ // Package docextract pulls plain text out of an uploaded knowledge file so // the RAG pipeline (chunk → embed → upsert) has something to index. // Pure-Go, no CGO: md/txt are passthrough, pdf via ledongthuc/pdf, docx via -// stdlib zip + a tiny XML walk. Scanned/image PDFs (no embedded text) and -// encrypted files yield empty/err — OCR is out of scope (ADR-028 phase-2). +// stdlib zip + a tiny XML walk. Scanned/image PDFs are rejected because OCR is +// out of scope (ADR-028 phase-2). package docextract import ( @@ -45,9 +45,89 @@ func Extract(filename string, data []byte) (string, error) { } } -// extractPDF pulls the embedded text layer. Returns a friendly error when -// the PDF carries no extractable text (scanned/image-only) so the operator -// knows OCR isn't supported rather than seeing a blank doc. +// ExtractPlainText returns a non-Markdown preview for office documents. It is +// intentionally separate from Extract: the compiler needs Markdown-shaped +// structure, while a Raw DOCX preview must not pretend to be a Markdown file. +func ExtractPlainText(filename string, data []byte) (string, error) { + switch strings.ToLower(filepath.Ext(filename)) { + case ".docx": + return docxPlainText(data) + case ".pdf": + return extractPDF(data) + case ".md", ".markdown", ".txt", ".text": + if !utf8.Valid(data) { + return "", fmt.Errorf("file is not valid UTF-8 text") + } + return string(data), nil + default: + return "", fmt.Errorf("unsupported file type %q", filepath.Ext(filename)) + } +} + +func docxPlainText(data []byte) (string, error) { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return "", fmt.Errorf("read docx (not a valid .docx zip): %w", err) + } + var document *zip.File + for _, file := range zr.File { + if file.Name == "word/document.xml" { + document = file + break + } + } + if document == nil { + return "", fmt.Errorf("docx missing word/document.xml") + } + rc, err := document.Open() + if err != nil { + return "", fmt.Errorf("open docx body: %w", err) + } + defer func() { _ = rc.Close() /* read-only cleanup */ }() + + var out strings.Builder + decoder := xml.NewDecoder(rc) + inText := false + for { + token, tokenErr := decoder.Token() + if tokenErr == io.EOF { + break + } + if tokenErr != nil { + return "", fmt.Errorf("parse docx xml: %w", tokenErr) + } + switch value := token.(type) { + case xml.StartElement: + switch value.Name.Local { + case "t", "delText": + inText = true + case "tab": + out.WriteByte('\t') + case "br", "cr": + out.WriteByte('\n') + } + case xml.CharData: + if inText { + out.Write([]byte(value)) + } + case xml.EndElement: + switch value.Name.Local { + case "t", "delText": + inText = false + case "p": + out.WriteByte('\n') + } + } + } + text := strings.TrimSpace(out.String()) + if text == "" { + return "", fmt.Errorf("no extractable text in docx") + } + return text, nil +} + +// extractPDF pulls the embedded text layer. Scanned/image-only PDFs return an +// error so the upload and compile flows never treat a placeholder as content. func extractPDF(data []byte) (string, error) { r, err := pdf.NewReader(bytes.NewReader(data), int64(len(data))) if err != nil { @@ -62,10 +142,7 @@ func extractPDF(data []byte) (string, error) { return "", fmt.Errorf("read pdf text: %w", err) } out := strings.TrimSpace(b.String()) - if out == "" { - return "", fmt.Errorf("no extractable text in pdf (scanned/image PDFs need OCR, not supported)") - } - return out, nil + return validatePDFText(out) } // extractDOCX unzips the .docx (a zip of OOXML) and walks word/document.xml, diff --git a/internal/pkg/docextract/pdf2md.go b/internal/pkg/docextract/pdf2md.go index e199f056c..b3c31feb8 100644 --- a/internal/pkg/docextract/pdf2md.go +++ b/internal/pkg/docextract/pdf2md.go @@ -2,11 +2,13 @@ package docextract import ( "bytes" + "errors" "fmt" "io" "math" "sort" "strings" + "unicode" "unicode/utf8" "github.com/ledongthuc/pdf" @@ -14,6 +16,8 @@ import ( const pdfHeadingRatio = 1.2 +var errPDFNoText = errors.New("PDF has no reliably extractable body text; image-only PDFs are not supported") + // pdf2md converts an embedded PDF text layer to Markdown. Text whose font // size is at least pdfHeadingRatio times the document's dominant body size is // emitted as a heading; remaining text is emitted as ordinary paragraphs. @@ -68,7 +72,7 @@ func pdf2md(data []byte) (out string, err error) { if out == "" { return pdfPlainText(r) } - return out, nil + return validatePDFText(out) } func pdfPlainText(r *pdf.Reader) (string, error) { @@ -81,10 +85,34 @@ func pdfPlainText(r *pdf.Reader) (string, error) { return "", fmt.Errorf("read pdf text: %w", err) } out := strings.TrimSpace(b.String()) - if out == "" { - return "", fmt.Errorf("no extractable text in pdf (scanned/image PDFs need OCR, not supported)") + return validatePDFText(out) +} + +// validatePDFText rejects empty text and text layers that contain a +// substantial amount of control data. Some scanned PDFs embed a bogus +// character map alongside their page images; ledongthuc/pdf can then return a +// non-empty string made mostly of control characters and isolated glyphs. +// Treating that string as source text produces plausible-looking but +// fabricated Wiki summaries. +func validatePDFText(text string) (string, error) { + var ( + runes int + controlRunes int + replacement int + ) + for _, r := range text { + runes++ + if r == '\uFFFD' { + replacement++ + } + if unicode.IsControl(r) && r != '\n' && r != '\r' && r != '\t' && r != '\f' { + controlRunes++ + } + } + if runes == 0 || replacement > 0 || controlRunes > 32 || controlRunes*100 > runes*5 { + return "", errPDFNoText } - return out, nil + return text, nil } func pdfBodyFontSize(texts []pdf.Text) float64 { diff --git a/internal/pkg/docextract/pdf2md_test.go b/internal/pkg/docextract/pdf2md_test.go index 78d3ba22a..06c2f0208 100644 --- a/internal/pkg/docextract/pdf2md_test.go +++ b/internal/pkg/docextract/pdf2md_test.go @@ -2,6 +2,7 @@ package docextract import ( "bytes" + "errors" "fmt" "strings" "testing" @@ -64,9 +65,55 @@ func TestPDF2MD_CapsHeadingLevelAtH3(t *testing.T) { } func TestPDF2MD_WithoutTextReturnsError(t *testing.T) { - _, err := pdf2md(testPDF(t, "")) - if err == nil || !strings.Contains(err.Error(), "no extractable text") { - t.Fatalf("empty PDF error = %v, want no extractable text", err) + got, err := pdf2md(testPDF(t, "")) + if !errors.Is(err, errPDFNoText) { + t.Fatalf("empty PDF error = %v, want errPDFNoText", err) + } + if got != "" { + t.Fatalf("empty PDF text = %q, want empty", got) + } +} + +func TestValidatePDFText_RejectsCorruptedTextLayer(t *testing.T) { + corrupted := strings.Repeat("\x1b\u0091x\n", 40) + got, err := validatePDFText(corrupted) + if !errors.Is(err, errPDFNoText) { + t.Fatalf("corrupted text error = %v, want errPDFNoText", err) + } + if got != "" { + t.Fatalf("corrupted text = %q, want empty", got) + } +} + +func TestValidatePDFText_AcceptsReadableText(t *testing.T) { + readable := "# 原则\n\n每条原则都应能被清晰验证。\n" + got, err := validatePDFText(readable) + if err != nil { + t.Fatalf("readable text rejected: %v", err) + } + if got != readable { + t.Fatalf("readable text changed: %q", got) + } +} + +func TestExtract_PDFWithoutTextReturnsError(t *testing.T) { + data := testPDF(t, "") + for _, tc := range []struct { + name string + call func(string, []byte) (string, error) + }{ + {name: "markdown", call: Extract}, + {name: "plain text", call: ExtractPlainText}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.call("scan.pdf", data) + if !errors.Is(err, errPDFNoText) { + t.Fatalf("error = %v, want errPDFNoText", err) + } + if got != "" { + t.Fatalf("text = %q, want empty", got) + } + }) } } diff --git a/internal/pkg/llm/client.go b/internal/pkg/llm/client.go index dcb0693d1..1a8f36852 100644 --- a/internal/pkg/llm/client.go +++ b/internal/pkg/llm/client.go @@ -110,14 +110,35 @@ type Usage struct { // "zhipu", "gemini"). Empty → router uses the default provider. The // non-multi-provider single-client path ignores Provider. type ChatReq struct { - Model string - Provider string - Messages []Message - Tools []ToolSchema - Temperature float32 - UserID uint64 // optional; used for budget scoping + logging only + Model string + Provider string + Messages []Message + Tools []ToolSchema + ResponseFormat *ResponseFormat + Temperature float32 + // MaxOutputTokens bounds visible output plus reasoning tokens when the + // provider supports the OpenAI-compatible completion limit. Zero leaves + // the provider default unchanged. + MaxOutputTokens int + UserID uint64 // optional; used for budget scoping + logging only +} + +// ResponseFormat describes an OpenAI-compatible structured response request. +// Schema is passed through as JSON because providers differ in the subset of +// JSON Schema they accept; request translation stays inside this package. +type ResponseFormat struct { + Type string + Name string + Description string + Schema json.RawMessage + Strict bool } +const ( + ResponseFormatJSONObject = "json_object" + ResponseFormatJSONSchema = "json_schema" +) + // ChatResp is the output of Client.Chat. type ChatResp struct { Assistant Message // role=assistant; may have empty Content + non-empty ToolCalls @@ -257,6 +278,13 @@ type openaiClient struct { // the isReasoningModel name heuristic. Keyed by the raw model string. noSamplingMu sync.RWMutex noSampling map[string]bool + + // jsonSchemaUnsupported records providers/models that reject the + // OpenAI structured-output response format. The key includes the + // normalized base URL because the same model name can be served by + // different providers with different capabilities. + jsonSchemaUnsupportedMu sync.RWMutex + jsonSchemaUnsupported map[string]bool } type sdkKey struct { @@ -439,6 +467,13 @@ func (c *openaiClient) Chat(ctx context.Context, req ChatReq) (*ChatResp, error) defer cancel() } + // A provider may accept JSON object mode but not the newer JSON Schema + // mode. Skip the known-incompatible schema request after the first + // observed rejection; the compiler still validates the returned JSON. + if c.modelRejectsJSONSchema(baseURL, model) { + downgradeJSONSchemaResponseFormat(&sdkReq) + } + // 4. Issue the request through the SDK matching the resolved creds. // 旧消息没有可恢复的思考字段；DeepSeek 接受显式空值，但 SDK 的 // omitempty 会删掉它。仅对需要此兼容处理的请求补齐空字段。 @@ -449,18 +484,28 @@ func (c *openaiClient) Chat(ctx context.Context, req ChatReq) (*ChatResp, error) start := time.Now() sdkResp, err := sdk.CreateChatCompletion(callCtx, sdkReq) - // Reactive self-heal for reasoning models the name heuristic did not - // catch (custom gateway aliases like "gpt-5.6-sol"). These fix - // temperature/top_p/n at 1 and penalties at 0, and 400 on any other - // value. If that's what we hit AND we actually sent a sampling param, - // remember the model, strip the params, and retry once. Safe: this is - // still the single completion call — no tool has executed yet, so the - // no-retry-on-tools rule below does not apply. - if err != nil && isSamplingParamError(err) && hasCustomSampling(sdkReq) { - c.rememberNoSampling(model) - stripSamplingParams(&sdkReq) - c.log.Warn("llm: model rejects custom sampling params; retrying without them", - slog.String("model", model)) + // Reactive compatibility retries are safe here: no tool has executed yet, + // so this is still one logical completion call. At most two request-shape + // changes are possible (JSON Schema -> JSON object and custom sampling -> + // provider defaults), which prevents an accidental retry loop. + for retries := 0; err != nil && retries < 2; retries++ { + changed := false + switch { + case isResponseFormatUnsupportedError(err) && downgradeJSONSchemaResponseFormat(&sdkReq): + c.rememberNoJSONSchema(baseURL, model) + changed = true + c.log.Warn("llm: provider rejects JSON Schema response format; retrying with JSON object mode", + slog.String("model", model)) + case isSamplingParamError(err) && hasCustomSampling(sdkReq): + c.rememberNoSampling(model) + stripSamplingParams(&sdkReq) + changed = true + c.log.Warn("llm: model rejects custom sampling params; retrying without them", + slog.String("model", model)) + } + if !changed { + break + } sdkResp, err = sdk.CreateChatCompletion(callCtx, sdkReq) } @@ -577,12 +622,54 @@ func (c *openaiClient) toOpenAIReq(req ChatReq, model string) (openai.ChatComple } } - return openai.ChatCompletionRequest{ + request := openai.ChatCompletionRequest{ Model: model, Messages: msgs, Tools: tools, Temperature: temp, - }, nil + } + if req.ResponseFormat != nil { + responseFormat, err := toOpenAIResponseFormat(req.ResponseFormat) + if err != nil { + return openai.ChatCompletionRequest{}, fmt.Errorf("response format: %w", err) + } + request.ResponseFormat = responseFormat + } + if req.MaxOutputTokens > 0 { + request.MaxCompletionTokens = req.MaxOutputTokens + } + return request, nil +} + +func toOpenAIResponseFormat(format *ResponseFormat) (*openai.ChatCompletionResponseFormat, error) { + if format == nil { + return nil, nil + } + switch format.Type { + case ResponseFormatJSONObject: + return &openai.ChatCompletionResponseFormat{ + Type: openai.ChatCompletionResponseFormatTypeJSONObject, + }, nil + case ResponseFormatJSONSchema: + name := strings.TrimSpace(format.Name) + if name == "" { + return nil, errors.New("json schema name is required") + } + if len(format.Schema) == 0 || !json.Valid(format.Schema) { + return nil, errors.New("json schema must be valid JSON") + } + return &openai.ChatCompletionResponseFormat{ + Type: openai.ChatCompletionResponseFormatTypeJSONSchema, + JSONSchema: &openai.ChatCompletionResponseFormatJSONSchema{ + Name: name, + Description: format.Description, + Schema: json.RawMessage(format.Schema), + Strict: format.Strict, + }, + }, nil + default: + return nil, fmt.Errorf("unsupported type %q", format.Type) + } } func toOpenAIMessage(m Message) (openai.ChatCompletionMessage, error) { @@ -702,6 +789,58 @@ func (c *openaiClient) rememberNoSampling(model string) { c.noSampling[model] = true } +// modelRejectsJSONSchema reports whether a provider/model pair previously +// rejected response_format.type=json_schema. The provider URL is part of the +// key because model names are not globally unique across OpenAI-compatible +// gateways. +func (c *openaiClient) modelRejectsJSONSchema(baseURL, model string) bool { + if strings.TrimSpace(model) == "" { + return false + } + key := responseFormatCapabilityKey(baseURL, model) + c.jsonSchemaUnsupportedMu.RLock() + defer c.jsonSchemaUnsupportedMu.RUnlock() + return c.jsonSchemaUnsupported[key] +} + +// rememberNoJSONSchema caches a provider/model capability discovered from a +// definitive response-format rejection, avoiding one failed request for +// every subsequent Wiki chunk. +func (c *openaiClient) rememberNoJSONSchema(baseURL, model string) { + if strings.TrimSpace(model) == "" { + return + } + key := responseFormatCapabilityKey(baseURL, model) + c.jsonSchemaUnsupportedMu.Lock() + defer c.jsonSchemaUnsupportedMu.Unlock() + if c.jsonSchemaUnsupported == nil { + c.jsonSchemaUnsupported = make(map[string]bool) + } + c.jsonSchemaUnsupported[key] = true +} + +func responseFormatCapabilityKey(baseURL, model string) string { + return normalizeOpenAIBaseURL(baseURL) + "\x00" + strings.TrimSpace(model) +} + +func hasJSONSchemaResponseFormat(req openai.ChatCompletionRequest) bool { + return req.ResponseFormat != nil && req.ResponseFormat.Type == openai.ChatCompletionResponseFormatTypeJSONSchema +} + +// downgradeJSONSchemaResponseFormat switches only JSON Schema mode to the +// older JSON object mode. The prompt still requires the exact JSON shape and +// the caller validates the decoded result, so this fallback is safe for +// providers that do not implement structured-output schemas. +func downgradeJSONSchemaResponseFormat(req *openai.ChatCompletionRequest) bool { + if req == nil || !hasJSONSchemaResponseFormat(*req) { + return false + } + req.ResponseFormat = &openai.ChatCompletionResponseFormat{ + Type: openai.ChatCompletionResponseFormatTypeJSONObject, + } + return true +} + // isSamplingParamError reports whether err is a provider 400 rejecting a // sampling param (temperature/top_p/n/penalties) as unsupported/fixed — the // signature of a reasoning model. Matched on message text because the shape @@ -725,6 +864,26 @@ func isSamplingParamError(err error) bool { strings.Contains(msg, "unsupported_value") } +// isResponseFormatUnsupportedError reports the compatibility error returned +// by OpenAI-compatible providers that do not implement JSON Schema output. +// It deliberately requires both the response_format field and an +// unsupported/unavailable phrase so malformed schemas are not silently +// retried with weaker validation. +func isResponseFormatUnsupportedError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + if !strings.Contains(msg, "response_format") && !strings.Contains(msg, "response format") { + return false + } + return strings.Contains(msg, "unavailable") || + strings.Contains(msg, "unsupported") || + strings.Contains(msg, "not supported") || + strings.Contains(msg, "does not support") || + strings.Contains(msg, "not available") +} + // hasCustomSampling reports whether req carries any sampling param that a // reasoning model would reject. Guards the reactive retry so we only re-issue // when stripping the params can actually change the outcome. diff --git a/internal/pkg/llm/client_test.go b/internal/pkg/llm/client_test.go index 3056b1a95..dc4ccbbb6 100644 --- a/internal/pkg/llm/client_test.go +++ b/internal/pkg/llm/client_test.go @@ -170,6 +170,135 @@ func TestChatRoundTrip(t *testing.T) { } } +func TestChatRoundTripCarriesStrictJSONSchema(t *testing.T) { + var responseFormat map[string]any + _, cfg := fakeServer(t, func(w http.ResponseWriter, r *http.Request) { + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read request body: %v", err) + return + } + var body struct { + ResponseFormat map[string]any `json:"response_format"` + } + if err := json.Unmarshal(raw, &body); err != nil { + t.Errorf("decode request body: %v", err) + return + } + responseFormat = body.ResponseFormat + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(sampleChatResponse(`{"ok":true}`, nil)) + }) + + client := newTestClient(t, cfg, nil) + _, err := client.Chat(context.Background(), ChatReq{ + Messages: []Message{{Role: "user", Content: "hi"}}, + ResponseFormat: &ResponseFormat{ + Type: ResponseFormatJSONSchema, + Name: "chunk_batch", + Schema: json.RawMessage(`{"type":"object","properties":{"chunks":{"type":"array"}}}`), + Strict: true, + }, + }) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if responseFormat["type"] != ResponseFormatJSONSchema { + t.Fatalf("response_format.type = %#v, want %q", responseFormat["type"], ResponseFormatJSONSchema) + } + schema, ok := responseFormat["json_schema"].(map[string]any) + if !ok { + t.Fatalf("response_format.json_schema = %#v, want object", responseFormat["json_schema"]) + } + if schema["name"] != "chunk_batch" || schema["strict"] != true { + t.Fatalf("json_schema metadata = %#v, want name and strict", schema) + } + if _, ok := schema["schema"].(map[string]any); !ok { + t.Fatalf("json_schema.schema = %#v, want object", schema["schema"]) + } +} + +func TestChatDowngradesUnsupportedJSONSchema(t *testing.T) { + var ( + calls int + seenTypes []string + rejected bool + ) + _, cfg := fakeServer(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + raw, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read request body: %v", err) + return + } + var body struct { + ResponseFormat struct { + Type string `json:"type"` + } `json:"response_format"` + } + if err := json.Unmarshal(raw, &body); err != nil { + t.Errorf("decode request body: %v", err) + return + } + seenTypes = append(seenTypes, body.ResponseFormat.Type) + if !rejected { + rejected = true + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"This response_format type is unavailable now","type":"invalid_request_error"}}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(sampleChatResponse(`{"ok":true}`, nil)) + }) + client := newTestClient(t, cfg, nil) + request := ChatReq{ + Messages: []Message{{Role: "user", Content: "return JSON"}}, + ResponseFormat: &ResponseFormat{ + Type: ResponseFormatJSONSchema, + Name: "chunk_summary", + Schema: json.RawMessage(`{"type":"object","properties":{"ok":{"type":"boolean"}}}`), + Strict: true, + }, + } + + if _, err := client.Chat(context.Background(), request); err != nil { + t.Fatalf("Chat: %v", err) + } + if calls != 2 { + t.Fatalf("server calls = %d, want 2 (schema rejection + JSON object retry)", calls) + } + if len(seenTypes) != 2 || seenTypes[0] != ResponseFormatJSONSchema || seenTypes[1] != ResponseFormatJSONObject { + t.Fatalf("response format types = %v, want [%s %s]", seenTypes, ResponseFormatJSONSchema, ResponseFormatJSONObject) + } + + // The capability is cached per provider/model, so later Wiki chunks skip + // the known-bad JSON Schema request entirely. + calls = 0 + seenTypes = nil + if _, err := client.Chat(context.Background(), request); err != nil { + t.Fatalf("Chat after capability learning: %v", err) + } + if calls != 1 || len(seenTypes) != 1 || seenTypes[0] != ResponseFormatJSONObject { + t.Fatalf("cached response format = calls %d types %v, want 1 [%s]", calls, seenTypes, ResponseFormatJSONObject) + } +} + +func TestChatRejectsInvalidResponseFormat(t *testing.T) { + client := newTestClient(t, Config{APIKey: "test-key", Model: "gpt-4o"}, nil) + _, err := client.Chat(context.Background(), ChatReq{ + Messages: []Message{{Role: "user", Content: "hi"}}, + ResponseFormat: &ResponseFormat{ + Type: ResponseFormatJSONSchema, + Name: "chunk_batch", + Schema: json.RawMessage(`{"type":`), + }, + }) + if err == nil || !strings.Contains(err.Error(), "response format") { + t.Fatalf("error = %v, want invalid response format", err) + } +} + // TestChatToolCallDecoded verifies that a server response with tool_calls // lands in ChatResp.Assistant.ToolCalls with args preserved verbatim. func TestChatToolCallDecoded(t *testing.T) { diff --git a/internal/pkg/prom/manager_metrics.go b/internal/pkg/prom/manager_metrics.go index 301956630..603e5d9c8 100644 --- a/internal/pkg/prom/manager_metrics.go +++ b/internal/pkg/prom/manager_metrics.go @@ -106,6 +106,20 @@ var ( // status = ok | error | timeout | rate_limited LLMCallsTotal *prometheus.CounterVec + // WikiCompileLLMStageCallsTotal counts logical compiler calls by bounded + // stage (leaf|leaf_batch|leaf_repair|leaf_batch_repair|hierarchy|source_synthesis|planner|planner_repair|canonical_matcher|canonical_matcher_repair|writer|writer_repair) and + // result (ok|error). Provider retry attempts remain in LLMCallsTotal. + WikiCompileLLMStageCallsTotal *prometheus.CounterVec + WikiCompileLLMStageTokensTotal *prometheus.CounterVec + WikiCompileChunkCacheTotal *prometheus.CounterVec + WikiCompileValidationFailuresTotal *prometheus.CounterVec + // WikiCompileStructuredOutputTotal counts strict structured-output decode + // outcomes by stage and bounded result category. + WikiCompileStructuredOutputTotal *prometheus.CounterVec + // WikiCompileRecoveryTotal counts framing normalization, repair, and batch + // degradation outcomes. No response text or identifier is a label. + WikiCompileRecoveryTotal *prometheus.CounterVec + // LLMCallDuration observes provider wall-clock latency (seconds). // status label omitted on the histogram for the same cardinality // reason as HTTPRequestDuration. @@ -140,6 +154,23 @@ var ( // this gauge; values pegged at the cap mean operators should // either bump the cap or expect skipped rows on new fires. InvestigatorInflight prometheus.Gauge + + // LLM Wiki compilation gauges expose the most recently completed source + // compilation. They intentionally carry no tenant/source labels, keeping + // cardinality bounded while still making page explosion and thin-page + // regressions visible. + WikiCompileSourceCount prometheus.Gauge + WikiCompileChunkCount prometheus.Gauge + WikiCompileCandidatePageCount prometheus.Gauge + WikiCompilePublishedPageCount prometheus.Gauge + WikiCompileDroppedPageCount prometheus.Gauge + WikiCompileAvgPageBytes prometheus.Gauge + WikiCompileMedianPageBytes prometheus.Gauge + WikiCompilePagesLT500Bytes prometheus.Gauge + WikiCompilePagesLT1000Bytes prometheus.Gauge + WikiCompileAvgSections prometheus.Gauge + WikiCompileLLMInputTokens prometheus.Gauge + WikiCompileLLMOutputTokens prometheus.Gauge ) // alertEvaluatorBuckets are the histogram buckets for AlertEvaluatorLatency. @@ -260,6 +291,48 @@ func RegisterManagerMetrics(reg *prometheus.Registry, log *slog.Logger) { }, []string{"provider", "model", "kind"}, ) + wikiStageCalls := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_llmwiki_stage_calls_total", + Help: "Logical LLM Wiki compiler calls by bounded stage and result.", + }, + []string{"stage", "result"}, + ) + wikiCache := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_llmwiki_chunk_cache_total", + Help: "LLM Wiki content-addressed chunk cache lookups by result (hit|miss|error).", + }, + []string{"result"}, + ) + wikiStageTokens := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_llmwiki_stage_tokens_total", + Help: "LLM Wiki compiler tokens by bounded stage and direction (input|output).", + }, + []string{"stage", "direction"}, + ) + wikiValidation := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_llmwiki_validation_failures_total", + Help: "LLM Wiki structured-output validation failures by bounded stage.", + }, + []string{"stage"}, + ) + wikiStructuredOutput := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_llm_wiki_structured_output_total", + Help: "LLM Wiki structured-output decode outcomes by bounded stage and result.", + }, + []string{"stage", "result"}, + ) + wikiRecovery := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ongrid_llm_wiki_recovery_total", + Help: "LLM Wiki structured-output recovery outcomes by bounded stage and strategy.", + }, + []string{"stage", "strategy", "result"}, + ) workerSess := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ongrid_chatruntime_worker_sessions", @@ -285,6 +358,28 @@ func RegisterManagerMetrics(reg *prometheus.Registry, log *slog.Logger) { Name: "ongrid_investigator_inflight", Help: "Live RCA investigator workers; capped at investigator.Config.MaxConcurrent.", }) + wikiGauges := []struct { + name string + help string + }{ + {"wiki_compile_source_count", "Sources included in the most recently completed LLM Wiki source compilation."}, + {"wiki_compile_chunk_count", "Chunks included in the most recently completed LLM Wiki source compilation."}, + {"wiki_compile_candidate_page_count", "Topic candidates in the most recently completed LLM Wiki source compilation."}, + {"wiki_compile_published_page_count", "Topic pages accepted in the most recently completed LLM Wiki source compilation."}, + {"wiki_compile_dropped_page_count", "Topic pages rejected in the most recently completed LLM Wiki source compilation."}, + {"wiki_compile_avg_page_bytes", "Average generated Topic page bytes in the most recently completed source compilation."}, + {"wiki_compile_median_page_bytes", "Median generated Topic page bytes in the most recently completed source compilation."}, + {"wiki_compile_pages_lt_500_bytes", "Generated Topic pages below 500 bytes in the most recently completed source compilation."}, + {"wiki_compile_pages_lt_1000_bytes", "Generated Topic pages below 1000 bytes in the most recently completed source compilation."}, + {"wiki_compile_avg_sections", "Average section count of generated Topic pages in the most recently completed source compilation."}, + {"wiki_compile_llm_input_tokens", "LLM input tokens consumed by the most recently completed source compilation."}, + {"wiki_compile_llm_output_tokens", "LLM output tokens consumed by the most recently completed source compilation."}, + } + registeredWikiGauges := make([]prometheus.Gauge, 0, len(wikiGauges)) + for _, definition := range wikiGauges { + gauge := prometheus.NewGauge(prometheus.GaugeOpts{Name: definition.name, Help: definition.help}) + registeredWikiGauges = append(registeredWikiGauges, registerOrExistingGauge(registerer, gauge, log, definition.name)) + } HTTPRequestsTotal = registerOrExistingCounterVec2(registerer, httpReqs, log, "ongrid_http_requests_total") HTTPRequestDuration = registerOrExistingHistogramVec(registerer, httpDur, log) @@ -295,10 +390,28 @@ func RegisterManagerMetrics(reg *prometheus.Registry, log *slog.Logger) { LLMCallsTotal = registerOrExistingCounterVec2(registerer, llmCalls, log, "ongrid_llm_calls_total") LLMCallDuration = registerOrExistingHistogramVec(registerer, llmDur, log) LLMTokensTotal = registerOrExistingCounterVec2(registerer, llmToks, log, "ongrid_llm_router_tokens_total") + WikiCompileLLMStageCallsTotal = registerOrExistingCounterVec2(registerer, wikiStageCalls, log, "ongrid_llmwiki_stage_calls_total") + WikiCompileLLMStageTokensTotal = registerOrExistingCounterVec2(registerer, wikiStageTokens, log, "ongrid_llmwiki_stage_tokens_total") + WikiCompileChunkCacheTotal = registerOrExistingCounterVec2(registerer, wikiCache, log, "ongrid_llmwiki_chunk_cache_total") + WikiCompileValidationFailuresTotal = registerOrExistingCounterVec2(registerer, wikiValidation, log, "ongrid_llmwiki_validation_failures_total") + WikiCompileStructuredOutputTotal = registerOrExistingCounterVec2(registerer, wikiStructuredOutput, log, "ongrid_llm_wiki_structured_output_total") + WikiCompileRecoveryTotal = registerOrExistingCounterVec2(registerer, wikiRecovery, log, "ongrid_llm_wiki_recovery_total") ChatRuntimeWorkerSessions = registerOrExistingGaugeVec(registerer, workerSess, log) AlertEvalTicksTotal = registerOrExistingCounterVec2(registerer, alertTicks, log, "ongrid_alert_eval_ticks_total") EdgeConnections = registerOrExistingGaugeVec(registerer, edgeConns, log) InvestigatorInflight = registerOrExistingGauge(registerer, investigatorInflight, log, "ongrid_investigator_inflight") + WikiCompileSourceCount = registeredWikiGauges[0] + WikiCompileChunkCount = registeredWikiGauges[1] + WikiCompileCandidatePageCount = registeredWikiGauges[2] + WikiCompilePublishedPageCount = registeredWikiGauges[3] + WikiCompileDroppedPageCount = registeredWikiGauges[4] + WikiCompileAvgPageBytes = registeredWikiGauges[5] + WikiCompileMedianPageBytes = registeredWikiGauges[6] + WikiCompilePagesLT500Bytes = registeredWikiGauges[7] + WikiCompilePagesLT1000Bytes = registeredWikiGauges[8] + WikiCompileAvgSections = registeredWikiGauges[9] + WikiCompileLLMInputTokens = registeredWikiGauges[10] + WikiCompileLLMOutputTokens = registeredWikiGauges[11] // Go runtime + process collectors give us goroutines / heap / GC / fd // for free. Idempotent — ignore AlreadyRegisteredError so a second diff --git a/scripts/test-upgrade-data-permissions.sh b/scripts/test-upgrade-data-permissions.sh index 4dd496453..706b26327 100755 --- a/scripts/test-upgrade-data-permissions.sh +++ b/scripts/test-upgrade-data-permissions.sh @@ -43,6 +43,7 @@ expected_owner_for_path() { "$data_dir/loki"|"$data_dir/tempo") printf '10001:10001\n' ;; "$data_dir/grafana") printf '472:472\n' ;; "$data_dir/embeddings"|"$data_dir/skills"|"$data_dir/pages"|"$data_dir/packet-captures"|\ + "$data_dir/llm-wiki"|\ "$data_dir/chat-attachments"|\ "$data_dir/workspace"|"$data_dir/tools") printf '65532:65532\n' ;; *) printf '0:0\n' ;; @@ -90,6 +91,8 @@ grep -Fqx "chown 65532:65532 $data_dir/skills" "$command_log" \ || fail "normal preparation did not set the skills root directory owner" grep -Fqx "chown 65532:65532 $data_dir/packet-captures" "$command_log" \ || fail "normal preparation did not set the packet capture root directory owner" +grep -Fqx "chown 65532:65532 $data_dir/llm-wiki" "$command_log" \ + || fail "normal preparation did not set the LLM Wiki root directory owner" grep -Fqx "chown 65532:65532 $data_dir/chat-attachments" "$command_log" \ || fail "normal preparation did not set the chat attachment root directory owner" if grep -Eq '^(chown|chmod) -R ' "$command_log"; then @@ -135,6 +138,8 @@ grep -Fqx "chown -R 10001:10001 $data_dir/loki" "$command_log" \ || fail "explicit repair did not recursively repair Loki" grep -Fqx "chown -R 65532:65532 $data_dir/skills" "$command_log" \ || fail "explicit repair did not recursively repair skills" +grep -Fqx "chown -R 65532:65532 $data_dir/llm-wiki" "$command_log" \ + || fail "explicit repair did not recursively repair LLM Wiki data" grep -Fqx "chown -R 65532:65532 $data_dir/chat-attachments" "$command_log" \ || fail "explicit repair did not recursively repair chat attachments" @@ -266,7 +271,7 @@ grep -Fq '"$INSTALL_DIR"/ongrid-v*-linux.tar.xz' "$upgrade_script" \ || fail "upgrade.sh does not include universal xz release packages in retention cleanup" grep -Fq '"$INSTALL_DIR"/ongrid-v*-linux-*.tar.xz' "$upgrade_script" \ || fail "upgrade.sh no longer includes legacy architecture-specific xz packages in cleanup" -for persistent_dir in mysql prometheus loki tempo grafana skills pages packet-captures chat-attachments workspace tools; do +for persistent_dir in mysql prometheus loki tempo grafana skills pages packet-captures llm-wiki chat-attachments workspace tools; do if grep -Eq "chown -R .*ONGRID_DATA_DIR/${persistent_dir}" "$upgrade_script"; then fail "upgrade.sh directly recurses through $persistent_dir outside the repair helper" fi diff --git a/web/src/api/knowledge.ts b/web/src/api/knowledge.ts index 146e94172..38a3d5af0 100644 --- a/web/src/api/knowledge.ts +++ b/web/src/api/knowledge.ts @@ -65,7 +65,15 @@ export function isBuiltinVault(repo: Pick): return u.startsWith('builtin://') || u.includes('ongridio/vault'); } -export type SearchHit = { doc: KnowledgeDoc; score: number }; +export type SearchHit = { + doc: KnowledgeDoc; + score: number; + layer?: 'raw' | 'wiki'; + page_type?: WikiPageType; + page_id?: string; + source_version_id?: string; + matched_node?: string; +}; export type PathRow = { path: string; count: number }; @@ -123,11 +131,18 @@ export function moveDoc(id: string, path: string) { export function searchKnowledge( q: string, - opts?: { limit?: number; path?: string; pathPrefix?: string; tags?: string[] }, + opts?: { + limit?: number; + path?: string; + pathPrefix?: string; + tags?: string[]; + mode?: 'hybrid' | 'rag' | 'wiki'; + }, ) { const params = new URLSearchParams(); params.set('q', q); params.set('limit', String(opts?.limit ?? 10)); + if (opts?.mode) params.set('mode', opts.mode); if (opts?.path) params.set('path', opts.path); if (opts?.pathPrefix) params.set('path_prefix', opts.pathPrefix); for (const t of opts?.tags ?? []) params.append('tag', t); @@ -205,6 +220,152 @@ export function deleteRepo(id: number) { return request('DELETE', `/knowledge/repos/${id}`); } +// ----- LLM Wiki ----- +// +// LLM Wiki deliberately has its own source/version lifecycle. It is not an +// alternative view of knowledge_docs: raw files remain inspectable and an +// explicit compile job publishes the derived Wiki pages. +export type WikiPageType = 'generated'; + +export type LLMWikiNode = { + id: string; + parent_id: string; + layer: 'raw' | 'wiki'; + kind: 'file' | 'folder'; + name: string; + relative_path: string; + has_children: boolean; + child_count: number; + document_count: number; + source_id?: string; + page_id?: string; + page_type?: WikiPageType; + status?: string; + updated_at?: string; + content?: string; + metadata?: LLMWikiNodeMetadata; +}; + +export type LLMWikiSourceReference = { + path: string; + node_id?: string; + version_id?: string; +}; + +export type LLMWikiReference = { + title: string; + path: string; + node_id?: string; + page_id?: string; +}; + +export type LLMWikiNodeMetadata = { + source_files?: LLMWikiSourceReference[]; + source_file?: string; + source_node_id?: string; + source_version?: string; + wiki_files?: LLMWikiReference[]; + page_type?: WikiPageType; + sha256?: string; +}; + +export type LLMWikiSource = { + id: string; + source_key: string; + source_type: string; + raw_path: string; + status: 'pending' | 'succeeded' | 'stale' | 'failed'; + current_version_id?: string; + updated_at: string; +}; + +export type LLMWikiJob = { + id: string; + status: 'pending' | 'running' | 'succeeded' | 'skipped' | 'failed' | 'cancelled'; + stage: string; + error?: string; + created_at: string; + updated_at: string; +}; + +// The legacy Knowledge endpoints return their payload directly; the newer +// Wiki handler follows the standard { code, message, data } envelope. +async function wikiRequest(method: 'GET' | 'POST' | 'DELETE', path: string, body?: unknown): Promise { + const result = await request<{ data: T }>(method, path, body); + return result.data; +} + +async function wikiFetch(path: string, init: RequestInit = {}): Promise { + const headers = new Headers(init.headers); + const token = getToken(); + if (token) headers.set('Authorization', `Bearer ${token}`); + const res = await fetch(`/api/v1${path}`, { ...init, headers }); + if (res.ok) return res; + + let message = `request failed (${res.status})`; + try { + const body = await res.json(); + message = body.message || body.error || message; + } catch { + // Keep the HTTP fallback when a proxy returns a non-JSON error. + } + throw new Error(message); +} + +async function wikiBlobRequest(path: string): Promise { + return (await wikiFetch(path)).blob(); +} + +export function listLLMWikiTree(layer: LLMWikiNode['layer'], parentID?: string) { + const q = new URLSearchParams({ layer }); + if (parentID) q.set('parent_id', parentID); + return wikiRequest<{ items: LLMWikiNode[]; total: number; document_count: number }>( + 'GET', + `/knowledge/llm-wiki/tree?${q.toString()}`, + ); +} + +export function getLLMWikiNode(id: string) { + return wikiRequest('GET', `/knowledge/llm-wiki/nodes/${encodeURIComponent(id)}`); +} + +export function deleteLLMWikiNode(id: string) { + return wikiRequest<{ deleted: boolean }>('DELETE', `/knowledge/llm-wiki/nodes/${encodeURIComponent(id)}`); +} + +export async function getLLMWikiNodePreview(id: string): Promise { + return wikiBlobRequest(`/knowledge/llm-wiki/nodes/${encodeURIComponent(id)}/preview`); +} + +export function listLLMWikiSources() { + return wikiRequest<{ items: LLMWikiSource[]; total: number }>('GET', '/knowledge/llm-wiki/sources'); +} + +export function listLLMWikiJobs() { + return wikiRequest<{ items: LLMWikiJob[]; total: number }>('GET', '/knowledge/llm-wiki/jobs'); +} + +export function compileLLMWiki(force = false, sourceIds?: string[]) { + return wikiRequest('POST', '/knowledge/llm-wiki/compile', { + force, + source_ids: sourceIds, + }); +} + +export function retryLLMWikiJob(id: string) { + return wikiRequest('POST', `/knowledge/llm-wiki/jobs/${id}/retry`, {}); +} + +export function cancelLLMWikiJob(id: string) { + return wikiRequest('POST', `/knowledge/llm-wiki/jobs/${id}/cancel`, {}); +} + +export type LLMWikiSyncResult = { created: number; updated: number; unchanged: number; deleted: number; total: number }; + +export function syncLLMWikiSources() { + return wikiRequest('POST', '/knowledge/llm-wiki/sync', {}); +} + // ----- SSH identities ----- // // One row = one stored SSH private key + the host patterns it's diff --git a/web/src/features/llm-wiki/LLMWikiJobs.tsx b/web/src/features/llm-wiki/LLMWikiJobs.tsx new file mode 100644 index 000000000..6e9d22a89 --- /dev/null +++ b/web/src/features/llm-wiki/LLMWikiJobs.tsx @@ -0,0 +1,94 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { Loader2, RefreshCw, RotateCcw, X } from 'lucide-react'; +import { ApiError } from '@/api/client'; +import { Card } from '@/components/ui'; +import { cn } from '@/lib/cn'; +import { useI18n } from '@/i18n/locale'; +import { cancelLLMWikiJob, listLLMWikiJobs, retryLLMWikiJob } from './api'; +import { type LLMWikiJob } from './types'; + +export function LLMWikiJobs({ refreshKey, onPublished }: { refreshKey: number; onPublished?: () => void }) { + const { tr } = useI18n(); + const [jobs, setJobs] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [actionID, setActionID] = useState(null); + const activeJobIDs = useRef>(new Set()); + const onPublishedRef = useRef(onPublished); + useEffect(() => { onPublishedRef.current = onPublished; }, [onPublished]); + + const loadJobs = useCallback(async (silent = false) => { + if (!silent) setLoading(true); + try { + const result = await listLLMWikiJobs(); + const nextJobs = result.items ?? []; + const published = nextJobs.some((job) => activeJobIDs.current.has(job.id) && (job.status === 'succeeded' || job.status === 'skipped')); + activeJobIDs.current = new Set(nextJobs.filter((job) => job.status === 'pending' || job.status === 'running').map((job) => job.id)); + setJobs(nextJobs); + if (published) onPublishedRef.current?.(); + setError(null); + } catch (e) { + setError(e instanceof ApiError ? e.message : (e as Error).message); + } finally { + if (!silent) setLoading(false); + } + }, []); + + useEffect(() => { void loadJobs(); }, [loadJobs, refreshKey]); + + useEffect(() => { + if (!jobs.some((job) => job.status === 'pending' || job.status === 'running')) return undefined; + const timer = window.setInterval(() => void loadJobs(true), 3000); + return () => window.clearInterval(timer); + }, [jobs, loadJobs]); + + const runAction = async (id: string, action: (jobID: string) => Promise) => { + setActionID(id); + setError(null); + try { + await action(id); + await loadJobs(); + } catch (e) { + setError(e instanceof ApiError ? e.message : (e as Error).message); + } finally { + setActionID(null); + } + }; + + const visibleJobs = jobs.filter((job) => job.status === 'pending' || job.status === 'running' || job.status === 'failed'); + if (!loading && visibleJobs.length === 0 && !error) return null; + + return ( + +
+
{tr('编译任务', 'Compile jobs')}
+ +
+ {error &&
{error}
} + {loading && jobs.length === 0 ?
{tr('加载任务中…', 'Loading jobs…')}
: ( +
+ {visibleJobs.map((job) => { + const active = job.status === 'pending' || job.status === 'running'; + const failed = job.status === 'failed'; + const statusLabel = job.status === 'pending' ? tr('排队中', 'Pending') : job.status === 'running' ? tr('编译中', 'Running') : tr('失败', 'Failed'); + return ( +
+
+
{statusLabel}{tr('阶段', 'Stage')}: {job.stage || 'queued'}
+
{tr('全部来源', 'All sources')}
+ {job.error &&
{job.error}
} +
+
+ + {failed && } +
+
+ ); + })} +
+ )} + + ); +} diff --git a/web/src/features/llm-wiki/LLMWikiPane.tsx b/web/src/features/llm-wiki/LLMWikiPane.tsx new file mode 100644 index 000000000..11d3ba79b --- /dev/null +++ b/web/src/features/llm-wiki/LLMWikiPane.tsx @@ -0,0 +1,265 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { ArrowLeft, BookOpen, Eye, FolderSync, ListTodo, Loader2, Play, RefreshCw, Trash2 } from 'lucide-react'; +import { ApiError } from '@/api/client'; +import { localizedPath } from '@/api/knowledge'; +import { Modal } from '@/components/Modal'; +import { Button, Card, EmptyState } from '@/components/ui'; +import { Hint } from '@/components/ui/Tooltip'; +import { cn } from '@/lib/cn'; +import { useI18n } from '@/i18n/locale'; +import { compileLLMWiki, deleteLLMWikiNode, listLLMWikiTree, syncLLMWikiSources } from './api'; +import { LLMWikiJobs } from './LLMWikiJobs'; +import { LLMWikiTree } from './LLMWikiTree'; +import { LLMWikiFileViewer } from './LLMWikiViewer'; +import { type LLMWikiNode } from './types'; + +type WikiLayer = LLMWikiNode['layer']; + +export function LLMWikiNavigation({ active, onOpen, onDocumentCount, onChanged, onJobsChanged, activeDirectory, activeLayer, jobsRefreshKey, refreshKey }: { active: boolean; onOpen: (directory: LLMWikiNode | null, layer: 'all' | WikiLayer) => void; onDocumentCount: (count: number) => void; onChanged: () => void; onJobsChanged: () => void; activeDirectory: LLMWikiNode | null; activeLayer: 'all' | WikiLayer; jobsRefreshKey: number; refreshKey: number }) { + const { tr } = useI18n(); + const [nodes, setNodes] = useState([]); + const [documentCounts, setDocumentCounts] = useState>>({}); + const [showJobs, setShowJobs] = useState(false); + const [syncing, setSyncing] = useState(false); + const [compiling, setCompiling] = useState(false); + const [syncError, setSyncError] = useState(null); + const load = useCallback(async () => { + try { + const results = await Promise.all([listLLMWikiTree('raw'), listLLMWikiTree('wiki')]); + setNodes(results.flatMap((result) => result.items ?? [])); + const counts = results.map((result) => { + const fileCount = (result.items ?? []).filter((node) => node.kind === 'file').length; + return fileCount > 0 ? fileCount : (result.document_count ?? 0); + }); + setDocumentCounts({ raw: counts[0], wiki: counts[1] }); + onDocumentCount(counts[1]); + } catch { + // Wiki is optional; the traditional Knowledge page remains usable when + // its storage or worker is temporarily unavailable. + } + }, [onDocumentCount]); + useEffect(() => { void load(); }, [load, refreshKey]); + const rootSelected = active && activeDirectory === null && activeLayer === 'all'; + return ( +
+
+ +
+ + + + + + + + + + + + +
+
+ {syncError &&
{syncError}
} +
+ onOpen(dir, ly)} /> +
+ {showJobs && } +
+ ); +} + +export function LLMWikiPane({ onExit, hideSidebar, initialDirectory, initialLayer, externalRefreshKey = 0 }: { onExit?: () => void; hideSidebar?: boolean; initialDirectory?: LLMWikiNode | null; initialLayer?: 'all' | WikiLayer; externalRefreshKey?: number }) { + const { tr } = useI18n(); + const [nodes, setNodes] = useState([]); + const [documentCounts, setDocumentCounts] = useState>>({}); + const [directory, setDirectory] = useState(initialDirectory ?? null); + const [layer, setLayer] = useState<'all' | WikiLayer>(initialLayer ?? 'all'); + const [refreshKey, setRefreshKey] = useState(0); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [error, setError] = useState(null); + const [viewing, setViewing] = useState(null); + const [deleting, setDeleting] = useState(null); + const [compilingIds, setCompilingIds] = useState>(new Set()); + + useEffect(() => { + setDirectory(initialDirectory ?? null); + setLayer(initialLayer ?? 'all'); + }, [initialDirectory, initialLayer]); + + const load = useCallback(async () => { + setLoading(true); + try { + const results = await Promise.all([ + listLLMWikiTree('raw'), + listLLMWikiTree('wiki'), + ]); + setNodes(results.flatMap((result) => result.items ?? [])); + setDocumentCounts({ raw: results[0].document_count, wiki: results[1].document_count }); + setError(null); + } catch (e) { + setError(e instanceof ApiError ? e.message : (e as Error).message); + } finally { + setLoading(false); + setRefreshing(false); + } + }, []); + + useEffect(() => { void load(); }, [load, refreshKey, externalRefreshKey]); + + const refresh = useCallback(() => { + setRefreshing(true); + setRefreshKey((key) => key + 1); + }, []); + + const compileFile = useCallback(async (file: LLMWikiNode) => { + if (!file.source_id) return; + setCompilingIds((prev) => new Set(prev).add(file.id)); + try { + await compileLLMWiki(false, [file.source_id!]); + refresh(); + } catch { + // compilation triggered; status shown in jobs panel + } finally { + setCompilingIds((prev) => { + const next = new Set(prev); + next.delete(file.id); + return next; + }); + } + }, [refresh]); + + const files = useMemo(() => nodes.filter((node) => node.kind === 'file'), [nodes]); + // 收集当前目录及全部子目录 ID，目录视图据此展示整棵子树里的文件。 + const descendantIds = useMemo(() => { + if (!directory) return null; + const folders = nodes.filter((node) => node.kind === 'folder' && node.layer === directory.layer); + const byParent = new Map(); + for (const f of folders) { + const list = byParent.get(f.parent_id) ?? []; + list.push(f); + byParent.set(f.parent_id, list); + } + const ids = new Set([directory.id]); + const stack = [directory.id]; + while (stack.length > 0) { + const pid = stack.pop()!; + for (const child of byParent.get(pid) ?? []) { + if (!ids.has(child.id)) { + ids.add(child.id); + stack.push(child.id); + } + } + } + return ids; + }, [nodes, directory]); + const visibleFiles = useMemo(() => { + if (directory && descendantIds) return files.filter((file) => file.layer === directory.layer && descendantIds.has(file.parent_id)); + return layer === 'all' ? files : files.filter((file) => file.layer === layer); + }, [directory, descendantIds, files, layer]); + + // 外层知识库页面已经提供统一文件树时，这里只渲染文件列表与弹窗。 + if (hideSidebar) { + return ( + <> + + {viewing && setViewing(null)} onOpenFile={setViewing} onDelete={() => { setViewing(null); setDeleting(viewing); }} />} + {deleting && setDeleting(null)} onDone={() => { setDeleting(null); refresh(); }} />} + + ); + } + + return ( +
+ +
+ +
+ {viewing && setViewing(null)} onOpenFile={setViewing} onDelete={() => { setViewing(null); setDeleting(viewing); }} />} + {deleting && setDeleting(null)} onDone={() => { setDeleting(null); refresh(); }} />} +
+ ); +} + +function LLMWikiFileList({ + files, + directory, + loading, + error, + onViewFile, + onDeleteFile, + onCompileFile, +}: { + files: LLMWikiNode[]; + directory: LLMWikiNode | null; + loading: boolean; + error: string | null; + onViewFile: (file: LLMWikiNode) => void; + onDeleteFile: (file: LLMWikiNode) => void; + onCompileFile: (file: LLMWikiNode) => void; +}) { + const { tr } = useI18n(); + if (loading) return
{tr('加载中…', 'Loading…')}
; + if (error) return
{error}
; + if (files.length === 0) return ; + return
{files.map((file) => )}
; +} + +function WikiFileCard({ file, onView, onDelete, onCompile }: { file: LLMWikiNode; onView: (file: LLMWikiNode) => void; onDelete: (file: LLMWikiNode) => void; onCompile: (file: LLMWikiNode) => void }) { + const { tr } = useI18n(); + const label = file.layer === 'raw' ? 'Raw' : 'Wiki'; + return onView(file)} className="flex cursor-pointer flex-col py-2.5 transition-colors hover:bg-zinc-800/40">
{file.layer === 'raw' ? file.name : file.name.replace(/\.md$/i, '')}
{label}{file.layer}
{localizedPath(file.relative_path)}
{file.layer === 'raw' && }{file.layer === 'raw' && }
; +} + +function DeleteLLMWikiFileDialog({ file, onClose, onDone }: { file: LLMWikiNode; onClose: () => void; onDone: () => void }) { + const { tr } = useI18n(); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const submit = async () => { + setSubmitting(true); + setError(null); + try { + await deleteLLMWikiNode(file.id); + onDone(); + } catch (e) { + setError(e instanceof ApiError ? e.message : (e as Error).message); + } finally { + setSubmitting(false); + } + }; + return }>
{error &&
{error}
}

{tr('删除 Raw 文件后，当前 Wiki 构建与搜索索引也会失效，需要重新编译。此操作不可恢复。', 'Deleting this raw file also invalidates the current Wiki build and search index. A new compile is required. This cannot be undone.')}

{localizedPath(file.relative_path)}

; +} diff --git a/web/src/features/llm-wiki/LLMWikiTree.test.tsx b/web/src/features/llm-wiki/LLMWikiTree.test.tsx new file mode 100644 index 000000000..983a77907 --- /dev/null +++ b/web/src/features/llm-wiki/LLMWikiTree.test.tsx @@ -0,0 +1,84 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, expect, it, vi } from 'vitest'; +import { LLMWikiTree } from './LLMWikiTree'; +import type { LLMWikiNode } from './types'; + +describe('LLMWikiTree', () => { + const nodes: LLMWikiNode[] = [ + { + id: 'raw-network', + parent_id: '', + layer: 'raw', + kind: 'folder', + name: 'network', + relative_path: 'network', + has_children: false, + child_count: 0, + document_count: 1, + }, + ]; + + it('localizes source directory names without changing file names', async () => { + render( + , + ); + + await userEvent.click(screen.getByRole('button', { name: /^原始来源/ })); + + expect(screen.getByText('网络')).toBeInTheDocument(); + expect(screen.queryByText('Network')).not.toBeInTheDocument(); + }); + + it('starts collapsed and toggles children from the folder row', async () => { + render( + , + ); + + const rawRoot = screen.getByRole('button', { name: /^原始来源/ }); + expect(rawRoot).toHaveAttribute('aria-expanded', 'false'); + expect(screen.queryByText('网络')).not.toBeInTheDocument(); + + await userEvent.click(rawRoot); + expect(rawRoot).toHaveAttribute('aria-expanded', 'true'); + expect(screen.getByText('网络')).toBeInTheDocument(); + + await userEvent.click(rawRoot); + expect(rawRoot).toHaveAttribute('aria-expanded', 'false'); + expect(screen.queryByText('网络')).not.toBeInTheDocument(); + }); + + // FolderNode (pages/Knowledge.tsx) puts its folder icon 42px into the row, + // behind a 6px inset, a 22px expand chevron, a 4px gap and the icon button's + // 10px padding. The Wiki roots started at 0px, so 原始来源/生成页面 sat flush + // left under the section header while the folders above them were inset. + it('insets rows like the knowledge-base tree so the folder rows line up', async () => { + render( + , + ); + + const rawRoot = screen.getByRole('button', { name: /^原始来源/ }); + expect(rawRoot).toHaveStyle({ paddingLeft: '42px' }); + + await userEvent.click(rawRoot); + expect(screen.getByRole('button', { name: /^网络/ })).toHaveStyle({ paddingLeft: '54px' }); + }); +}); diff --git a/web/src/features/llm-wiki/LLMWikiTree.tsx b/web/src/features/llm-wiki/LLMWikiTree.tsx new file mode 100644 index 000000000..d1b42ec8a --- /dev/null +++ b/web/src/features/llm-wiki/LLMWikiTree.tsx @@ -0,0 +1,152 @@ +import { useMemo, useState } from 'react'; +import { Folder, FolderOpen } from 'lucide-react'; +import { cn } from '@/lib/cn'; +import { tr as trInline } from '@/i18n/locale'; +import { localizedPathSegment } from '@/api/knowledge'; +import { type LLMWikiNode } from './types'; + +type WikiLayer = LLMWikiNode['layer']; + +type Props = { + nodes: LLMWikiNode[]; + layers?: WikiLayer[]; + documentCounts: Partial>; + activeDirectory: LLMWikiNode | null; + activeLayer: 'all' | WikiLayer; + onSelectDirectory: (node: LLMWikiNode | null, layer: 'all' | WikiLayer) => void; +}; + +const layerOrder: WikiLayer[] = ['raw', 'wiki']; + +export function LLMWikiTree({ nodes, layers = layerOrder, documentCounts, activeDirectory, activeLayer, onSelectDirectory }: Props) { + const { byParent, roots } = useMemo(() => { + const byParent = new Map(); + for (const node of nodes) { + if (node.kind !== 'folder') continue; + const children = byParent.get(node.parent_id) ?? []; + children.push(node); + byParent.set(node.parent_id, children); + } + for (const children of byParent.values()) { + children.sort((left, right) => left.name.localeCompare(right.name)); + } + const roots = layers.map((layer) => ({ + id: `${layer}:root`, + parent_id: '', + layer, + kind: 'folder' as const, + name: layerLabel(layer), + relative_path: '', + has_children: (nodes.some((node) => node.layer === layer && node.kind === 'folder')), + child_count: byParent.get('')?.filter((node) => node.layer === layer).length ?? 0, + document_count: documentCounts[layer] ?? nodes.filter((node) => node.layer === layer && node.kind === 'file').length, + })); + return { byParent, roots }; + }, [documentCounts, layers, nodes]); + + const [expanded, setExpanded] = useState>(() => new Set()); + + const toggle = (id: string) => { + setExpanded((current) => { + const next = new Set(current); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }; + + return ( +
+ {roots.map((root) => ( + node.layer === root.layer) ?? []} + byParent={byParent} + expanded={expanded} + activeDirectory={activeDirectory} + activeLayer={activeLayer} + onToggle={toggle} + onSelectDirectory={onSelectDirectory} + onRootSelect={() => onSelectDirectory(null, root.layer)} + depth={0} + /> + ))} +
+ ); +} + +function WikiTreeBranch({ + node, + children, + byParent, + expanded, + activeDirectory, + activeLayer, + onToggle, + onSelectDirectory, + onRootSelect, + depth, +}: { + node: LLMWikiNode; + children: LLMWikiNode[]; + byParent: Map; + expanded: Set; + activeDirectory: LLMWikiNode | null; + activeLayer: 'all' | WikiLayer; + onToggle: (id: string) => void; + onSelectDirectory: (node: LLMWikiNode | null, layer: 'all' | WikiLayer) => void; + onRootSelect: () => void; + depth: number; +}) { + const isOpen = expanded.has(node.id); + const isRoot = node.relative_path === ''; + const isSelected = isRoot + ? activeLayer === node.layer && !activeDirectory + : activeDirectory?.id === node.id; + return ( +
+
+ +
+ {isOpen && children.map((child) => ( + + ))} +
+ ); +} + +function layerLabel(layer: WikiLayer): string { + if (layer === 'raw') return trInline('原始来源', 'Raw sources'); + return trInline('生成页面', 'Generated pages'); +} diff --git a/web/src/features/llm-wiki/LLMWikiViewer.tsx b/web/src/features/llm-wiki/LLMWikiViewer.tsx new file mode 100644 index 000000000..d89d0f948 --- /dev/null +++ b/web/src/features/llm-wiki/LLMWikiViewer.tsx @@ -0,0 +1,177 @@ +import { useEffect, useMemo, useState } from 'react'; +import { ArrowUpRight, Trash2 } from 'lucide-react'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import { Modal } from '@/components/Modal'; +import { localizedPath } from '@/api/knowledge'; +import { fullDateTime } from '@/lib/format'; +import { splitFrontmatter } from '@/lib/frontmatter'; +import { useI18n } from '@/i18n/locale'; +import { getLLMWikiNode, getLLMWikiNodePreview } from './api'; +import { type LLMWikiNode } from './types'; + +type RawPreview = + | { kind: 'pdf'; url: string } + | { kind: 'docx'; text: string }; + +type WikiReference = { + path: string; + node_id?: string; + title?: string; +}; + +export function LLMWikiFileViewer({ + file, + onClose, + onOpenFile, + onDelete, +}: { + file: LLMWikiNode; + onClose: () => void; + onOpenFile: (file: LLMWikiNode) => void; + onDelete: () => void; +}) { + const { tr } = useI18n(); + const [detail, setDetail] = useStateWithReset(null, file.id); + const [error, setError] = useStateWithReset(null, file.id); + const [preview, setPreview] = useStateWithReset(null, file.id); + const [previewError, setPreviewError] = useStateWithReset(null, file.id); + const previewKind = rawPreviewKind(file); + + useEffect(() => { + let cancelled = false; + void getLLMWikiNode(file.id) + .then((node) => { if (!cancelled) setDetail(node); }) + .catch((reason: unknown) => { if (!cancelled) setError((reason as Error).message); }); + return () => { cancelled = true; }; + }, [file.id, setDetail, setError]); + + useEffect(() => { + let cancelled = false; + let objectURL: string | null = null; + if (!previewKind) return () => { cancelled = true; }; + void getLLMWikiNodePreview(file.id) + .then(async (blob) => { + if (cancelled) return; + if (previewKind === 'pdf') { + objectURL = URL.createObjectURL(blob); + setPreview({ kind: 'pdf', url: objectURL }); + } else { + setPreview({ kind: 'docx', text: await blob.text() }); + } + }) + .catch((reason: unknown) => { if (!cancelled) setPreviewError((reason as Error).message); }); + return () => { + cancelled = true; + if (objectURL) URL.revokeObjectURL(objectURL); + }; + }, [file.id, previewKind, setPreview, setPreviewError]); + + return ( + + {file.layer === 'raw' && ( + + )} + + + } + > +
+
{localizedPath(file.relative_path)}
+ {detail &&
{tr('更新时间', 'Updated')}：{fullDateTime(detail.updated_at)}
} + {detail?.layer === 'raw' ? ( + + ) : ( + detail &&
+ +
+ )} +
+ {error ?
{error}
: detail ? previewKind ? : :
{tr('加载中…', 'Loading…')}
} +
+
+ + ); +} + +// This tiny hook keeps the viewer reset logic local to the feature. It avoids +// showing the previous file while the detail request for a linked file runs. +function useStateWithReset(initial: T, resetKey: string): [T, (value: T) => void] { + const [value, setValue] = useState(initial); + useEffect(() => setValue(initial), [initial, resetKey]); + return [value, setValue]; +} + +function rawPreviewKind(file: LLMWikiNode): 'pdf' | 'docx' | null { + if (file.layer !== 'raw') return null; + const extension = file.relative_path.toLowerCase().split('.').pop(); + return extension === 'pdf' || extension === 'docx' ? extension : null; +} + +function RawPreviewView({ kind, preview, error, fileName, tr }: { kind: 'pdf' | 'docx'; preview: RawPreview | null; error: string | null; fileName: string; tr: (zh: string, en: string) => string }) { + if (error) return
{error}
; + if (!preview) return
{tr('加载预览…', 'Loading preview…')}
; + if (kind === 'pdf' && preview.kind === 'pdf') return