From cfc0c2311a542ccca7a076fc38295c79df195863 Mon Sep 17 00:00:00 2001 From: Krasimir Kralev <263465593+KrasimirKralev@users.noreply.github.com> Date: Fri, 18 Sep 2026 13:31:18 +0300 Subject: [PATCH 1/2] feat: index memory in the ClawBox AI cloud on the Hermes edition too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Memory Shard offered the cloud embedder on one SKU only. Where ClawBox itself is the indexer (no OpenClaw), `memory-index-local.ts` accepted a loopback endpoint and nothing else, and the provider route answered `cloudSupported: false, cloudReason: "edition"` — so that box downloaded 639 MB and embedded on its own hardware while the subscription it pays for sat unused. The fence is what made that true, and it did not come down: it now has exactly two gates instead of one. `src/lib/memory-embedder.ts` resolves the embedder per pass, per search and per status read, and every request is checked at the moment the socket opens — the loopback proxy in front of the model on this device, or the ClawBox AI embeddings endpoint the image was built with, which is the box's own account. Anything else is refused, on both arms. The choice is a WORD in ClawBox's own store (`memory_shard_embedder`), never an address and never a copy of the credential: the endpoint comes from the environment and the bearer from the credential store, per request. That is the rule `clawai_cloud_embeddings` is already written around — a restored backup or a hand-edited data/config.json must not be able to redirect where the owner's documents go. The default is the cloud, on both editions. A box nobody has pinned follows the same cloud-defaults verdict the OpenClaw arm follows — linked, a paid plan, a probe the cloud embedder answered — so the model on the box is the opt-in rather than the starting point, and a box that links a subscription later is promoted by the applier that already does this for speech, with the full pass the move needs. `readEmbeddingPlacement` is the one reader both editions use, and its `recorded` half is what stops that promotion running again at every boot. The identity stamp carries provider|model|baseUrl, so a switch reports `mismatched` and the card's existing amber "Run a full reindex" and the full pass behind every switch rebuild the index. It is never stamped over an index that holds vectors: that would report `valid` for rows every query misses. The status the card draws follows the embedder as well — the chip reads ClawBox AI cloud, the model name is the cloud model, and semantic search is available there without the GGUF on disk. No new UI and no new locale keys: the wizard's picker and the settings card light up from the route. (cherry picked from commit 03026df0f88f5ad2fd222ace9b1cd26fbceb654d) --- CLAUDE.md | 2 +- .../clawkeep/memory/provider/route.ts | 61 +-- .../setup-api/clawkeep/memory/search/route.ts | 12 +- src/components/MemoryShardEmbedderCard.tsx | 7 + src/components/MemoryShardWizard.tsx | 37 +- src/lib/clawai-cloud-defaults-state.ts | 24 +- src/lib/clawai-cloud-defaults.ts | 107 ++++- src/lib/clawai-cloud-embeddings.ts | 99 +++- src/lib/clawkeep-memory.ts | 10 +- src/lib/memory-embedder.ts | 249 ++++++++++ src/lib/memory-index-local.ts | 450 ++++++++++++++--- src/lib/memory-shard-state.ts | 51 +- src/lib/memory-shard.ts | 104 +++- .../components/memory-shard-wizard.test.tsx | 40 +- .../routes/memory-shard-provider.test.ts | 64 ++- src/tests/routes/memory-shard-search.test.ts | 49 +- .../unit/clawai-cloud-defaults-apply.test.ts | 121 ++++- .../unit/clawai-cloud-embeddings.test.ts | 38 +- src/tests/unit/harness-credentials.test.ts | 60 +++ src/tests/unit/memory-embedder.test.ts | 185 +++++++ src/tests/unit/memory-index-local.test.ts | 452 +++++++++++++++++- .../unit/memory-shard-embedding-keys.test.ts | 183 ++++++- 22 files changed, 2188 insertions(+), 217 deletions(-) create mode 100644 src/lib/memory-embedder.ts create mode 100644 src/tests/unit/memory-embedder.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index f1f7eb655..d2bfed9b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -176,7 +176,7 @@ Handles three concerns (plus one carve-out: `UI_LANGUAGE_READ` — a GET of `/se - **`CodingAgentApp.tsx`** — the Coding Agent desktop app: harness readiness, the Test-harness button, the owner's projects (git-initialised folders and code projects), recent runs (this icon used to open an interactive `claude-ds` terminal) with summaries, and a "New app" wizard that composes one message and hands it to the mascot chat (`CHAT_MESSAGE_EVENT`) — the assistant carries on from there. Its Settings button opens the settings page embedded in the app itself; the home page leads with a Create New Project wizard (handed to the chat), and each project expands into its own page with a git block (branch, commits, origin — GET /setup-api/coding-agent/git?projectId=) plus that project's runs. A project row and its page carry the picture the box drew for the folder while a run worked in it (`iconUrl`, `InstalledAppIcon`), and fall back to the lettered placeholder until there is one — which is most of a fresh box. Every page but home carries ONE breadcrumb row (`CodingAgentBreadcrumb.tsx`: an arrow that goes up a level — it keeps the old Back buttons' test ids `coding-agent-settings-back` / `-project-back` / `-run-back` — then Projects › project › run, earlier crumbs as buttons, the current one `aria-current="page"`); it replaced three differently styled Back pills, and the header's Settings button no longer doubles as Back. The home list is lean on purpose — icon, name, last commit, chips, a chevron — the folder path and its copy button live on the project page. A project's page is a title row, ONE git line (branch · commits · newest commit · origin or "not on GitHub yet" · Backup) and the workspace (`CodingProjectWorkspace.tsx`): a **Files** tab — the folder as a tree read through `GET /setup-api/coding-agent/tree?projectId|directory&path=` (`src/lib/coding-project-tree.ts`, rooted at the PROJECT because a project can live anywhere the owner pointed the agent at, `.git` skipped, links not followed, `isProtectedFilePath` refused, every refusal a 404) with the shared editor (`&file=`, capped, binary flagged; `PUT coding-agent/tree { projectId|directory, file, content }` saves the OWNER's edit — a session cookie, never the MCP bearer, since the agent edits with its own tools — over a file that is already there through `writeProjectFile`, the same descriptor-by-descriptor walk as the read with O_WRONLY|O_NOFOLLOW at the end and no O_CREAT, capped at the read cap so a file opened whole is saved whole; a cut file is shown read-only because saving it would lose its tail, a save while a run works in the folder is allowed with an amber note that the run's next commit takes the edit along, and a second file tapped over unsaved changes is asked about first) and the Files app's icons from `file-icons.tsx`, plus **Open in Files** (`coding-agent-open-in-files`, folder projects only, not on the standalone page) — and a **Changes** tab: `GET coding-agent/git?…&changes=1[&ref=]` (`gitChanges`/`gitLog` in `coding-git.ts`: the working tree via `status --porcelain=v2` + `diff --numstat`, untracked files counted by hand, or ONE commit's changes via `diff-tree`, because a run commits its work when it settles and leaves the tree clean) with a picker over the recent commits, and `&diff=` for one file's unified diff drawn by `DiffView` (text only, header lines folded). A ref reaches git only as a hex sha or HEAD (`isSafeGitRef`), a file only as a normalised in-project path (`safeProjectRelativePath`); the Changes tab reads nothing until it is opened and polls every 5 s while a run works in the folder. A run has its OWN PAGE (`view.face === "run"`, `data-testid="coding-agent-run-page"`) instead of a row that unfolds: status, task, project chip, controls (pause/resume/start, stop or discard, terminal, backup), a figures grid (steps, files, duration, tokens, helpers by type, commit, denials, models), helpers at work, the plan, the error, the summary rendered as Markdown, the files changed, the evidence folder (a generated clip gets a player rather than a download link — a download is not how you check what a narration says), the denied actions and an activity log. A run held to a deliverable also carries a card naming what it had to leave behind, the verdict on it and how much of its attempt budget went on it — on a run that GAVE UP that card is the only thing on the page that says what is actually absent, since the chip says only "Gave up" and the error repeats the reason — with Resume in the ordinary row of run controls beside it. A refused action's sentence (a Resume the folder is gone for, a Stop that failed) is drawn ABOVE whichever face is up (`coding-agent-action-error`, scrolled into view), never under the page — at the bottom of a run's page it sat off screen and a refused Resume looked like nothing happened; the sidebar's recent runs list every HELD run (running, paused, drafted) ahead of the settled ones whatever its age, so a run waiting on the owner is never out of sight; and the Settings page's Clear history also drops a paused run or a draft whose folder is GONE (neither can be resumed or started, and kept it was immortal). A settled run whose `leftover` is true — something it started is still listening — says so in an amber card with the Kill button beside it, because the device leaving an app's server up is a documented pattern and killing it unasked would break the pattern, while saying nothing would make it a leak nobody can see. Opened from a row's Details button, a review chip, or from outside through `dispatchOpenCodingRun(runId)` / `handoffCodingRun` + `OPEN_CODING_RUN_EVENT` in `ui-events.ts` (the desktop's finish card and the chat's run card use it). - **`CodingAgentSettingsPanel.tsx`** — the Coding Agent app's embedded Settings page: the owner's switch for delegated Claude Code runs, the default project folder, effort, the per-run ceilings, the automatic review pass switch (POST /setup-api/coding-agent/enable `{ reviewPass }`), the attempts a run with a DELIVERABLE gets at it (`{ completionAttempts }`, 1–6, beside the pull-request controls because the commonest deliverable IS the pull request those switches ask for, and drawn only when the server answers with the field), the two media switches under it — `{ generateImages }` (which also draws the project's own icon and favicon) and `{ generateAudio }`, both rendered `?? true` because an older server answers with neither and both are on by default, and the real-browser switch beside them (`{ realBrowser }`, `?? true` for the same reason, with a hint saying a run drives the screen the owner can watch and that off means an invisible browser), the Vercel integration switch (`{ vercelEnabled }`, `coding_vercel_enabled`, `?? false`) worn as a BETA flag — a visible Beta badge and a line saying it is experimental and off by default, the ONE place the integration is offered (the setup wizard never mentions it, pinned by `coding-agent-setup-wizard.test.tsx`); off, no Vercel surface renders anywhere (the project page's link card, a run's deployment card, Deploy buttons and pipeline strip, the delete dialog's link lines, the MCP status's deployment line, the `coding_deploy_*` tools, and the deploy watch, which arms nothing and abandons a pending build at its next tick), and `migrateVercelEnabled` switches it on ONLY to honour an explicit earlier opt-in, a Vercel project the owner had already attached — and the GitHub card (device-flow login, sign-out, terminal fallback), with the ClawBox Improvement Program card (`ImprovementProgramCard.tsx`) mounted directly under it. Emits `CODING_AGENT_CHANGED_EVENT` after every saved change so an open Coding Agent window refreshes. - **`CodingAgentSetupWizard.tsx`** — the Coding Agent's first-run wizard: intro, then the numbered steps github → **improvement** → project → **browser** → harness (five: the browser step was added between the project folder and the harness check, and the Improvement Program step right after GitHub, whose credential its reports go out on — Off / Ask / Automatic with Automatic preselected, an answer the box already holds (`ask`/`auto`) kept, Next POSTing `{ mode }` to `improvement-program` and staying on the step with the route's sentence when that is refused; there is no Skip, Next with Off chosen is the way to decline and is still written explicitly). The browser step says that a run can verify its work on the device's own screen, which the owner can watch: **Enable** writes `{ realBrowser: true }` and makes the browser ready — installing Chromium through `browser/manage` when it is missing (that path runs `playwright install chromium`) and then opening it — saying in the device's own words which of the two failed and offering a Continue button rather than stranding the owner on a step whose only two buttons have just refused (the setting is written before the window is touched, so a failed launch loses nothing); **Skip** writes `{ realBrowser: false }` and is an answer, not a deferral. Both land on harness, whose **Try it once** starts the harness run and finishes setup with its id (`onDone(runId)`), so the owner lands on the run's own page rather than on home with the run a dot in the rail. -- **`MemoryShardApp.tsx`** / **`MemoryShardSettingsPanel.tsx`** — the Memory Shard desktop app (`memory-shard`, on EVERY edition since the Hermes port, the Material `diamond` glyph — the shard the app is named for rather than a memory chip; the `diamond_shine` variant it replaced put sparkle marks on top of the facet lines and read as a blob at 48px): the memory index card that used to sit inside ClawKeep — index now, schedule, status. ClawKeep does NOT point at it — `ClawKeepApp.tsx` names memory nowhere, its dashboard is the backup surface alone, and the app is reached from the shelf, the launcher or `/app/memory-shard`. (A pointer card there would need `clawkeep.memory.pointer*` keys, which the catalogue does not carry; adding the card means adding them in all ten locales first.) The shared card/stat/dialog helpers live in `clawkeep-ui.tsx`. Four exclusive faces in this order — settings, then the loader, then the wizard, then home — because settings is the way back out of itself and nothing but the loader may paint before the first read answers: that read shells out to the OpenClaw CLI on the edition that has one and can take a minute and a half on a cold box, so the window shows `MemoryShardArt` rather than an empty column, and hands the status it already read to the index card instead of paying for a second probe. The embedded settings page (the Coding Agent's pattern, for the same reason: a switch belongs beside the thing it governs) carries the on/off switch and Start over; `onMemoryShardChanged` (`ui-events.ts`) makes a second window follow the switch instead of going on offering "Index now". The switch is real, not a label: `clawkeep/memory/enable` refreshes the scheduler so "off" disarms tonight's slot here and now, the scheduler re-reads it when a slot FIRES as well as when it arms, `getMemoryStatus` reports no `nextRunAtMs` while it is off (a "next run" at an hour nothing happens at), and `clawkeep/memory/index` answers 409 `disabled` — nothing is wrong with WHO asked. Switching it ON — and finishing the wizard — also needs a paid ClawBox AI plan since 2026-09-14 (`clawkeep/memory/enable` refuses 402 `paid_plan_required`, the same `src/lib/paid-plan-gate.ts` rule the Coding Agent uses, with the same shape: only a body that would enable or complete setup is refused, nothing is auto-disabled, and the status carries `planGate`). Only an EXPLICIT `enabled: false` greys the buttons, since a server that predates the switch sends no such field and that box IS indexing. `clawkeep/memory/reset` (owner-only, like every write here, because middleware admits the MCP bearer and the agent must not reopen its owner's onboarding) writes an explicit false to both flags rather than deleting them — an absent completion flag falls back to the switch, so a deleted key would leave a switched-on box believing setup was finished — and clears the schedule, while deliberately leaving the folders, the embedding model and the index alone: re-running the wizard costs nothing when they are there, and a "start over" that deleted a working index would be an hours-long re-embed nobody asked for. WHERE the index is embedded is the owner's to choose since 2026-09-15 (the ask: a cloud/local switch in its settings, and a wizard that uses the cloud model when the box has a ClawBox AI plan instead of downloading 640 MB): `clawkeep/memory/provider` answers a GET — `source` (`cloud` | `local`, from the configured endpoint, loopback or none being the box's own), `cloudSupported` (false on the SKU where ClawBox is the indexer, whose client refuses a non-loopback embedder by design), `cloudAvailable` (the cloud-defaults resolver's OWN facts, so it means what the default that promotes a box means: linked, a paid plan, a cloud embedder that answered the probe) and `localInstalled` — and its POST takes `{ source }`, owner cookie AND same origin: no body is the model on this box, the call every older caller made; `cloud` is refused 409 `cloud_unsupported` or `cloud_unavailable` before any pin is written, and otherwise writes `noteOwnerChoice("embeddings")` then `switchToCloudEmbeddings`. The wizard's last step (`MemoryShardWizard.tsx`) reads that GET on mount and PRESELECTS the cloud model whenever it is on offer — then fetches nothing — with a cloud/on-this-box picker the owner's pick overrides; an older server or a failed read leaves the model on this box. The settings page's embedder card (`MemoryShardEmbedderCard.tsx`, above the Folders card) switches either way: it fetches the GGUF through `embed/install` first when the box has none, and posts a FULL pass after the switch, because the switch changes the index identity. Its strings are `clawkeep.memory.embedder.*` and `clawkeep.memory.setup.provisionBodyCloud`, in all ten locales. Between the switch and Start over sits a Folders card (`memory-shard-folders-card`, "Folders it reads") — `MemoryShardFolders.tsx`, the list with a Remove per row and the same picker as the wizard, the ONE component the wizard's folders step renders too so the two cannot drift, with one busy state for the whole thing: every folder control is disabled while an add or a remove is in flight, the pressed button reads "Adding…"/"Removing…", the picker closes only when the add SUCCEEDED, a refused write is shown in the route's own words (`clawkeep.memory.folders.*` in all ten locales), and `onBusyChange` lets a host hold its own control — the wizard's Next — while a write is in flight. Behind it every change to `memory.search.extraPaths` goes through `mutateExtraPaths` in `src/lib/memory-shard.ts` — a module-level promise chain that runs each read-modify-write after the previous one settled (a failed write is its caller's to report, never the next caller's to inherit) and writes only when the list differs — because on the box an add and a remove that overlapped (each a ~5 s `openclaw config set` the gateway restarts on) left openclaw.json with no folder while the add had answered with one. Inside that queue the list is read through `readConfigStrict` (`ExtraPathsUnreadableError`), never the lenient `readExtraPaths` the GET and the wizard's first paint use, because a half-written or EACCES'd openclaw.json read as `[]` would be WRITTEN — one folder saved over the owner's whole list — so the sources route answers 500 `read_failed` (the read BEFORE the write failed; nothing touched) or `write_failed` (the CLI refused), each with a sentence, and marks the status cache stale after a write the way the provider route does; a read-back that fails AFTER a landed write is not either of those — the CLI has saved the list by then — so `mutateExtraPaths` answers the written list and the route says success. **On the SKU with no OpenClaw ClawBox owns the index itself** (`src/lib/memory-index-local.ts`, added because Hermes ships no memory index, no embeddings and no recall of any kind — `hermes memory` configures third-party providers and has no reindex verb, which `scripts/clawbox-identity-sync.sh` records the hard way): a sqlite store at `data/memory-index/index.sqlite` (`meta`/`files`/`chunks`, vectors as float32 blobs normalised at write time so a search is a dot product), fed by the SAME embedder every edition installs — reached through `/setup-api/local-ai/embed/v1`, never port 8081, because the proxy is what wakes the unit, restores the Qwen3 query instruction and trims an input that would not fit the batch — over the SAME `extractDocuments` pipeline, with the derived folder walked by the pass rather than registered as a second source. The seam is two branches on `openclawIsAbsent()` (never `hasHermesHarness()`: a `dual` box has the binary and must keep using OpenClaw's index under either harness): `loadMemoryStatus` sources its probe from `localMemoryStatusJson()`, which emits the row shape `openclaw memory status --deep --json` emits so `parseMemoryStatus`, the health rules, the fingerprint and the whole `MemoryStatusErrorCode` catalogue are shared and NO new locale key exists; and `startMemoryIndex` picks `startLocalPass` over `startOpenclawPass` behind one `IndexPass` interface, so the decline, `RUN_LOCK_PATH`, the run-state file, the liveness reconcile (`childPid: process.pid` for an in-process pass), the two-hour budget and the cache invalidation are written once. `memory.search.extraPaths` has a ClawBox-side counterpart there (`memory_shard_sources`) which is not a mirror but THE setting, since ClawBox is what indexes; `switchToLocalEmbeddings` becomes `stampLocalEmbeddingIdentity` (provider|model|baseUrl, compared on every pass and every status read — an embedder change reports `mismatched` and draws the existing amber "Run a full reindex"); `MAX_INDEX_CHUNKS` (20,000) is reported, never silent. `GET /setup-api/clawkeep/memory/search` is the read half and is the ONE route in this family that is deliberately not owner-only — the MCP bearer is the intended caller — behind the MCP tool `memory_shard_search` (`editions: ["hermes"]`, `profile: "core"`, so a small local model keeps it), which answers display names and never a path under the owner's home. `install.sh`'s `step_embed_model` lost its harness bail so `--step embed_model` works there; the MAIN flow kept it, so only the owner's wizard click spends the 639 MB. +- **`MemoryShardApp.tsx`** / **`MemoryShardSettingsPanel.tsx`** — the Memory Shard desktop app (`memory-shard`, on EVERY edition since the Hermes port, the Material `diamond` glyph — the shard the app is named for rather than a memory chip; the `diamond_shine` variant it replaced put sparkle marks on top of the facet lines and read as a blob at 48px): the memory index card that used to sit inside ClawKeep — index now, schedule, status. ClawKeep does NOT point at it — `ClawKeepApp.tsx` names memory nowhere, its dashboard is the backup surface alone, and the app is reached from the shelf, the launcher or `/app/memory-shard`. (A pointer card there would need `clawkeep.memory.pointer*` keys, which the catalogue does not carry; adding the card means adding them in all ten locales first.) The shared card/stat/dialog helpers live in `clawkeep-ui.tsx`. Four exclusive faces in this order — settings, then the loader, then the wizard, then home — because settings is the way back out of itself and nothing but the loader may paint before the first read answers: that read shells out to the OpenClaw CLI on the edition that has one and can take a minute and a half on a cold box, so the window shows `MemoryShardArt` rather than an empty column, and hands the status it already read to the index card instead of paying for a second probe. The embedded settings page (the Coding Agent's pattern, for the same reason: a switch belongs beside the thing it governs) carries the on/off switch and Start over; `onMemoryShardChanged` (`ui-events.ts`) makes a second window follow the switch instead of going on offering "Index now". The switch is real, not a label: `clawkeep/memory/enable` refreshes the scheduler so "off" disarms tonight's slot here and now, the scheduler re-reads it when a slot FIRES as well as when it arms, `getMemoryStatus` reports no `nextRunAtMs` while it is off (a "next run" at an hour nothing happens at), and `clawkeep/memory/index` answers 409 `disabled` — nothing is wrong with WHO asked. Switching it ON — and finishing the wizard — also needs a paid ClawBox AI plan since 2026-09-14 (`clawkeep/memory/enable` refuses 402 `paid_plan_required`, the same `src/lib/paid-plan-gate.ts` rule the Coding Agent uses, with the same shape: only a body that would enable or complete setup is refused, nothing is auto-disabled, and the status carries `planGate`). Only an EXPLICIT `enabled: false` greys the buttons, since a server that predates the switch sends no such field and that box IS indexing. `clawkeep/memory/reset` (owner-only, like every write here, because middleware admits the MCP bearer and the agent must not reopen its owner's onboarding) writes an explicit false to both flags rather than deleting them — an absent completion flag falls back to the switch, so a deleted key would leave a switched-on box believing setup was finished — and clears the schedule, while deliberately leaving the folders, the embedding model and the index alone: re-running the wizard costs nothing when they are there, and a "start over" that deleted a working index would be an hours-long re-embed nobody asked for. WHERE the index is embedded is the owner's to choose since 2026-09-15 (the ask: a cloud/local switch in its settings, and a wizard that uses the cloud model when the box has a ClawBox AI plan instead of downloading 640 MB): `clawkeep/memory/provider` answers a GET — `source` (`cloud` | `local`), `recorded` (is that WRITTEN DOWN, or the default rule speaking), `cloudSupported` (true on EVERY edition since 2026-09-18 — the indexer's client now accepts exactly two addresses, the loopback proxy and this box's ClawBox AI endpoint, so the fence is still a fence; the field is kept for an older server and for the build-time kill switch `CloudDefaultsFacts.embeddingsSupported`), `cloudAvailable` (the cloud-defaults resolver's OWN facts, so it means what the default that promotes a box means: linked, a paid plan, a cloud embedder that answered the probe) and `localInstalled` — and its POST takes `{ source }`, owner cookie AND same origin: no body is the model on this box, the call every older caller made; `cloud` is refused 409 `cloud_unsupported` or `cloud_unavailable` before any pin is written, and otherwise writes `noteOwnerChoice("embeddings")` then `switchToCloudEmbeddings`. **THE DEFAULT IS THE CLOUD, ALWAYS** (the owner's ruling of 2026-09-18): with nothing recorded the index embeds through ClawBox AI wherever the box's subscription covers it, "On this box" is the opt-in, and a box with no credential indexes on itself, names the reason and moves on its own once linked. `recorded` is what the wizard's last step turns on — `source` alone says "cloud" on a box that has written nothing down, so a wizard that read it as "nothing to do" finished without the pin and the next boot's promotion rebuilt an index that was already correct. On the SKU where ClawBox indexes, the PIN (`memory_shard_embedder`) is what records a choice and `memory_embeddings_choice_source: "owner"` alone is not one: every box that finished the old wizard carries that mark because the wizard could only ever post the model on this box, so `defaultEmbedderSource` and `applyClawaiCloudDefaults` both honour it only where a pin stands beside it — a legacy box is promoted WITH the pin write and the full rebuild (and `clearOwnerChoice`, so the card stops claiming a choice nobody made), while a pick made through the settings card after that change has both and stays local. `clawai_cloud_embeddings: "off"` is read by `resolveMemoryEmbedder` itself, not only on the unpinned path, so the support lever still reaches a box the promotion has pinned. The wizard's last step (`MemoryShardWizard.tsx`) reads that GET on mount and PRESELECTS the cloud model whenever it is on offer — then fetches nothing — with a cloud/on-this-box picker the owner's pick overrides; an older server or a failed read leaves the model on this box. The settings page's embedder card (`MemoryShardEmbedderCard.tsx`, above the Folders card) switches either way: it fetches the GGUF through `embed/install` first when the box has none, and posts a FULL pass after the switch, because the switch changes the index identity. Its strings are `clawkeep.memory.embedder.*` and `clawkeep.memory.setup.provisionBodyCloud`, in all ten locales. Between the switch and Start over sits a Folders card (`memory-shard-folders-card`, "Folders it reads") — `MemoryShardFolders.tsx`, the list with a Remove per row and the same picker as the wizard, the ONE component the wizard's folders step renders too so the two cannot drift, with one busy state for the whole thing: every folder control is disabled while an add or a remove is in flight, the pressed button reads "Adding…"/"Removing…", the picker closes only when the add SUCCEEDED, a refused write is shown in the route's own words (`clawkeep.memory.folders.*` in all ten locales), and `onBusyChange` lets a host hold its own control — the wizard's Next — while a write is in flight. Behind it every change to `memory.search.extraPaths` goes through `mutateExtraPaths` in `src/lib/memory-shard.ts` — a module-level promise chain that runs each read-modify-write after the previous one settled (a failed write is its caller's to report, never the next caller's to inherit) and writes only when the list differs — because on the box an add and a remove that overlapped (each a ~5 s `openclaw config set` the gateway restarts on) left openclaw.json with no folder while the add had answered with one. Inside that queue the list is read through `readConfigStrict` (`ExtraPathsUnreadableError`), never the lenient `readExtraPaths` the GET and the wizard's first paint use, because a half-written or EACCES'd openclaw.json read as `[]` would be WRITTEN — one folder saved over the owner's whole list — so the sources route answers 500 `read_failed` (the read BEFORE the write failed; nothing touched) or `write_failed` (the CLI refused), each with a sentence, and marks the status cache stale after a write the way the provider route does; a read-back that fails AFTER a landed write is not either of those — the CLI has saved the list by then — so `mutateExtraPaths` answers the written list and the route says success. **On the SKU with no OpenClaw ClawBox owns the index itself** (`src/lib/memory-index-local.ts`, added because Hermes ships no memory index, no embeddings and no recall of any kind — `hermes memory` configures third-party providers and has no reindex verb, which `scripts/clawbox-identity-sync.sh` records the hard way): a sqlite store at `data/memory-index/index.sqlite` (`meta`/`files`/`chunks`, vectors as float32 blobs normalised at write time so a search is a dot product), fed by the SAME embedder every edition installs — reached through `/setup-api/local-ai/embed/v1`, never port 8081, because the proxy is what wakes the unit, restores the Qwen3 query instruction and trims an input that would not fit the batch — over the SAME `extractDocuments` pipeline, with the derived folder walked by the pass rather than registered as a second source. The seam is two branches on `openclawIsAbsent()` (never `hasHermesHarness()`: a `dual` box has the binary and must keep using OpenClaw's index under either harness): `loadMemoryStatus` sources its probe from `localMemoryStatusJson()`, which emits the row shape `openclaw memory status --deep --json` emits so `parseMemoryStatus`, the health rules, the fingerprint and the whole `MemoryStatusErrorCode` catalogue are shared and NO new locale key exists; and `startMemoryIndex` picks `startLocalPass` over `startOpenclawPass` behind one `IndexPass` interface, so the decline, `RUN_LOCK_PATH`, the run-state file, the liveness reconcile (`childPid: process.pid` for an in-process pass), the two-hour budget and the cache invalidation are written once. `memory.search.extraPaths` has a ClawBox-side counterpart there (`memory_shard_sources`) which is not a mirror but THE setting, since ClawBox is what indexes; `switchToLocalEmbeddings` becomes `stampLocalEmbeddingIdentity` (provider|model|baseUrl, compared on every pass and every status read — an embedder change reports `mismatched` and draws the existing amber "Run a full reindex"); the chunk ceiling is a MEMORY BUDGET rather than a number (`maxIndexChunks(dimensions)`, ~78 MiB of float32 — 20,000 chunks of Qwen3's 1,024 dimensions, 6,666 of the cloud model's 3,072, since the same 20,000 there would have been ~234 MiB allocated contiguously by `loadVectors` on the agent's first search after a restart) and reaching it is reported, never silent; a batch the CLOUD endpoint refuses with 429 or 5xx is retried with backoff honouring `Retry-After` (a rebuild empties the store first, so one transient refusal at request 500 of ~1,250 used to leave the box with no index at all), while the loopback proxy's 502 — the MemAvailable wake guard — still ends the pass, because that is the box saying it cannot hold the model right now; and `semanticAvailable` on the cloud arm asks `clawaiCredentialRefused()` as well as the token, since a credential the proxy has rejected sits in the store looking exactly like a working one. That memo is armed from the embedding path the same way the picture and voice paths arm it — through `proxyRefusedClawaiCredential`, which accepts 401 `missing_token` / 403 `invalid_token` and NOTHING else: 402 is what ClawBox's own routes and the portal answer, never the proxy, and the proxy's PLAN gate is a 403 that names the plan ("TTS is Max-only on the proxy, which answers 403 to Free and Pro") and is deliberately not read as a dead credential, since a refused credential is what sends an owner to re-link the device. A plan that lapses after the embedder was pinned to the cloud therefore still reads as available; closing that needs a fact about the ACCOUNT from the portal poll, not a wider reading of a status here. `GET /setup-api/clawkeep/memory/search` is the read half and is the ONE route in this family that is deliberately not owner-only — the MCP bearer is the intended caller — behind the MCP tool `memory_shard_search` (`editions: ["hermes"]`, `profile: "core"`, so a small local model keeps it), which answers display names and never a path under the owner's home. `install.sh`'s `step_embed_model` lost its harness bail so `--step embed_model` works there; the MAIN flow kept it, so only the owner's wizard click spends the 639 MB. - **`ToastHost.tsx`** — the desktop's toast surface (its Dismiss and the notice destinations' phrases are catalogue keys — `desktop.toast.*`, `notifyActionLabel(action, t)` — since the shell's aria labels were the one thing a German desktop's screen reader still heard in English); the only listener for the `clawbox:toast` event every server-side owner notice ends in. A notice may carry an ALLOWLISTED destination (`src/lib/notify-action.ts`, e.g. `{ open: "settings", section: "email" }`): its body — icon and text, everything but the X — is then a button that opens Settings on that section through `dispatchOpenSettingsSection` and dismisses the toast, which is how the email-approval notice reaches the queue. `/setup-api/kv` strips the destination from a notice posted through the legacy slot, so the agent-driven `ui_notify` and `clawbox notify` can never make a toast clickable - A window record can carry `meta` (`openApp(appId, forceNew, meta)`); the Terminal window's command travels that way (`clawbox:open-terminal` → `meta.command`), never in shared state, so a plain Terminal opened later types nothing. - Top-right notices — the finished-run card, the update card, the ClawBox AI offer and a Telegram access request — all leave on their own after `NOTICE_AUTO_HIDE_MS` (`src/lib/use-auto-hide.ts`, `useAutoHide(keys, onExpire)`): a key keeps its clock across re-renders, a hand dismissal clears it, and none of the timeouts is recorded as a dismissal (the update card is back after a reload; a timed-out pairing request stays off the desktop for the session only and is still in Settings → Telegram). diff --git a/src/app/setup-api/clawkeep/memory/provider/route.ts b/src/app/setup-api/clawkeep/memory/provider/route.ts index 7aff67ad0..8e4011540 100644 --- a/src/app/setup-api/clawkeep/memory/provider/route.ts +++ b/src/app/setup-api/clawkeep/memory/provider/route.ts @@ -1,11 +1,9 @@ import { NextResponse } from "next/server"; import { hasOwnerSession } from "@/lib/owner-session"; import { isSameOriginRequest } from "@/lib/same-origin"; -import { readEmbeddingChoice, switchToLocalEmbeddings } from "@/lib/memory-shard"; +import { readEmbeddingPlacement, switchToLocalEmbeddings } from "@/lib/memory-shard"; import { noteOwnerChoice } from "@/lib/clawai-cloud-choice"; import { invalidateMemoryStatusCache } from "@/lib/clawkeep-memory"; -import { isLoopbackBaseUrl } from "@/lib/embed-runtime-ids"; -import { openclawIsAbsent } from "@/lib/openclaw-config"; // The PURE half of the cloud-defaults rule (client-safe, one type import // behind it): the server half is loaded lazily below with the probe it owns. import { resolveClawaiCloudDefaults } from "@/lib/clawai-cloud-defaults-state"; @@ -58,7 +56,6 @@ export async function GET() { } async function readEmbedderChoice(): Promise { - const cloudSupported = !openclawIsAbsent(); // Loaded here rather than at the top: they pull the cloud-defaults resolver, // the embedder probe and the provisioning check, and the POST's local path — // the one every older caller takes — needs none of them. @@ -66,30 +63,40 @@ async function readEmbedderChoice(): Promise { import("@/lib/embed-server"), import("@/lib/clawai-cloud-defaults"), ]); - const [choice, provisioning, facts] = await Promise.all([ - cloudSupported ? readEmbeddingChoice() : Promise.resolve(null), + const [provisioning, facts] = await Promise.all([ getEmbedProvisioningStatus().catch(() => null), // The resolver's own facts, so "the cloud model is on offer here" means the - // same thing it means to the default that promotes a box onto it: linked, - // a paid plan, and a probe the cloud embedder answered. - cloudSupported ? readCloudDefaultsFacts().catch(() => null) : Promise.resolve(null), + // same thing it means to the default that puts a box on it: linked, a paid + // plan, and a probe the cloud embedder answered. + readCloudDefaultsFacts().catch(() => null), ]); - const baseUrl = choice?.baseUrl ?? null; - // No endpoint at all is the on-device answer, the resolver's own rule. - const source: EmbeddingSource = cloudSupported && baseUrl && !isLoopbackBaseUrl(baseUrl) ? "cloud" : "local"; // The SAME verdict the automatic default acts on, reasons and all, rather // than a second reading of one of its facts: `embeddingsRouteReady` alone // answered false to a box with no credential, an unpaid plan and a proxy that // did not answer alike, and the switch could only say "not available on this - // box right now" to all three. The edition that indexes on the box itself is - // answered without reading the facts — nothing the resolver could say would - // change it, and asking would buy an 8 s probe for a fixed answer. - const verdict = cloudSupported - ? facts && resolveClawaiCloudDefaults(facts).embeddings - : ({ source: "local", reason: "edition" } as const); + // box right now" to all three. + const verdict = facts && resolveClawaiCloudDefaults(facts).embeddings; + // And the same verdict again as the DEFAULT for a box nobody has pinned, so + // the card cannot say "the cloud is on offer" over a box that is already + // embedding there — the owner's ruling of 2026-09-18: with no choice made, + // the cloud IS the embedder wherever the subscription covers it. Where the + // facts could not be read at all, `local` is passed EXPLICITLY rather than + // left to `readEmbeddingPlacement`'s own second reading, which pays for the + // probe again and can answer differently: that is how one GET came back + // `source: "cloud"` beside `cloudAvailable: false` — a card saying the index + // is embedded somewhere it has just said is not on offer. + const placement = await readEmbeddingPlacement(verdict?.source ?? "local"); return { - source, - cloudSupported, + source: placement.source, + // Is that WRITTEN DOWN, or the default rule speaking? The wizard's cloud + // path needs the difference: with the cloud as the default, `source` + // already says "cloud" on a box that has recorded nothing, and a wizard + // that read that as "nothing to do" finished without the pin — leaving the + // next boot's promotion to write it and rebuild a correct index. + recorded: placement.recorded, + // Every edition can reach the ClawBox AI endpoint now; the fence that made + // this false is two allowed addresses rather than one (`memory-embedder.ts`). + cloudSupported: true, cloudAvailable: verdict?.source === "cloud", // Null, never a guess, when the facts could not be read at all. cloudReason: verdict?.reason ?? null, @@ -173,18 +180,12 @@ async function requestedSource(request: Request): Promise { - if (openclawIsAbsent()) { - return NextResponse.json( - { error: "This edition indexes memory on the box itself and cannot use the cloud model.", kind: "cloud_unsupported" }, - { status: 409 }, - ); - } const [{ readCloudDefaultsFacts }, { resolveClawaiToken }, cloud, { switchToCloudEmbeddings }] = await Promise.all([ import("@/lib/clawai-cloud-defaults"), import("@/lib/harness/credentials"), diff --git a/src/app/setup-api/clawkeep/memory/search/route.ts b/src/app/setup-api/clawkeep/memory/search/route.ts index 01ce49327..ddd2d1dd1 100644 --- a/src/app/setup-api/clawkeep/memory/search/route.ts +++ b/src/app/setup-api/clawkeep/memory/search/route.ts @@ -2,7 +2,7 @@ import { NextResponse } from "next/server"; import type { NextRequest } from "next/server"; import { openclawIsAbsent } from "@/lib/openclaw-config"; import { getMemoryShardEnabled } from "@/lib/memory-shard"; -import { searchLocalMemory } from "@/lib/memory-index-local"; +import { MEMORY_SEARCH_DEADLINE_MS, searchLocalMemory } from "@/lib/memory-index-local"; export const dynamic = "force-dynamic"; @@ -65,7 +65,15 @@ export async function GET(request: NextRequest) { const limit = Number.isFinite(asked) ? Math.min(MAX_LIMIT, Math.max(1, Math.trunc(asked))) : DEFAULT_LIMIT; try { - const results = await searchLocalMemory(query, limit, request.signal); + // A DEADLINE OF THE BOX'S OWN, combined with the caller's. `request.signal` + // alone is whatever the client does, and a client that simply waits left one + // embed attempt holding the request for the whole `EMBED_TIMEOUT_MS`. The + // bound is `MEMORY_SEARCH_DEADLINE_MS` — sized against the MCP tool's own + // 60 s, past which nothing is listening — and it is a bound, not a + // cancellation of the work it started: a cold llama.cpp wake carries on and + // makes the next search warm. + const deadline = AbortSignal.any([request.signal, AbortSignal.timeout(MEMORY_SEARCH_DEADLINE_MS)]); + const results = await searchLocalMemory(query, limit, deadline); return NextResponse.json({ results }, { headers: { "Cache-Control": "no-store" } }); } catch (err) { // The embedder not answering is the one failure worth naming: it is the diff --git a/src/components/MemoryShardEmbedderCard.tsx b/src/components/MemoryShardEmbedderCard.tsx index 291374acd..fa00950c0 100644 --- a/src/components/MemoryShardEmbedderCard.tsx +++ b/src/components/MemoryShardEmbedderCard.tsx @@ -143,6 +143,13 @@ export default function MemoryShardEmbedderCard() {

{t(status.source === "cloud" ? "clawkeep.memory.embedder.cloudHint" : "clawkeep.memory.embedder.localHint")}

+ {/* This branch is unreachable against any server on this branch — the + route hard-codes `cloudSupported: true` since 2026-09-18 — and is kept + for the two cases that can still produce a false: an OLDER server this + page is talking to across an update, and the build-time kill switch + `CloudDefaultsFacts.embeddingsSupported` documents. Its string is in + all ten locales; deleting the branch would mean writing it back under + whichever of those happened first. */} {!status.cloudSupported ? (

{t("clawkeep.memory.embedder.cloudUnsupported")} diff --git a/src/components/MemoryShardWizard.tsx b/src/components/MemoryShardWizard.tsx index 0a0c6ac82..14b84d5c4 100644 --- a/src/components/MemoryShardWizard.tsx +++ b/src/components/MemoryShardWizard.tsx @@ -260,14 +260,26 @@ export default function MemoryShardWizard({ onDone }: { onDone: () => void }) { setPhase("switching-provider"); setDetail(null); - // A box whose index is ALREADY embedded in the cloud has nothing to - // switch: the cloud can be picked there whatever the live probe said - // (see `cloudEmbedderPickable`), but the route's switch re-checks that - // probe and the token and answers 409 on a hiccup — a failed wizard over - // an index that was never going to move. The local path always posts: - // its owner's-choice mark is what keeps the next boot's cloud default - // from moving the index back. - const alreadyInCloud = source === "cloud" && embedder?.source === "cloud"; + // A box whose cloud embedder is ALREADY RECORDED has nothing to switch: + // the cloud can be picked there whatever the live probe said (see + // `cloudEmbedderPickable`), but the route's switch re-checks that probe + // and the token and answers 409 on a hiccup — a failed wizard over an + // index that was never going to move. The local path always posts: its + // owner's-choice mark is what keeps the next boot's cloud default from + // moving the index back. + // + // `recorded`, NOT `source`, and that is the whole of the fix. Since the + // cloud became the default, `source` says "cloud" on a box that has + // written nothing down — so this shortcut skipped the POST on every + // freshly onboarded, subscribed box, and with it the pin and the owner + // mark that POST exists for. The index was then built in the cloud, and + // the next web-server restart's automatic promotion saw an unrecorded + // box, wrote the pin and asked for a FULL rebuild: hours of re-embedding + // paid for twice over an index that was already correct, with memory + // search answering nothing throughout. A server that predates the field + // sends no `recorded`, which parses to false and makes the wizard post — + // the safe direction. + const alreadyInCloud = source === "cloud" && embedder?.source === "cloud" && embedder.recorded; const provider = alreadyInCloud ? null : await fetch( "/setup-api/clawkeep/memory/provider", source === "cloud" @@ -471,8 +483,13 @@ export default function MemoryShardWizard({ onDone }: { onDone: () => void }) { {t(source === "cloud" ? "clawkeep.memory.setup.provisionBodyCloud" : "clawkeep.memory.setup.provisionBody")}

- {/* The choice is drawn only where both halves exist: the edition that - indexes on the box itself has no cloud model to offer. */} + {/* The choice is drawn only where both halves exist. Every edition + can offer the cloud model since 2026-09-18, so the server this + page is talking to answers true — the gate is kept for the one + case that still says otherwise: an OLDER server, whose `false` + `parseEmbedderChoiceStatus` carries through, where drawing a + picker over a route that cannot honour it would be worse than + drawing nothing. */} {embedder?.cloudSupported && (
diff --git a/src/lib/clawai-cloud-defaults-state.ts b/src/lib/clawai-cloud-defaults-state.ts index ba3a18cbb..7e28991a8 100644 --- a/src/lib/clawai-cloud-defaults-state.ts +++ b/src/lib/clawai-cloud-defaults-state.ts @@ -65,7 +65,14 @@ export type CloudUnavailableReason = | "plan" /** The proxy route this capability needs did not answer. */ | "route_unavailable" - /** This edition cannot use the cloud for it at all — see the embeddings note. */ + /** + * This box cannot use the cloud for it at all. + * + * No edition answers this today — `embeddingsSupported` is a constant `true` + * — and the word is kept because that fact is the feature's kill switch: a + * build that flips it needs this reason, its note key and the card's branch + * to be there already, not written under a live incident. + */ | "edition" /** The owner picked the engine on the box. */ | "owner"; @@ -85,10 +92,17 @@ export interface CloudDefaultsFacts { /** * This box can point its memory index off the device at all. * - * False where ClawBox itself is the indexer (the edition with no OpenClaw): - * `memory-index-local.ts` refuses any embedder endpoint that is not loopback, - * deliberately — the owner's document text is the request body there — and - * that fence is not something a default may open. + * TRUE ON EVERY EDITION SINCE 2026-09-18, and kept as a fact rather than + * deleted because it is the KILL SWITCH for the whole feature: its only + * producer (`readCloudDefaultsFacts`) is one line, and a `false` there takes + * every box off the cloud embedder — with the `"edition"` reason already + * worded in all ten locales — without touching this rule or any caller. + * + * It used to be false where ClawBox itself is the indexer, because that + * client accepted a loopback endpoint and nothing else. It now accepts + * exactly two — the loopback proxy and the ClawBox AI endpoint the image was + * built with (`memory-embedder.ts`) — so the fence is still a fence and the + * SKU is no longer a reason to keep a subscriber off what they pay for. */ embeddingsSupported: boolean; } diff --git a/src/lib/clawai-cloud-defaults.ts b/src/lib/clawai-cloud-defaults.ts index 33c87db86..a46d4e3dd 100644 --- a/src/lib/clawai-cloud-defaults.ts +++ b/src/lib/clawai-cloud-defaults.ts @@ -23,6 +23,10 @@ * person pick records `owner` in this capability's source key, and a box * that predates the key is read through `ownerChoiceFrom`: a stored `local` * could only have come from a person, because nothing else ever wrote one. + * The memory embedder has one edition-specific twist on that — see + * `embeddingsOwnerChoice` — because on the SKU where ClawBox indexes, a + * wizard that could only ever post one answer stamped that key on every box + * that ran it, so the mark there records a click and not a decision. * * So the worst this can do on a box that was working is point it at a cloud * engine its subscription covers, which is what was asked for; and the most it @@ -30,7 +34,7 @@ */ import { readClawaiEntitlementTier } from "@/lib/clawai-plan-tier"; -import { readChoiceSource } from "@/lib/clawai-cloud-choice"; +import { clearOwnerChoice, readChoiceSource } from "@/lib/clawai-cloud-choice"; import { get } from "@/lib/config-store"; import { ownerChoiceFrom, @@ -48,8 +52,9 @@ import { import { invalidateMemoryStatusCache, startMemoryIndex } from "@/lib/clawkeep-memory"; import { getActiveHarness } from "@/lib/harness"; import { resolveClawaiToken } from "@/lib/harness/credentials"; -import { isLoopbackBaseUrl } from "@/lib/embed-runtime-ids"; -import { readEmbeddingChoice, switchToCloudEmbeddings } from "@/lib/memory-shard"; +import { readEmbedderPin } from "@/lib/memory-embedder"; +import { readEmbeddingPlacement, switchToCloudEmbeddings } from "@/lib/memory-shard"; +import type { EmbeddingSource } from "@/lib/memory-shard-state"; import { openclawIsAbsent } from "@/lib/openclaw-config"; import { createSerialLock } from "@/lib/serial-lock"; import { syncChannelAudio } from "@/lib/stt-channel"; @@ -105,11 +110,14 @@ async function embeddingsRouteReady(linked: boolean, plan: ClawboxAiPlanTier | n export async function readCloudDefaultsFacts(): Promise { const [token, entitlement] = await Promise.all([resolveClawaiToken(), readClawaiEntitlementTier()]); const linked = token !== null; - // ClawBox is the indexer on the edition with no OpenClaw, and its embedder - // client refuses any endpoint that is not loopback — deliberately, because - // the owner's document text is the request body there. See the fact's own - // docblock; a default may not open that fence. - const embeddingsSupported = !openclawIsAbsent(); + // Both editions since 2026-09-18. The fence the old `false` stood for is + // still there — ClawBox's own index sends the owner's text to the loopback + // proxy or to this box's ClawBox AI account and nowhere else + // (`memory-embedder.ts`) — but it is no longer a reason to keep an edition + // off a subscription it pays for. The fact is kept as a KILL SWITCH (see its + // own docblock): one `false` here takes every box off the cloud embedder + // without touching the rule that reads it. + const embeddingsSupported = true; return { linked, entitlement, @@ -153,15 +161,15 @@ function voiceSourceOf(status: VoiceOutputStatus): CapabilitySource { return (status.activeEngine ?? status.preferredEngine ?? "local") === "cloud" ? "cloud" : "local"; } -/** Where the memory index is embedded right now. */ -async function currentEmbeddingSource(): Promise { - if (openclawIsAbsent()) return "local"; - const { baseUrl } = await readEmbeddingChoice(); - // No endpoint at all is the on-device answer: the only thing this box points - // at without one is its own embedder, and claiming "cloud" over an unset key - // would make the applier skip the write that puts it right. - if (!baseUrl) return "local"; - return isLoopbackBaseUrl(baseUrl) ? "local" : "cloud"; +/** + * Where the memory index is embedded right now. + * + * @param fallback the verdict this run already computed, for a box that has + * pinned nothing: the default rule is what decides there, and reading the + * facts a second time to learn it would buy a second probe. + */ +async function currentEmbeddingSource(fallback: CapabilitySource): Promise { + return (await readEmbeddingPlacement(fallback)).source; } /** @@ -185,7 +193,7 @@ async function readStatusAndVoice(): Promise<{ status: CloudDefaultsStatus; voic const [stt, voice, embeddings, owners] = await Promise.all([ getSttPrimary(), readVoiceSnapshot(), - currentEmbeddingSource(), + currentEmbeddingSource(defaults.embeddings.source), readOwnerChoices(), ]); const tts = voiceSourceOf(voice.status); @@ -214,23 +222,53 @@ async function readStatusAndVoice(): Promise<{ status: CloudDefaultsStatus; voic * value only a person could have produced. See `ownerChoiceFrom`. */ async function readOwnerChoices(): Promise> { - const [ttsSource, sttSource, embedSource, storedStt, voiceState] = await Promise.all([ + const [ttsSource, sttSource, embedSource, storedStt, voiceState, embedPin] = await Promise.all([ readChoiceSource("tts"), readChoiceSource("stt"), readChoiceSource("embeddings"), get(STT_PRIMARY_KEY), readVoiceState(), + openclawIsAbsent() ? readEmbedderPin() : Promise.resolve(null), ]); return { tts: ownerChoiceFrom(ttsSource, voiceState.choice === "local"), stt: ownerChoiceFrom(sttSource, storedStt === "local"), - // Nothing before this feature recorded an embedding pick — the boot script - // wrote the on-device embedder on every box — so there is no earlier - // "only a person could have done this" value to grandfather here. - embeddings: ownerChoiceFrom(embedSource, false), + embeddings: embeddingsOwnerChoice(embedSource, embedPin), }; } +/** + * Did the owner choose where the memory index is embedded? + * + * ON OPENCLAW the mark is the whole answer, as it is for the other two. Nothing + * before this feature recorded an embedding pick — the boot script wrote the + * on-device embedder on every box — so there is no earlier "only a person could + * have done this" value to grandfather. + * + * ON THE EDITION WHERE CLAWBOX INDEXES, THE MARK ALONE IS NOT A CHOICE, and + * reading it as one is what this fixes. Every Hermes box that finished the + * Memory Shard wizard before 2026-09-18 carries `memory_embeddings_choice_source: + * "owner"` — the wizard's last step POSTed the model on this box because it was + * the only thing the route could offer there, and the route marks every pick as + * the owner's. Honouring that mark left the whole of that population in the + * worst of the three states: `resolveMemoryEmbedder` ignores it and started + * sending their documents to the cloud, while this applier honoured it and so + * never wrote the pin or asked for the rebuild the move needs — the stored + * identity stayed the 1,024-dimension local one, `identityOf` read `mismatched` + * and `searchLocalMemory` answered `[]` silently until some later pass happened + * to rebuild. + * + * THE PIN IS WHAT RECORDS A CHOICE THERE (`memory-embedder.ts` says the same + * thing from the reader's side), so a mark is honoured only where a pin stands + * beside it. A box whose owner picks "On this box" through the settings card + * after this change has both — the route writes the mark and + * `switchToLocalEmbeddings` writes the pin — and stays local for good. + */ +function embeddingsOwnerChoice(recorded: unknown, pin: EmbeddingSource | null): boolean { + if (openclawIsAbsent() && pin === null) return false; + return ownerChoiceFrom(recorded, false); +} + /** * One applier at a time in this process. * @@ -397,12 +435,29 @@ async function promoteTts({ harness, status }: VoiceSnapshot): Promise * boot. */ async function promoteEmbeddings(): Promise { - // Already pointed off the box: nothing to write, and writing anyway would - // invalidate a perfectly good index and buy a reindex for nothing. - if ((await currentEmbeddingSource()) === "cloud") return false; + // Already pointed off the box AND WRITTEN DOWN: nothing to write, and writing + // anyway would invalidate a perfectly good index and buy a reindex for + // nothing. `recorded` is the second half and it is load-bearing on the + // edition where ClawBox indexes: there an unpinned box ALREADY embeds in the + // cloud by default, so reading the source alone would have skipped the one + // write that records it — and with it the full pass that rebuilds an index + // whose vectors were made by the model on the box. + const placement = await readEmbeddingPlacement("cloud"); + if (placement.recorded && placement.source === "cloud") return false; const token = await resolveClawaiToken(); if (!token) return false; await switchToCloudEmbeddings(cloudEmbeddingsUrl(), token); + // THE APPLIER OWNS THIS CAPABILITY NOW, said out loud in the key that records + // who decided. A legacy Hermes box reaches here carrying `owner` from a wizard + // that could only ever post one answer (see `embeddingsOwnerChoice`); leaving + // that word in place would have the card report an owner choice for a move the + // owner never made. `auto` is the truthful value and the one `clearOwnerChoice` + // exists to write. It is not what stops a second promotion — the PIN this + // switch just wrote is, through `placement.recorded` above — so a write that + // fails here costs nothing but the label. + await clearOwnerChoice("embeddings").catch((err) => { + console.warn("[clawai-cloud-defaults] could not record who chose the embedder:", message(err)); + }); invalidateMemoryStatusCache(); // The rebuild is best-effort and reported separately: the config write has // LANDED by now, so a pass that could not start (one already running, the diff --git a/src/lib/clawai-cloud-embeddings.ts b/src/lib/clawai-cloud-embeddings.ts index 88f2df705..80b8c7b28 100644 --- a/src/lib/clawai-cloud-embeddings.ts +++ b/src/lib/clawai-cloud-embeddings.ts @@ -102,19 +102,27 @@ const MAX_ENDPOINT_CHARS = 2048; * prompt-injected coding run, a hand-edited `data/config.json` and a restored * backup all reach the store and none of them reaches here. * - * WHAT IT MAY BE. Plain `http:` is accepted, for the one case it exists for: a - * staging proxy on a trusted LAN. That is a STAGING CONTRACT and not a - * loopback restriction — pinning it to loopback would delete the staging setup - * it was added for. Outside a trusted LAN the endpoint must be `https:`, - * because the request carries the box's `claw_` bearer and the text being - * embedded in cleartext otherwise (CWE-319). Nothing here can tell a trusted - * LAN from the open internet, so that half is the operator's to honour. + * WHAT IT MAY BE. `https:`, or a plain `http:` endpoint on THIS DEVICE. Every + * request to it carries the box's `claw_` bearer and the owner's document text + * as the body, so cleartext off the device is CWE-319 and is refused by + * default — the earlier "a staging proxy on a trusted LAN is the operator's to + * honour" left the one failure that cannot be undone to a promise nothing here + * could check. A loopback `http:` leaves no interface, which is why the local + * embedder's own proxy is reached that way and is not a hole in this. * - * WHAT IS ENFORCED. The scheme must be `http:` or `https:` and the whole URL - * must parse and stay under {@link MAX_ENDPOINT_CHARS}. Anything else — a - * `file:`, a `data:`, an unparseable string, a length nothing legitimate needs - * — is refused and this box falls back to the built-in proxy route, which is - * its own account's endpoint and never a third party's. + * THE STAGING LANE IS STILL THERE AND IS NOW EXPLICIT. An image built with + * `CLAWBOX_AI_EMBEDDINGS_INSECURE=1` accepts a plain-`http:` endpoint anywhere, + * for the LAN proxy the staging contract was written for. It is the same trust + * boundary as the address itself — root's environment, never the device store — + * so the operator who needs it can still have it, and a box that was never told + * to accept cleartext cannot be talked into it by a file. + * + * WHAT IS ENFORCED. The whole URL must parse, stay under + * {@link MAX_ENDPOINT_CHARS}, and be `https:`, loopback `http:`, or `http:` + * under the opt-in above. Anything else — a `file:`, a `data:`, an unparseable + * string, a cleartext host on the network, a length nothing legitimate needs — + * is refused and this box falls back to the built-in proxy route, which is its + * own account's endpoint and never a third party's. */ function usableEndpoint(raw: string | undefined): string | null { const candidate = raw?.trim(); @@ -125,7 +133,56 @@ function usableEndpoint(raw: string | undefined): string | null { } catch { return null; } - return parsed.protocol === "https:" || parsed.protocol === "http:" ? parsed.toString() : null; + if (parsed.protocol === "https:") return parsed.toString(); + if (parsed.protocol !== "http:") return null; + return isLoopbackHost(parsed.hostname) || insecureEmbeddingsAllowed() ? parsed.toString() : null; +} + +/** A host that cannot leave this device. IPv6 arrives from `URL` in brackets. */ +function isLoopbackHost(hostname: string): boolean { + const host = hostname.toLowerCase().replace(/^\[|\]$/g, ""); + return host === "localhost" || host === "::1" || /^127\./.test(host); +} + +/** + * The staging opt-in, read where the address is read: the image's environment. + * + * Read per call rather than frozen at module load, for the same reason + * `respawnWaitMs` is: a test — and a staging operator — must be able to set it + * without a rebuild, and this is not a hot path. + */ +function insecureEmbeddingsAllowed(): boolean { + const raw = process.env.CLAWBOX_AI_EMBEDDINGS_INSECURE?.trim().toLowerCase(); + return raw === "1" || raw === "true" || raw === "yes"; +} + +/** + * Could ANYTHING embed through `raw`, as an address? + * + * Deliberately WEAKER than {@link usableEndpoint}, and the difference is whose + * client is being judged. That one is ClawBox's own fence: these are the + * addresses THIS process may send the owner's documents to, and plain `http:` + * off the device is refused because ClawBox would be the one putting them on + * the wire. This one is asked by `readEmbeddingPlacement` about OpenClaw's + * `memory.search.remote.baseUrl` — where the CORE is the embedding client and + * ClawBox has no jurisdiction at all. An owner who pointed OpenClaw's memory + * search at a llama.cpp on another machine (`http://192.168.1.50:8080/v1`) has a + * recorded placement, and reading it as unset had the boot promotion replace + * their endpoint with the ClawBox AI one and start a full reindex — a + * configuration change nobody asked for. + * + * What it still refuses is what a truncated URL, a `file:`/`data:` scheme or a + * half-finished migration leaves behind: not an address, so not a placement. + */ +export function embeddingEndpointParseable(raw: string | undefined): boolean { + const candidate = raw?.trim(); + if (!candidate || candidate.length > MAX_ENDPOINT_CHARS) return false; + try { + const parsed = new URL(candidate); + return (parsed.protocol === "https:" || parsed.protocol === "http:") && parsed.hostname !== ""; + } catch { + return false; + } } /** @@ -140,7 +197,16 @@ function usableEndpoint(raw: string | undefined): string | null { */ export function cloudEmbeddingsUrl(): string { const override = usableEndpoint(process.env.CLAWBOX_AI_EMBEDDINGS_URL); - return override || `${CLAWBOX_AI_PROXY_URL.replace(/\/+$/, "")}/embeddings`; + if (override) return override; + // THE BUILT-IN ROUTE GOES THROUGH THE SAME GATE. `CLAWBOX_AI_PROXY_URL` is + // itself env-overridable, so exempting it would have left the one address + // every unconfigured box uses as the way round the rule above — a proxy + // override of `http://…` would have carried the bearer and the owner's + // documents in cleartext while the override beside it was refused for it. + // An empty string is the answer when even that cannot be used: it is not an + // address, so every caller fails closed on it ({@link embedEndpointAllowed} + // refuses it, the probe refuses it, and a switch to the cloud throws). + return usableEndpoint(`${CLAWBOX_AI_PROXY_URL.replace(/\/+$/, "")}/embeddings`) ?? ""; } /** Has the owner (or a support engineer) switched the cloud embedder off here? */ @@ -340,6 +406,11 @@ async function askCloudEmbedder(endpoint: string): Promise { try { const res = await fetch(target, { method: "POST", + // The same rule as the indexer's own request: `usableEndpoint` judged THIS + // address, and a followed 307/308 would re-send the POST — and this box's + // `claw_` bearer, which a same-site redirect keeps — somewhere it did not. + // Manual turns that answer into the `!res.ok` "the probe said no" below. + redirect: "manual", headers: { "content-type": "application/json", authorization: `Bearer ${token}` }, body: JSON.stringify({ model: CLOUD_EMBEDDING_MODEL, input: "clawbox" }), signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), diff --git a/src/lib/clawkeep-memory.ts b/src/lib/clawkeep-memory.ts index 76c5a1442..a5989a546 100644 --- a/src/lib/clawkeep-memory.ts +++ b/src/lib/clawkeep-memory.ts @@ -31,7 +31,7 @@ import { readPlanGate } from "@/lib/paid-plan-gate-server"; import { CLAWKEEP_DATA_DIR } from "@/lib/clawkeep"; import { CONFIG_PATH, findOpenclawBin, openclawIsAbsent } from "@/lib/openclaw-config"; -import { getEmbedProxyBaseUrl } from "@/lib/embed-server"; +import { resolveMemoryEmbedder } from "@/lib/memory-embedder"; import { IndexPassAbortedError, localMemoryStatusJson, @@ -832,9 +832,11 @@ async function loadMemoryStatus(): Promise { // openclaw.json is what points the OTHER arm's client at an embedder, and // on this SKU there is no such file — asked anyway it answers null, which // `providerLocation` reports as "unknown" and the card draws as an - // embedder it cannot place. The local arm embeds through this box's own - // loopback proxy, so that URL is the answer. - local ? Promise.resolve(getEmbedProxyBaseUrl()) : readEmbeddingRemoteBaseUrl(), + // embedder it cannot place. The local arm embeds through whichever of its + // two endpoints it is pointed at, so THAT url is the answer: the loopback + // proxy, or this box's ClawBox AI account. Hard-coded to the proxy, the + // card said "On device" over an index being embedded in the cloud. + local ? resolveMemoryEmbedder().then((embedder) => embedder.baseUrl) : readEmbeddingRemoteBaseUrl(), ]); if (!probe.ok) return unavailableStatus(run, schedule); try { diff --git a/src/lib/memory-embedder.ts b/src/lib/memory-embedder.ts new file mode 100644 index 000000000..863cb7b07 --- /dev/null +++ b/src/lib/memory-embedder.ts @@ -0,0 +1,249 @@ +/** + * WHICH embedder Memory Shard's own index uses, on the edition where ClawBox is + * the indexer — and the two places it is allowed to send the owner's text. + * + * SERVER ONLY. + * + * The OpenClaw edition has had this since 2026-09-15 and keeps it in OpenClaw's + * own `memory.search` config, because there the CORE is the embedding client. + * Here ClawBox is the client (`memory-index-local.ts`), so the same choice lives + * in ClawBox's own store — one key, one copy, and never a second store beside + * the one that governs. + * + * THE DEFAULT IS THE CLOUD (the owner's ruling of 2026-09-18: "the default + * should be cloud, always"). Nothing stored means the box follows the SAME + * cloud-defaults verdict the OpenClaw arm follows — linked, a paid plan and a + * cloud embedder that answered a probe — and the model on this box is what the + * owner opts IN to from the settings card. A box that cannot reach the cloud + * indexes on itself and the card names the reason, in the resolver's own + * vocabulary. + * + * WHAT IS STORED IS A WORD AND NEVER AN ADDRESS, the same rule + * `CLAWAI_CLOUD_EMBEDDINGS_KEY` is written around: everything the owner has + * indexed becomes the body of these requests, and `data/config.json` is reachable + * by a restored backup and by a hand edit. Where the cloud embedder IS comes from + * the environment the image was built with, every time, and the loopback proxy's + * address comes from the embed runtime. Anything the key holds that is not one of + * the two words is ignored, which lands on the default rule above. + */ + +import { get as configGet, set as configSet } from "@/lib/config-store"; +import { + cloudEmbeddingsSwitchedOff, + cloudEmbeddingsUrl, + embeddingsBaseUrlOf, + CLOUD_EMBEDDING_DIMENSIONS, + CLOUD_EMBEDDING_MODEL, + CLOUD_EMBEDDING_PROVIDER, +} from "@/lib/clawai-cloud-embeddings"; +import { getEmbedProxyBaseUrl } from "@/lib/embed-server"; +import { isLoopbackBaseUrl } from "@/lib/embed-runtime-ids"; +import { resolveClawaiToken } from "@/lib/harness/credentials"; +import { resolveClawaiCloudDefaults } from "@/lib/clawai-cloud-defaults-state"; +import { + LOCAL_EMBEDDING_DIMENSIONS, + LOCAL_EMBEDDING_MODEL, + LOCAL_EMBEDDING_PROVIDER, + MEMORY_SHARD_EMBEDDER_KEY, + type EmbeddingSource, +} from "@/lib/memory-shard-state"; + +/** + * One embedder, fully resolved: who it is, where the request goes, and what it + * is allowed to carry. + * + * `baseUrl` is the endpoint WITHOUT `/embeddings`, because that is the string + * the index identity is built from and the one OpenClaw stores as + * `memory.search.remote.baseUrl` — the two editions describe an embedder the + * same way, so a box that swaps harnesses does not silently change what its + * index belongs to. + */ +export interface ResolvedEmbedder { + source: EmbeddingSource; + provider: string; + model: string; + /** Identity and fence: the endpoint with no `/embeddings` on the end. */ + baseUrl: string; + /** Exactly where the POST goes. */ + requestUrl: string; + /** + * How wide a vector it answers with. + * + * CARRIED, not written in a comment, because the index's memory budget is a + * function of it: a chunk costs `dimensions × 4` bytes of float32, and the + * cloud model is three times as wide as the one on the box. The ceiling and + * the vector cache are derived from this (`maxIndexChunks`), so the budget + * they were sized against holds on either arm instead of tripling under one + * of them. What the endpoint ACTUALLY returns is still measured per pass — + * `assertDimension` — because a declared width is a plan, not a fact. + */ + dimensions: number; + /** + * The bearer. Null only on the cloud arm of a box that holds no ClawBox AI + * credential — a state {@link embedderUsable} answers for, rather than one + * that throws inside a status read. + */ + token: string | null; + /** + * Label each input `query`/`document`. + * + * ONLY on the loopback proxy, which is what reads the label and restores + * Qwen3's query instruction (`embed-query-instruction.ts`) before dropping it. + * The cloud route is OpenAI-shaped and `input_type` is an unknown field there + * — the same reason `switchToCloudEmbeddings` UNSETS `queryInputType`/ + * `documentInputType` on the OpenClaw edition rather than leaving them. + */ + labelInputs: boolean; +} + +/** Is this embedder in a state that can actually embed? */ +export function embedderUsable(embedder: ResolvedEmbedder): boolean { + return embedder.source === "local" || embedder.token !== null; +} + +/** The word in the store, or null for a box nobody has pinned. */ +export async function readEmbedderPin(): Promise { + const stored = await configGet(MEMORY_SHARD_EMBEDDER_KEY); + return stored === "cloud" || stored === "local" ? stored : null; +} + +/** Record where the index is embedded. A word: see the module docblock. */ +export async function writeEmbedderPin(source: EmbeddingSource): Promise { + await configSet(MEMORY_SHARD_EMBEDDER_KEY, source); +} + +/** + * What a box nobody has pinned uses: the cloud-defaults resolver's own verdict. + * + * The SAME rule and the same facts the OpenClaw edition's automatic default acts + * on — linked, a paid plan, a cloud embedder that answered — so "the default is + * cloud" means one thing on both editions. Facts that cannot be read at all are + * the model on this box: an index that cannot be embedded at all is worse than + * one embedded slowly, and the next read tries again. + * + * THE PIN IS WHAT RECORDS A CHOICE HERE, and deliberately not + * `memory_embeddings_choice_source`. Every box that finished the wizard before + * this change carries `owner` in that key — the wizard's last step POSTed the + * model on this box because it was the only thing the route could offer on this + * SKU, and the route marks every pick as the owner's. Reading that mark as "the + * owner asked for the model on this box" would keep every one of those boxes off + * the subscription it pays for, for good, which is the opposite of the ruling + * above. + * + * AND THE AUTOMATIC PROMOTION READS IT THE SAME WAY, which is the half that was + * missing: `readOwnerChoices` in `clawai-cloud-defaults.ts` honours that mark + * only where a PIN stands beside it, so a legacy box is promoted properly — the + * pin written and the full rebuild asked for — instead of this reader moving it + * to the cloud while the applier believed the owner had spoken and left the + * index stamped for the other model. One mark, one meaning, on both sides. + */ +export async function defaultEmbedderSource(): Promise { + // Lazily, and only here: the server half pulls the probe, the OpenClaw CLI and + // the memory index, and this module is imported by the index itself. + const { readCloudDefaultsFacts } = await import("@/lib/clawai-cloud-defaults"); + const facts = await readCloudDefaultsFacts().catch(() => null); + if (!facts) return "local"; + return resolveClawaiCloudDefaults(facts).embeddings.source; +} + +function localEmbedder(): ResolvedEmbedder { + const baseUrl = getEmbedProxyBaseUrl(); + return { + source: "local", + provider: LOCAL_EMBEDDING_PROVIDER, + model: LOCAL_EMBEDDING_MODEL, + baseUrl, + requestUrl: `${baseUrl}/embeddings`, + dimensions: LOCAL_EMBEDDING_DIMENSIONS, + // The proxy holds the per-install service token; read at request time by the + // caller, which is why it is not carried here. + token: null, + labelInputs: true, + }; +} + +async function cloudEmbedder(): Promise { + const endpoint = cloudEmbeddingsUrl(); + return { + source: "cloud", + provider: CLOUD_EMBEDDING_PROVIDER, + model: CLOUD_EMBEDDING_MODEL, + baseUrl: embeddingsBaseUrlOf(endpoint), + requestUrl: endpoint, + dimensions: CLOUD_EMBEDDING_DIMENSIONS, + token: await resolveClawaiToken(), + labelInputs: false, + }; +} + +/** + * The embedder this box indexes and searches with right now. + * + * THE FIELD SWITCH IS READ HERE, not only on the default path. `"off"` in + * `CLAWAI_CLOUD_EMBEDDINGS_KEY` exists so a box already in a customer's hands + * can be taken off the cloud embedder without an update, whatever its plan says + * — and it used to be consulted only inside `probeCloudEmbeddings`, i.e. only + * where nothing had been pinned. Since the automatic promotion writes a pin at + * the first boot of every linked, paid box, that is precisely the population the + * lever no longer reached: support switched it off and the owner's documents + * kept going to the cloud. The switch now wins over a stored `cloud` too. + * + * It does NOT rewrite the pin. The lever is a support action and reversible; the + * index it leaves behind reads `mismatched` and draws the card's existing "Run a + * full reindex", exactly as any other embedder change does, and clearing the key + * puts the box back where the owner left it. + * + * @param fallback what an unpinned box uses, when the caller has already worked + * the cloud-defaults verdict out and would otherwise pay for it twice. + */ +export async function resolveMemoryEmbedder(fallback?: EmbeddingSource): Promise { + const pinned = await readEmbedderPin(); + const source = pinned ?? fallback ?? (await defaultEmbedderSource()); + if (source !== "cloud") return localEmbedder(); + return (await cloudEmbeddingsSwitchedOff()) ? localEmbedder() : await cloudEmbedder(); +} + +/** Trailing slashes are not part of an address. */ +function normalised(url: string): string { + return url.replace(/\/+$/, ""); +} + +/** + * THE FENCE. The owner's own documents are the body of these requests, so there + * are exactly two addresses this box will send them to and nothing composes a + * third: + * + * - the loopback proxy in front of the embedder on this device, and + * - the ClawBox AI embeddings endpoint the image was built with, which is the + * box's own subscription account. + * + * Checked at the moment the socket is opened and not only where the choice was + * made — a fence with one gate is a fence. Off either of them the index would be + * quietly shipping the customer's files to a third party, so it refuses instead. + */ +export function embedEndpointAllowed(source: EmbeddingSource, baseUrl: string): boolean { + if (source === "local") return isLoopbackBaseUrl(baseUrl); + // An empty endpoint on either side is not an address and must never compare + // equal to one: `cloudEmbeddingsUrl()` answers "" on a box whose built-in + // route did not pass its own trust boundary, and "" === "" would have turned + // that refusal into a pass. + const allowed = normalised(embeddingsBaseUrlOf(cloudEmbeddingsUrl())); + if (!allowed || !normalised(baseUrl)) return false; + return normalised(baseUrl) === allowed; +} + +/** + * Refuse an endpoint that is neither, in the words the switch shows. + * + * Its own function because two callers need the same refusal for the same + * reason: the index before it embeds, and `switchToCloudEmbeddings` before it + * records a cloud that is not ClawBox AI. + */ +export function assertEmbedEndpointAllowed(source: EmbeddingSource, baseUrl: string): void { + if (embedEndpointAllowed(source, baseUrl)) return; + throw new Error( + source === "local" + ? "The embedder endpoint is not on this device." + : "The embedder endpoint is not this box's ClawBox AI account.", + ); +} diff --git a/src/lib/memory-index-local.ts b/src/lib/memory-index-local.ts index db8bf589f..c5761d3e1 100644 --- a/src/lib/memory-index-local.ts +++ b/src/lib/memory-index-local.ts @@ -38,17 +38,27 @@ import path from "path"; import { DATA_DIR, get as configGet, set as configSet } from "@/lib/config-store"; import type { DatabaseSync as DatabaseSyncType } from "node:sqlite"; import { openSqlite } from "@/lib/openclaw-session-store"; -import { getEmbedProvisioningStatus, getEmbedProxyBaseUrl } from "@/lib/embed-server"; -import { isLoopbackBaseUrl } from "@/lib/embed-runtime-ids"; +import { getEmbedProvisioningStatus } from "@/lib/embed-server"; import { getLocalAiToken } from "@/lib/local-ai-token"; +import { + clawaiCredentialGeneration, + clawaiCredentialRefused, + noteClawaiCredentialRefused, + proxyRefusedClawaiCredential, +} from "@/lib/harness/credentials"; +import { + assertEmbedEndpointAllowed, + embedderUsable, + resolveMemoryEmbedder, + type ResolvedEmbedder, +} from "@/lib/memory-embedder"; import { EXTRACT_ROOT, MAX_DOCUMENT_BYTES, extractDocuments, newWalkBudget, walkFiles } from "@/lib/memory-extract"; import { isInside } from "@/lib/file-guard"; import { readConfig as readOpenclawConfig, runOpenclawConfigSetBatch } from "@/lib/openclaw-config"; import { EXTRA_PATHS_CONFIG_PATH, INDEXABLE_EXTENSIONS, - LOCAL_EMBEDDING_MODEL, - LOCAL_EMBEDDING_PROVIDER, + LOCAL_EMBEDDING_DIMENSIONS, MEMORY_SHARD_SOURCES_KEY, extraPathsOf, stringList, @@ -99,15 +109,41 @@ const CHUNK_OVERLAP = 150; const EMBED_BATCH_INPUTS = 16; /** - * The ceiling on the whole index. + * THE CEILING IS A MEMORY BUDGET, so it is derived from the budget and not + * written down as a number. + * + * Every chunk costs `dimensions x 4` bytes of float32 twice over — once in the + * sqlite blob and once in the vector cache a search loads — and the two + * embedders this box can use are not the same width: Qwen3-Embedding-0.6B is + * 1,024 dimensions (4 KB a chunk) and the ClawBox AI cloud model is 3,072 + * (12 KB a chunk). The old flat 20,000 was sized against the first of those — + * ~80 MB of vectors over ~25 MB of source text, which is a great deal of + * personal notes and still fits beside the agent on an Orin — and the cloud + * arm tripled the width underneath it, which would have been ~234 MiB of + * vectors in one contiguous allocation on a box with 7.4 GB shared with a + * local model and a 2 GB embedder. + * + * So the BUDGET is the constant now and the ceiling follows the embedder. The + * number is exactly what 20,000 chunks of the on-device model weigh, so that + * arm is unchanged to the chunk and the cloud arm lands at 6,666. + */ +const MAX_LOCAL_INDEX_CHUNKS = 20_000; + +const INDEX_VECTOR_BUDGET_BYTES = MAX_LOCAL_INDEX_CHUNKS * LOCAL_EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT; + +/** + * How many chunks this box will hold for an embedder of this width. * - * Qwen3-Embedding-0.6B is 1,024 dimensions, so a chunk costs 4 KB of float32: - * 20,000 chunks is ~80 MB of vectors over ~25 MB of source text, which is a - * great deal of personal notes and still fits beside the agent on an Orin. * Reaching it is reported, never silent — a cap nobody is told about reads as - * "everything is indexed". + * "everything is indexed". A width that is not a positive number (nothing + * resolves one today, but the field is carried rather than computed here) + * falls back to the on-device ceiling rather than to zero, because a ceiling + * of zero is an index that refuses every file. */ -export const MAX_INDEX_CHUNKS = 20_000; +export function maxIndexChunks(dimensions: number): number { + if (!Number.isFinite(dimensions) || dimensions <= 0) return MAX_LOCAL_INDEX_CHUNKS; + return Math.max(1, Math.floor(INDEX_VECTOR_BUDGET_BYTES / (dimensions * Float32Array.BYTES_PER_ELEMENT))); +} /** One embeddings request's budget. The unit may be cold on the first call. */ const EMBED_TIMEOUT_MS = 120_000; @@ -132,12 +168,102 @@ const MAX_INDEXABLE_BYTES = MAX_DOCUMENT_BYTES; * file that cannot be READ is the other kind and is counted and stepped over. */ export class EmbeddingUnavailableError extends Error { - constructor(detail: string) { + /** + * The far side said "not now" rather than "no": a rate limit or a server + * fault, which the SAME request may well survive a moment later. Carried on + * the error rather than decided by the retry loop, because only the place + * that read the status knows. + */ + readonly retryable: boolean; + /** What `Retry-After` asked for, in milliseconds, when it asked for anything. */ + readonly retryAfterMs: number | null; + constructor(detail: string, retryable = false, retryAfterMs: number | null = null) { super(`The embedding model did not answer: ${detail}`); this.name = "EmbeddingUnavailableError"; + this.retryable = retryable; + this.retryAfterMs = retryAfterMs; } } +/** + * A REBUILD IS ~1,250 REQUESTS TO A RATE-LIMITED ENDPOINT, and it starts by + * emptying the store. + * + * Over the network a transient refusal is the EXPECTED case: one `429` or one + * `503` at request 500 of a full rebuild used to end the pass with the tables + * already emptied, and memory search then answered nothing until some later + * scheduled pass happened to succeed — on a box with no armed slot, + * indefinitely. So a batch the far side refused with "not now" is asked again, + * a bounded number of times, with a widening wait it will honour `Retry-After` + * over. + * + * THE CLOUD ARM ONLY, deliberately. On the loopback proxy the commonest refusal + * is the MemAvailable wake guard's 502, and that is not a hiccup — it is the box + * saying it cannot hold the model right now, which the pass is meant to respect + * by ending. Retrying it would spend the guard's own answer three times over and + * push the wake through on a device that had just refused it. A refusal that is + * not transient at all (a 4xx that is not 429 — a bad model id, a lapsed plan) + * is not retried on either arm: the next attempt would be refused the same way + * and the pass would take three times as long to say so. + */ +const EMBED_RETRY_ATTEMPTS = 3; +const EMBED_RETRY_BASE_MS = 1_000; +/** No `Retry-After` may hold one batch longer than this. */ +const EMBED_RETRY_MAX_WAIT_MS = 30_000; +/** + * …and none may make it shorter than this. + * + * `Retry-After: 0` is a legal header and `err.retryAfterMs ?? wait` read the + * zero as a number rather than as "nothing asked for", so a rate-limited + * endpoint answering it was asked three times with NO pause between them — + * precisely the hammer the backoff exists to prevent, aimed at the endpoint that + * had just asked for room. + */ +export const EMBED_RETRY_MIN_WAIT_MS = 1_000; + +/** + * How long ONE interactive search may take, end to end. + * + * Sized against the caller that times it: `memory_shard_search` abandons the + * call at 60 s (`mcp/tools/memory.ts`), so past that the box is spending a cold + * llama.cpp wake on an answer nobody is waiting for any more — and the wake it + * started carries on regardless, which is what makes the NEXT search warm. The + * search route combines this with the caller's own `request.signal`. + */ +export const MEMORY_SEARCH_DEADLINE_MS = 60_000; + +/** Is this status the far side saying "not now"? */ +function retryableStatus(status: number): boolean { + return status === 429 || status >= 500; +} + +/** `Retry-After` as milliseconds — seconds or an HTTP date — or null. */ +function retryAfterMs(res: Response): number | null { + const raw = res.headers.get("retry-after")?.trim(); + if (!raw) return null; + const seconds = Number(raw); + if (Number.isFinite(seconds) && seconds >= 0) return Math.min(seconds * 1000, EMBED_RETRY_MAX_WAIT_MS); + const at = Date.parse(raw); + if (!Number.isFinite(at)) return null; + return Math.min(Math.max(0, at - Date.now()), EMBED_RETRY_MAX_WAIT_MS); +} + +/** Wait, unless the pass is being abandoned — in which case say so at once. */ +async function pauseBeforeRetry(ms: number, signal: AbortSignal | undefined): Promise { + if (signal?.aborted) throw new IndexPassAbortedError(); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(new IndexPassAbortedError()); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + /** The pass was abandoned — the two-hour budget, or a web server going down. */ export class IndexPassAbortedError extends Error { constructor() { @@ -298,6 +424,15 @@ async function openIndexForWrite(): Promise { * answering nothing about a database that is entirely intact. Nothing on this * path writes a row. */ +/** Is there a store at all? A stat, so asking cannot create one. */ +async function indexStoreExists(): Promise { + try { + return (await fs.stat(LOCAL_INDEX_PATH)).isFile(); + } catch { + return false; + } +} + async function openIndexForRead(): Promise { try { if (!(await fs.stat(LOCAL_INDEX_PATH)).isFile()) return null; @@ -346,27 +481,49 @@ function countOf(db: IndexDb, sql: string, ...params: unknown[]): number { * provisioning step, before anything has been embedded. A dimension change is * caught on its own, where it becomes known — see `assertDimension`. */ -export function localEmbeddingIdentity(): string { +export function embedderIdentity(embedder: ResolvedEmbedder): string { return crypto .createHash("sha256") - .update(`${LOCAL_EMBEDDING_PROVIDER}|${LOCAL_EMBEDDING_MODEL}|${getEmbedProxyBaseUrl()}|v${SCHEMA_VERSION}`) + .update(`${embedder.provider}|${embedder.model}|${embedder.baseUrl}|v${SCHEMA_VERSION}`) .digest("hex") .slice(0, 16); } +/** The identity of the embedder this box uses right now. */ +export async function localEmbeddingIdentity(): Promise { + return embedderIdentity(await resolveMemoryEmbedder()); +} + /** * Record what the index is built for, without building anything. * - * The Hermes half of `switchToLocalEmbeddings`. On OpenClaw that call points an - * external client at this box's embedder; here there is no external client to - * point — ClawBox is the client — so the write that remains is the one fact - * that would otherwise be missing: which model the vectors about to be written - * belong to. + * The Hermes half of `switchToLocalEmbeddings`/`switchToCloudEmbeddings`. On + * OpenClaw those calls point an external client at an embedder; here there is + * no external client to point — ClawBox is the client — so the write that + * remains is the one fact that would otherwise be missing: which model the + * vectors about to be written belong to. + * + * IT NEVER STAMPS OVER AN INDEX THAT HOLDS SOMETHING. The vectors on disk + * belong to the embedder that wrote them, and a switch is exactly when that + * stops being the one this returns: stamping the new embedder over them would + * report `valid` — the shared parser's `healthy` — for an index whose rows are + * the wrong width for every query, so search would answer nothing behind a + * green panel. Left alone, the identity reads `mismatched`, the card draws the + * amber "Run a full reindex", and the full pass every switch posts rebuilds it. */ export async function stampLocalEmbeddingIdentity(): Promise { + // AND IT NEVER CREATES THE STORE. `openIndexForRead`'s own invariant is that + // a status read and a search must not leave a database behind on a box whose + // owner never switched the feature on, and this call reaches that box now: + // the automatic cloud promotion runs at BOOT, unattended, where it used to be + // an owner pressing a switch. With no store there is nothing to stamp either + // — `identityOf` answers `missing`, and the first pass stamps what it wrote. + if (!(await indexStoreExists())) return; + const identity = await localEmbeddingIdentity(); const db = await openIndexForWrite(); try { - metaSet(db, "identity", localEmbeddingIdentity()); + if (countOf(db, "SELECT COUNT(*) AS n FROM chunks") > 0) return; + metaSet(db, "identity", identity); metaSet(db, "schema_version", SCHEMA_VERSION); } finally { db.close(); @@ -384,10 +541,10 @@ function sourceListKey(sources: readonly string[]): string { return crypto.createHash("sha256").update(JSON.stringify([...sources].sort())).digest("hex").slice(0, 16); } -function identityOf(db: IndexDb, sources: readonly string[]): "valid" | "missing" | "mismatched" { +function identityOf(db: IndexDb, sources: readonly string[], identity: string): "valid" | "missing" | "mismatched" { const stored = metaGet(db, "identity"); if (!stored) return "missing"; - if (stored !== localEmbeddingIdentity()) return "mismatched"; + if (stored !== identity) return "mismatched"; if (countOf(db, "SELECT COUNT(*) AS n FROM chunks") > 0) return "valid"; // ZERO CHUNKS IS TWO DIFFERENT STATES, and only one of them is wrong. An // index emptied by a rebuild whose first embed failed must not read `valid`, @@ -453,51 +610,172 @@ function identityOf(db: IndexDb, sources: readonly string[]): "valid" | "missing // ─── embedding ─────────────────────────────────────────────────────────────── /** - * Embed a batch through THIS BOX'S OWN PROXY, never llama-server directly. + * Embed a batch with the embedder this box is pointed at — and at exactly one + * of the two addresses that is allowed to be. + * + * LOCAL goes through THIS BOX'S OWN PROXY, never llama-server directly. Going + * through `/setup-api/local-ai/embed/v1` is what wakes the unit on the first + * request (and re-arms the idle stop that puts it away again), what restores + * the Qwen3 query instruction that keeps recall from quietly degrading, and + * what trims an input too long for the server's batch. A client that talked to + * port 8081 would have to reimplement all three and would get one of them + * subtly wrong. + * + * CLOUD goes to the ClawBox AI embeddings endpoint with the box's own `claw_` + * bearer — the same request the OpenClaw edition's core makes from + * `memory.search.remote.*`, and the same one `probeCloudEmbeddings` proved the + * box can make before the switch was offered. No `input_type` there: it is an + * unknown field on an OpenAI-shaped route, which is why the OpenClaw arm unsets + * those two keys rather than leaving them. * - * Going through `/setup-api/local-ai/embed/v1` is what wakes the unit on the - * first request (and re-arms the idle stop that puts it away again), what - * restores the Qwen3 query instruction that keeps recall from quietly - * degrading, and what trims an input too long for the server's batch. A client - * that talked to port 8081 would have to reimplement all three and would get - * one of them subtly wrong. + * Anywhere else is refused — see `embedEndpointAllowed`. */ async function embedBatch( texts: readonly string[], inputType: "query" | "document", signal: AbortSignal | undefined, + embedder: ResolvedEmbedder, +): Promise { + let wait = EMBED_RETRY_BASE_MS; + // A QUERY IS SOMEBODY WAITING, and the rebuild's budget is the wrong one for + // it: three attempts of up to `EMBED_TIMEOUT_MS` with two waits of up to 30 s + // is about seven minutes inside a search the MCP tool abandons at 60 s. A + // rebuild retries because the alternative is an emptied index; a query that + // cannot be embedded now is answered now, and the person asks again. + const attempts = inputType === "query" ? 1 : EMBED_RETRY_ATTEMPTS; + for (let attempt = 1; ; attempt += 1) { + try { + return await embedBatchOnce(texts, inputType, signal, embedder); + } catch (err) { + const transient = err instanceof EmbeddingUnavailableError && err.retryable; + if (!transient || attempt >= attempts) throw err; + const pause = Math.max( + EMBED_RETRY_MIN_WAIT_MS, + (err as EmbeddingUnavailableError).retryAfterMs ?? wait, + ); + console.warn( + `[memory-index] the embedder answered "not now" (attempt ${attempt}/${attempts}); retrying in ${Math.round(pause / 100) / 10}s`, + ); + await pauseBeforeRetry(pause, signal); + wait = Math.min(wait * 2, EMBED_RETRY_MAX_WAIT_MS); + } + } +} + +/** One request, one answer. The retry above is what decides to ask again. */ +async function embedBatchOnce( + texts: readonly string[], + inputType: "query" | "document", + signal: AbortSignal | undefined, + embedder: ResolvedEmbedder, ): Promise { if (texts.length === 0) return []; // The owner's document text is about to become an HTTP body, so where it is - // going is checked rather than assumed. `getEmbedProxyBaseUrl()` is built - // from `CLAWBOX_LOCAL_AI_PROXY_BASE_URL`/`PORT` and is loopback on every box; - // this is what keeps that true if either ever becomes settable from anywhere - // less trustworthy. Off-box, the index would be quietly shipping the - // customer's files to a third party — so it refuses instead. - const endpoint = getEmbedProxyBaseUrl(); - if (!isLoopbackBaseUrl(endpoint)) { - throw new EmbeddingUnavailableError("the embedder endpoint is not on this device"); + // going is checked rather than assumed — at the moment the socket is opened, + // not only where the choice was made. + try { + assertEmbedEndpointAllowed(embedder.source, embedder.baseUrl); + } catch (err) { + throw new EmbeddingUnavailableError(err instanceof Error ? err.message : String(err)); } + if (!embedderUsable(embedder)) { + // Pointed at the cloud with no credential on the box. Said out loud rather + // than quietly embedded on this box instead: that would write vectors from + // another model into an index stamped for this one. + throw new EmbeddingUnavailableError("this box holds no ClawBox AI credential for the cloud embedder"); + } + const bearer = embedder.source === "cloud" ? embedder.token : getLocalAiToken(); + // Snapshotted BEFORE the request, so a refusal about a credential the box has + // since replaced is dropped rather than remembered against the new one. + const generation = clawaiCredentialGeneration(); let res: Response; try { - res = await fetch(`${endpoint}/embeddings`, { + res = await fetch(embedder.requestUrl, { method: "POST", + // A REDIRECT IS NOT A DESTINATION THIS BOX CHECKED. The fence above judges + // `embedder.baseUrl`, and `fetch` follows a 307/308 on its own — with the + // POST method and the body, which here is the owner's document or query + // text, carried to wherever the answer pointed. Manual makes the redirect + // the ANSWER, so it lands in the `!res.ok` branch below and the pass is + // told the embedder did not answer, which is exactly what happened. + redirect: "manual", headers: { "content-type": "application/json", - authorization: `Bearer ${getLocalAiToken()}`, + authorization: `Bearer ${bearer}`, }, - body: JSON.stringify({ model: LOCAL_EMBEDDING_MODEL, input: [...texts], input_type: inputType }), + body: JSON.stringify({ + model: embedder.model, + input: [...texts], + ...(embedder.labelInputs ? { input_type: inputType } : {}), + }), signal: AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(EMBED_TIMEOUT_MS)]), }); } catch (err) { if (signal?.aborted) throw new IndexPassAbortedError(); - throw new EmbeddingUnavailableError(err instanceof Error ? err.message : String(err)); + // THE SOCKET FAILING IS THE COMMONEST TRANSIENT FAILURE, and it never + // reaches an HTTP status: a reset connection, a DNS blip, a TLS error or the + // `EMBED_TIMEOUT_MS` abort all arrive here. Constructed non-retryable, one + // of them at request 200 of a 420-request rebuild ended the pass with + // `DELETE FROM chunks` already done — no index at all until some later + // scheduled pass happened to succeed. The caller's own abort is separated + // one line above, so this cannot swallow a cancelled pass. + // + // The CLOUD ARM ONLY, the same fence the status rule uses: on the loopback + // proxy a failure to connect is the embed unit being down or refusing the + // wake, which the pass is meant to respect by ending rather than by dialling + // a socket that is not there twice more. + throw new EmbeddingUnavailableError( + err instanceof Error ? err.message : String(err), + embedder.source === "cloud", + ); } if (!res.ok) { + // A CREDENTIAL THE PROXY ITSELF NAMES AS THE PROBLEM is remembered here, the + // same two lines the picture and voice paths already use. Without them + // nothing on the EMBEDDING path ever armed `clawaiCredentialRefused()`, so a + // box whose credential the proxy refuses — revoked, re-minted on another + // device, lost in a migration — and whose owner uses neither of those + // features kept reporting `semanticAvailable: true` and `mode: active` while + // every pass failed and every search threw: the guard was written and its + // input never arrived. + // + // It has to be the proxy saying so, and `proxyRefusedClawaiCredential` is + // the one place that judges it: 401 `missing_token` / 403 `invalid_token` + // and nothing else. A bare 401/403 can come from an edge rule or an + // interception proxy, and remembering one of those would send a customer + // with a good credential to re-pair their box. NOT the proxy's PLAN gate + // either, which is also a 403 (`clawai-cloud-defaults-state.ts`: "TTS is + // Max-only on the proxy, which answers 403 to Free and Pro") — a plan that + // does not cover the cloud embedder is a fact about the ACCOUNT, and this + // memo is the one the box acts on by telling the owner to re-link a device + // whose credential is perfectly good. 402 is not on the wire here at all: + // it is what ClawBox's OWN routes answer (`refusePaidPlan`) and what the + // portal answers to a device poll, never what the proxy sends. + // Both bodies are cancelled on the way out: the helper reads the clone + // only for a 401/403, so a 429 or a 5xx — the statuses the cloud arm + // retries up to three times — would otherwise leave two unread bodies + // holding their connection out of the fetch pool the next batch needs. + const refusalCopy = embedder.source === "cloud" ? res.clone() : null; + try { + if (refusalCopy && (await proxyRefusedClawaiCredential(refusalCopy))) { + await noteClawaiCredentialRefused(res.status, generation); + } + } finally { + // Together, never one after the other: a clone tees the stream, and a + // teed branch's cancel settles only once BOTH branches are cancelled. + await Promise.all([ + refusalCopy?.body?.cancel().catch(() => {}), + res.body?.cancel().catch(() => {}), + ]); + } // 502 here is the wake being refused — most often the MemAvailable guard // in `ensureLocalAiReady`, which is a real answer and not a bug: the box // is too busy to hold the model right now. - throw new EmbeddingUnavailableError(`HTTP ${res.status}`); + throw new EmbeddingUnavailableError( + `HTTP ${res.status}`, + embedder.source === "cloud" && retryableStatus(res.status), + retryAfterMs(res), + ); } let payload: unknown; try { @@ -587,7 +865,7 @@ export interface LocalIndexPassResult { chunks: number; /** Files that could not be read or embedded. Counted, never fatal. */ failures: number; - /** True when MAX_INDEX_CHUNKS stopped the pass short. */ + /** True when the chunk ceiling (`maxIndexChunks`) stopped the pass short. */ capped: boolean; } @@ -770,6 +1048,17 @@ export async function runLocalIndexPass( signal?: AbortSignal, onProgress?: LocalIndexProgressReporter, ): Promise { + // ONE reading of the embedder for the whole pass. Resolved before the store is + // opened, so a pass cannot embed its first files with one embedder and its + // last with another — a switch landing mid-pass would leave an index of two + // widths under a single identity. + const embedder = await resolveMemoryEmbedder(); + const identityNow = embedderIdentity(embedder); + // And ONE ceiling, from that embedder's width — the vectors this pass is + // about to write are the ones the budget is about. Resolved with the + // embedder for the same reason: a ceiling that moved mid-pass would cap the + // last files of a run against a different budget from its first. + const ceiling = maxIndexChunks(embedder.dimensions); const db = await openIndexForWrite(); try { // THE SCAN COMES FIRST, before anything is emptied. A `full` pass wipes the @@ -781,7 +1070,7 @@ export async function runLocalIndexPass( const sources = await readLocalSources(); const scan = await scanSources(sources, signal); - const identity = identityOf(db, sources); + const identity = identityOf(db, sources, identityNow); const schema = metaGet(db, "schema_version"); const rebuild = mode === "full" || identity === "mismatched" || schema !== SCHEMA_VERSION; if (rebuild) { @@ -802,7 +1091,7 @@ export async function runLocalIndexPass( // a path about to spend thousands, it is the same call that wakes the // unit for them, and a refusal ends the pass (`EmbeddingUnavailableError`) // with the index it was going to replace still on disk. - await embedBatch([REBUILD_PROBE_TEXT], "document", signal); + await embedBatch([REBUILD_PROBE_TEXT], "document", signal, embedder); // The identity goes WITH the rows it describes. Stamping it here — before // a single vector had been written — meant a rebuild whose first embed // failed left a valid identity over an empty index, which the shared @@ -924,7 +1213,7 @@ export async function runLocalIndexPass( const pieces = chunkText(text); const existing = row ? countOf2(sql.chunksFor, entry.file) : 0; - if (chunkCount - existing + pieces.length > MAX_INDEX_CHUNKS) { + if (chunkCount - existing + pieces.length > ceiling) { // `continue`, not `break`: one large document early in the scan must not // shut out the thousand small ones behind it that still fit. capped = true; @@ -937,7 +1226,7 @@ export async function runLocalIndexPass( vectors = []; for (let at = 0; at < pieces.length; at += EMBED_BATCH_INPUTS) { throwIfAborted(signal); - vectors.push(...await embedBatch(pieces.slice(at, at + EMBED_BATCH_INPUTS), "document", signal)); + vectors.push(...await embedBatch(pieces.slice(at, at + EMBED_BATCH_INPUTS), "document", signal, embedder)); } } catch (err) { // The embedder is the shared resource; a failure there is not this @@ -1048,7 +1337,7 @@ export async function runLocalIndexPass( // in one transaction — five autocommits is five fsyncs for six short rows. db.exec("BEGIN"); try { - metaSet(db, "identity", localEmbeddingIdentity()); + metaSet(db, "identity", identityNow); metaSet(db, "built_at", String(Date.now())); // Everything this pass had to ACCOUNT FOR, which is not the same as what // it could open: a document the extractor refused never becomes an @@ -1076,7 +1365,7 @@ export async function runLocalIndexPass( } if (capped) { console.warn( - `[memory-index] the index reached its ${MAX_INDEX_CHUNKS}-chunk ceiling; some files in this pass were not indexed`, + `[memory-index] the index reached its ${ceiling}-chunk ceiling; some files in this pass were not indexed`, ); } if (scan.unreadableSources.size) { @@ -1229,13 +1518,39 @@ function assertDimension(db: IndexDb, sample: Float32Array | undefined): boolean * comment for why it is worth impersonating rather than parallel-implementing. */ export async function localMemoryStatusJson(): Promise { - const [sources, provisioning] = await Promise.all([ + const [sources, provisioning, embedder] = await Promise.all([ readLocalSources(), getEmbedProvisioningStatus().catch(() => null), + resolveMemoryEmbedder(), ]); - // The model on disk is what makes semantic search POSSIBLE; whether it is - // awake right now is not the question — the proxy wakes it on demand. - const ready = provisioning?.installed === true; + // What makes semantic search POSSIBLE, per embedder. On this box that is the + // model being on disk — whether it is awake right now is not the question, + // the proxy wakes it on demand. In the cloud it is the box holding the + // credential the endpoint wants AND that credential not having been REFUSED: + // a credential the proxy has rejected (401 `missing_token` / 403 + // `invalid_token` — revoked, re-minted elsewhere, lost in a migration) sits in + // the store looking exactly like a working one, so "a token is present" + // reported `semanticAvailable: true` and `mode: active` over a box where every + // pass failed and every search threw. `clawaiCredentialRefused()` is the fact + // the rest of the box already keeps for the picture and microphone paths; a + // refusal expires on its own, so this reports degraded only while one is + // actually on record. The GGUF need not be there at all, and reading + // `installed` on the cloud arm would report a perfectly good cloud index as + // degraded on a box that never downloaded 639 MB it does not use. + // + // WHAT THIS STILL DOES NOT COVER, said out loud rather than implied: the + // proxy's PLAN gate is a 403 too, and it names the plan rather than the + // credential, so `proxyRefusedClawaiCredential` refuses it — correctly, since + // a refused credential is the one the box tells the owner to re-link. A plan + // that lapses after the embedder was pinned to the cloud therefore still + // reads as available here. Closing that needs a fact about the ACCOUNT, from + // the portal poll that already reads the plan, not a wider reading of a + // status on the embedding path. + const ready = + embedder.source === "cloud" + ? embedderUsable(embedder) && clawaiCredentialRefused() === null + : provisioning?.installed === true; + const identityNow = embedderIdentity(embedder); // ONE object literal for both cases, so the "there is no index yet" shape and // the real one cannot drift into disagreeing about a field name. // @@ -1243,7 +1558,7 @@ export async function localMemoryStatusJson(): Promise { // cached count is only true while every pass finishes: one that dies partway // — the embedder refusing a wake is the common case — leaves the tables and // the recorded totals describing different indexes, and this probe is what - // the panel believes. Measured at ~0.1 ms warm on a store at its 20,000-chunk + // the panel believes. Measured at ~0.1 ms warm on a store at its chunk // ceiling (sqlite answers both from `chunks_by_path`), which is not a price // worth paying in honesty. const db = await openIndexForRead(); @@ -1252,8 +1567,8 @@ export async function localMemoryStatusJson(): Promise { agentId: "main", scan: { totalFiles: db ? Number(metaGet(db, "scan_total_files") ?? 0) : 0 }, status: { - provider: LOCAL_EMBEDDING_PROVIDER, - model: LOCAL_EMBEDDING_MODEL, + provider: embedder.provider, + model: embedder.model, files: db ? countOf(db, "SELECT COUNT(*) AS n FROM files") : 0, chunks: db ? countOf(db, "SELECT COUNT(*) AS n FROM chunks") : 0, dbPath: LOCAL_INDEX_PATH, @@ -1271,7 +1586,7 @@ export async function localMemoryStatusJson(): Promise { // wizard's own stamp leaves too. `identityOf` is what tells an index // that is empty because there was nothing to index from one that is // empty because the work did not happen. - indexIdentity: { status: db ? identityOf(db, sources) : "missing" }, + indexIdentity: { status: db ? identityOf(db, sources, identityNow) : "missing" }, }, }, }; @@ -1307,11 +1622,14 @@ let cacheRelease: ReturnType | null = null; /** * How long the vectors stay resident after the last search. * - * At the 20,000-chunk ceiling the cache is ~82 MB of float32, and this box has + * At the ceiling the cache is ~78 MiB of float32 WHICHEVER embedder this box + * uses — that is what `INDEX_VECTOR_BUDGET_BYTES` is, and the ceiling is + * derived from it rather than fixed at 20,000, so the cloud model's 3,072 + * dimensions buy fewer chunks instead of three times the memory. This box has * 7.4 GB shared with a local model and a 2 GB embedder. The embedder itself is * put away after ten idle minutes (`LOCAL_AI_IDLE_TIMEOUT_MS`); a search index - * that pinned 82 MB for the life of the process while the model it belongs to - * was handing memory back would be the odd one out. + * that pinned ~78 MiB for the life of the process while the model it belongs + * to was handing memory back would be the odd one out. */ const VECTOR_CACHE_IDLE_MS = 10 * 60 * 1000; @@ -1337,9 +1655,10 @@ export function _resetLocalMemoryCacheForTests(): void { /** * Every vector as ONE Float32Array. * - * 20,000 chunks of 1,024 floats is a 20M-multiply scan, which is tens of - * milliseconds in JS — no vector extension, no approximate index, and nothing - * to keep in step with the rows. + * The ceiling is a memory budget, so the scan is about the same size on either + * embedder: 20,000 chunks of 1,024 floats, or 6,666 of 3,072, is ~20M + * multiplies — tens of milliseconds in JS, with no vector extension, no + * approximate index and nothing to keep in step with the rows. * * `iterate`, not `all`, and the bytes go STRAIGHT into the destination. `all` * built 20,000 row objects each holding a 4 KB buffer before one could be @@ -1406,6 +1725,11 @@ export async function searchLocalMemory( // Clamped HERE, not only in the two callers: the top-k loop indexes // `best[best.length - 1]` and a limit of zero makes that `best[-1]`. const want = Math.max(1, Math.min(50, Math.trunc(limit) || 1)); + // The SAME embedder the last pass wrote with, or the query lands in a space + // the stored vectors do not live in. A switch that has not been rebuilt for + // yet answers nothing here (the widths differ), which is what the card's + // mismatched fingerprint and its "Run a full reindex" are about. + const embedder = await resolveMemoryEmbedder(); const db = await openIndexForRead(); if (!db) return []; try { @@ -1414,7 +1738,7 @@ export async function searchLocalMemory( // for exactly that reason — and loading 82 MB of vectors needs nothing from // it. Started first, the wake covers the whole cold load, which is the // worst case this feature has: the agent's first search after a restart. - const embedding = embedBatch([text], "query", signal); + const embedding = embedBatch([text], "query", signal, embedder); // A handler on a DERIVED promise, attached before anything can throw: the // load below can return early or fail, and an in-flight rejection with // nobody listening takes the whole process down. `await embedding` still diff --git a/src/lib/memory-shard-state.ts b/src/lib/memory-shard-state.ts index aeba23693..9eec57069 100644 --- a/src/lib/memory-shard-state.ts +++ b/src/lib/memory-shard-state.ts @@ -65,6 +65,17 @@ export const LOCAL_EMBEDDING_MODEL = "qwen3-embedding-0.6b"; * OpenAI-compatible client, pointed at ClawBox's local-AI proxy. */ export const LOCAL_EMBEDDING_PROVIDER = "openai-compatible"; +/** + * How wide a vector the model on this box answers with. + * + * Qwen3-Embedding-0.6B is 1,024 dimensions. It is a CONSTANT here rather than a + * number in a comment because the index's memory budget is a function of it — + * `maxIndexChunks` in `memory-index-local.ts` — and the cloud model is three + * times as wide. A width written in prose beside a ceiling derived from it is + * how the ceiling came to be sized for a model the box had stopped using. + */ +export const LOCAL_EMBEDDING_DIMENSIONS = 1024; + /** The engine, for the sentences that name it ("Qwen 3 via llama.cpp"). */ export const LOCAL_EMBEDDING_ENGINE = "llama.cpp"; @@ -85,6 +96,20 @@ export const EXTRA_PATHS_CONFIG_PATH = "memory.search.extraPaths"; */ export const MEMORY_SHARD_SOURCES_KEY = "memory_shard_sources"; +/** + * Where the index is embedded on that same edition: `"cloud"` | `"local"`. + * + * The counterpart of OpenClaw's `memory.search.provider`/`.remote.baseUrl`, for + * the same reason the line above exists — the thing that INDEXES owns the + * setting, and on this SKU that is ClawBox. A WORD and never an address: see + * `src/lib/memory-embedder.ts`, which is the only reader and writer of it. + * + * Absent is not "local". Absent is "nobody has pinned this box", which the + * cloud-defaults resolver answers — the cloud whenever the box's subscription + * covers it (the owner's ruling of 2026-09-18). + */ +export const MEMORY_SHARD_EMBEDDER_KEY = "memory_shard_embedder"; + /** Documents ClawBox can turn into Markdown for the indexer. */ export const EXTRACTABLE_EXTENSIONS = [".pdf", ".docx", ".odt", ".rtf", ".txt"] as const; @@ -135,9 +160,25 @@ export interface EmbedderChoiceStatus { /** Where the index is embedded right now. */ source: EmbeddingSource; /** - * False on the edition where ClawBox itself is the indexer: its embedder - * client refuses any endpoint that is not loopback, because the owner's - * document text is the request body there. + * …and whether that is WRITTEN DOWN, or is the default rule speaking. + * + * The half a caller cannot work out afterwards, and the one the wizard's last + * step turns on: with no choice recorded the box already embeds in the cloud + * wherever its subscription covers it, so a wizard that read `source: "cloud"` + * as "nothing to do" finished without writing the pin — and the next boot's + * automatic promotion then wrote it and rebuilt an index that was already + * correct. False from a server that predates the field, which is the safe + * direction: it makes the caller post. + */ + recorded: boolean; + /** + * This box can point its memory index at the ClawBox AI cloud at all. + * + * True on every edition since 2026-09-18. It used to be false where ClawBox + * itself is the indexer, because that client accepted only a loopback + * endpoint; it now accepts exactly two — the loopback proxy and the ClawBox + * AI endpoint the image was built with — so the fence is still a fence and + * the owner's choice is the same one on both editions. */ cloudSupported: boolean; /** Linked, on a paid ClawBox AI plan, and the cloud embedder answered this box. */ @@ -219,6 +260,10 @@ export function parseEmbedderChoiceStatus(raw: unknown): EmbedderChoiceStatus | if (r.source !== "cloud" && r.source !== "local") return null; return { source: r.source, + // An older server sends no such field, and `false` there is the direction + // that costs nothing: the caller posts a choice that was already made + // rather than skipping the one write that records it. + recorded: r.recorded === true, cloudSupported: r.cloudSupported === true, cloudAvailable: r.cloudAvailable === true, // An older server sends no reason at all; null is the generic note. diff --git a/src/lib/memory-shard.ts b/src/lib/memory-shard.ts index 7b960f4a3..d613ad2fa 100644 --- a/src/lib/memory-shard.ts +++ b/src/lib/memory-shard.ts @@ -19,10 +19,18 @@ import { runOpenclawConfigUnset, } from "@/lib/openclaw-config"; import { readLocalSources, stampLocalEmbeddingIdentity, writeLocalSources } from "@/lib/memory-index-local"; +import { + assertEmbedEndpointAllowed, + defaultEmbedderSource, + readEmbedderPin, + writeEmbedderPin, +} from "@/lib/memory-embedder"; import { getEmbedProxyBaseUrl } from "@/lib/embed-server"; import { CLOUD_EMBEDDING_MODEL, CLOUD_EMBEDDING_PROVIDER, + cloudEmbeddingsSwitchedOff, + embeddingEndpointParseable, embeddingsBaseUrlOf, } from "@/lib/clawai-cloud-embeddings"; import { getLocalAiToken } from "@/lib/local-ai-token"; @@ -33,8 +41,10 @@ import { LOCAL_EMBEDDING_PROVIDER, MEMORY_SHARD_ENABLED_KEY, MEMORY_SHARD_SETUP_KEY, + type EmbeddingSource, type MemorySource, } from "@/lib/memory-shard-state"; +import { isLoopbackBaseUrl } from "@/lib/embed-runtime-ids"; /** The owner's consent for the index to run. Off on a new box. */ export async function getMemoryShardEnabled(): Promise { @@ -265,6 +275,11 @@ export async function switchToLocalEmbeddings(): Promise { // belong to, so a later embedder change is caught rather than silently // degrading search. if (openclawIsAbsent()) { + // The pin FIRST, then the stamp: the stamp reads where the box is pointed, + // and in the other order it would record the embedder being switched away + // from. Both halves are idempotent, so a failure between them asks for a + // retry that costs nothing. + await writeEmbedderPin("local"); await stampLocalEmbeddingIdentity(); return; } @@ -326,15 +341,85 @@ export async function readEmbeddingChoice(): Promise<{ provider: string | null; return { provider: at(["provider"]), model: at(["model"]), baseUrl: at(["remote", "baseUrl"]) }; } +/** + * Where the index is embedded right now — the ONE reader both editions use. + * + * Two arms because the thing that INDEXES owns the setting: OpenClaw's own + * `memory.search.remote.baseUrl` where the core is the embedding client, and + * ClawBox's `memory_shard_embedder` pin where ClawBox is. + * + * `recorded` is the half a caller cannot work out afterwards: FALSE means + * nothing has been written down and the answer is the default rule speaking — + * the cloud whenever this box's subscription covers it (the owner's ruling of + * 2026-09-18). The automatic promotion reads exactly that to decide whether it + * still has a write to make, and it is why a box that has been promoted once is + * not promoted — and reindexed — again at every boot. + */ +export interface EmbeddingPlacement { + source: EmbeddingSource; + recorded: boolean; +} + +export async function readEmbeddingPlacement(fallback?: EmbeddingSource): Promise { + if (openclawIsAbsent()) { + const pinned = await readEmbedderPin(); + if (pinned) { + // THE SUPPORT LEVER OUTRANKS THE PIN, on this reader as it already does on + // `resolveMemoryEmbedder`. With `clawai_cloud_embeddings: "off"` every + // embed and every search goes to the loopback proxy, and this reader + // answering "cloud" from the pin alone made the embedder card draw the + // cloud hint, preselect the cloud segment and the provider GET report + // `source: "cloud"` beside `cloudAvailable: false` — the very shape that + // route's own comment records as a defect and fixed for the unpinned path, + // while `clawkeep-memory.ts` read the resolved embedder and said "local". + // The pin is NOT rewritten: the lever is a support action and reversible, + // and clearing the key must put the box back where the owner left it. + if (pinned === "cloud" && (await cloudEmbeddingsSwitchedOff())) { + return { source: "local", recorded: true }; + } + return { source: pinned, recorded: true }; + } + return { source: fallback ?? (await defaultEmbedderSource()), recorded: false }; + } + const { baseUrl } = await readEmbeddingChoice(); + // No endpoint at all is the on-device answer: the only thing this box points + // at without one is its own embedder, and claiming "cloud" over an unset key + // would make the automatic default skip the write that puts it right. + if (!baseUrl) return { source: "local", recorded: false }; + if (isLoopbackBaseUrl(baseUrl)) return { source: "local", recorded: true }; + // AN ADDRESS NOTHING CAN EMBED THROUGH IS NOT A PLACEMENT. `memory.search` + // is a file a restored backup, a hand edit or a half-finished migration can + // leave holding a truncated URL or a `file:` scheme. Reading one of those as + // "recorded: cloud" is the false-success shape on the one reader the automatic + // default asks before it decides whether it still owes this box a write: it + // would skip the write, and the box would keep a `memory.search` it cannot + // embed with and no surface saying so. Unrecorded hands it back to the default + // rule, which writes a configuration that works. + // + // CLEARTEXT OFF THE DEVICE IS NOT ONE OF THOSE, and refusing it here was a + // regression of its own: on this arm the client is OPENCLAW'S, not ClawBox's + // (see `embeddingEndpointParseable`), so an owner's own LAN llama.cpp is a + // real placement and overwriting it would be the configuration change. + if (!embeddingEndpointParseable(baseUrl)) { + console.warn( + "[memory-shard] the configured memory embedder endpoint is not an address anything can embed through; treating it as unset", + ); + return { source: "local", recorded: false }; + } + return { source: "cloud", recorded: true }; +} + /** * Point the memory index at the ClawBox AI cloud embedder. * - * The mirror of {@link switchToLocalEmbeddings}, and deliberately NOT available - * on the edition where ClawBox itself is the indexer: `memory-index-local.ts` - * refuses an embedder endpoint that is not loopback, on purpose — the owner's - * document text is the request body there — and a default is not allowed to - * open that fence. The caller checks; this refuses too, because a fence with - * one gate is a fence. + * The mirror of {@link switchToLocalEmbeddings}, and since 2026-09-18 available + * on BOTH editions. Where ClawBox itself is the indexer there is no external + * client to point, so what is written is the word `cloud` in ClawBox's own store + * — never the address and never a copy of the credential: `memory-index-local.ts` + * re-derives both per request, from the image's environment and the box's own + * credential store. The fence did not move, it widened by exactly one address: + * the endpoint offered here is checked against the ClawBox AI endpoint this box + * knows, because a fence with one gate is a fence. * * The two `*InputType` keys are REMOVED rather than left: they exist to make * OpenClaw label each request so ClawBox's own proxy can restore the Qwen3 @@ -348,7 +433,12 @@ export async function readEmbeddingChoice(): Promise<{ provider: string | null; */ export async function switchToCloudEmbeddings(endpoint: string, token: string): Promise { if (openclawIsAbsent()) { - throw new Error("This edition indexes memory on the box itself and cannot use a cloud embedder."); + if (!token.trim()) throw new Error("The ClawBox AI credential is missing."); + assertEmbedEndpointAllowed("cloud", embeddingsBaseUrlOf(endpoint)); + // Same order as the local arm, for the same reason. + await writeEmbedderPin("cloud"); + await stampLocalEmbeddingIdentity(); + return; } const home = embeddingConfigHome(await installedOpenclawVersion()); await runOpenclawConfigSetBatch([ diff --git a/src/tests/components/memory-shard-wizard.test.tsx b/src/tests/components/memory-shard-wizard.test.tsx index 7beca95a9..bc9963349 100644 --- a/src/tests/components/memory-shard-wizard.test.tsx +++ b/src/tests/components/memory-shard-wizard.test.tsx @@ -251,7 +251,7 @@ describe("MemoryShardWizard", () => { // already indexing in the cloud onto the 640 MB model on this box: the // settings card has never allowed that (its own `blocked` exempts a box // already on the cloud) and Index now here would post the move. - const done = await atProvisionStep({ source: "cloud", cloudSupported: true, cloudAvailable: false, localInstalled: false }); + const done = await atProvisionStep({ source: "cloud", recorded: true, cloudSupported: true, cloudAvailable: false, localInstalled: false }); expect(screen.getByTestId("memory-shard-source-cloud")).toHaveAttribute("aria-checked", "true"); expect(screen.getByTestId("memory-shard-source-cloud")).not.toBeDisabled(); expect(screen.queryByTestId("memory-shard-source-cloud-unavailable")).toBeNull(); @@ -259,13 +259,43 @@ describe("MemoryShardWizard", () => { fireEvent.click(screen.getByTestId("memory-shard-index-now")); await waitFor(() => expect(done).toHaveBeenCalled()); expect(posts.find((p) => p.url === "/setup-api/embed/install")).toBeUndefined(); - // …and no switch is asked for either: the index is already there, and the - // route's switch re-checks the very probe that just answered false, so - // posting it would turn a hiccup into a 409 over an index that was never - // going to move. The wizard finishes on what the box already has. + // …and no switch is asked for either: the choice is RECORDED, the index is + // already there, and the route's switch re-checks the very probe that just + // answered false — so posting it would turn a hiccup into a 409 over an + // index that was never going to move. The wizard finishes on what the box + // already has. expect(posts.find((p) => p.url === "/setup-api/clawkeep/memory/provider")).toBeUndefined(); }); + it("STILL posts on a box that embeds in the cloud by DEFAULT, because nothing is written down", async () => { + // The whole of H-1. Since the owner's ruling of 2026-09-18 the GET answers + // `source: "cloud"` from an UNWRITTEN default — every freshly onboarded, + // subscribed box — so a wizard that read `source` alone as "nothing to do" + // finished without the pin and without the owner mark. The first pass then + // indexed the owner's whole Documents folder in the cloud, correctly and at + // their expense, and the very next web-server restart's automatic promotion + // saw an unrecorded box, wrote the pin and asked for a FULL rebuild: hours + // of re-embedding paid for twice, with memory search answering nothing + // throughout. The POST is what records the choice, so the cloud path takes + // it whenever `recorded` is false. + const done = await atProvisionStep({ source: "cloud", recorded: false, cloudSupported: true, cloudAvailable: true, localInstalled: false }); + fireEvent.click(screen.getByTestId("memory-shard-index-now")); + await waitFor(() => expect(done).toHaveBeenCalled()); + expect(posts.find((p) => p.url === "/setup-api/clawkeep/memory/provider")?.body).toEqual({ source: "cloud" }); + // And still no 639 MB download: the cloud model needs none. + expect(posts.find((p) => p.url === "/setup-api/embed/install")).toBeUndefined(); + }); + + it("posts on a server too old to say whether anything is recorded", async () => { + // `parseEmbedderChoiceStatus` reads a missing `recorded` as false, which is + // the safe direction: posting a choice that was already made costs one + // idempotent write, skipping the one that records it costs a full reindex. + const done = await atProvisionStep({ source: "cloud", cloudSupported: true, cloudAvailable: true, localInstalled: false }); + fireEvent.click(screen.getByTestId("memory-shard-index-now")); + await waitFor(() => expect(done).toHaveBeenCalled()); + expect(posts.find((p) => p.url === "/setup-api/clawkeep/memory/provider")?.body).toEqual({ source: "cloud" }); + }); + it("says WHY the cloud model cannot be picked and what makes it available, reason by reason", async () => { // "not available on this box right now" named neither the cause nor the // cure, so the commonest one of all — a box with no ClawBox AI credential diff --git a/src/tests/routes/memory-shard-provider.test.ts b/src/tests/routes/memory-shard-provider.test.ts index 62f43760c..db076198c 100644 --- a/src/tests/routes/memory-shard-provider.test.ts +++ b/src/tests/routes/memory-shard-provider.test.ts @@ -4,17 +4,21 @@ * (2026-09-15). * * Pinned: the GET's "on offer" means what the cloud-defaults resolver means by - * it (linked, paid, answered), and the edition that indexes on the box itself - * never offers the cloud; a POST with no body is still the model on this box; - * the cloud is refused where it cannot work rather than written and left to - * find nothing; and every refusal lands before anything is written. + * it (linked, paid, answered), on BOTH editions since 2026-09-18 — the one + * where ClawBox itself indexes included; a box nobody has pinned reads as the + * cloud wherever the subscription covers it (the owner's ruling, same day); a + * POST with no body is still the model on this box; the cloud is refused where + * it cannot work rather than written and left to find nothing; and every + * refusal lands before anything is written. */ import { beforeEach, describe, expect, it, vi } from "vitest"; const h = vi.hoisted(() => ({ owner: true, absent: false, - baseUrl: "http://127.0.0.1/setup-api/local-ai/embed/v1" as string | null, + placement: { source: "local", recorded: true } as { source: string; recorded: boolean }, + /** What an unpinned box was told the default is, as the route passed it. */ + placementFallback: undefined as unknown, choiceThrows: false, routeReady: true, factsThrow: false, @@ -35,9 +39,10 @@ vi.mock("@/lib/openclaw-config", async (importOriginal) => ({ vi.mock("@/lib/memory-shard", () => ({ switchToLocalEmbeddings: () => h.switchLocal(), switchToCloudEmbeddings: (...a: unknown[]) => h.switchCloud(...a), - readEmbeddingChoice: async () => { + readEmbeddingPlacement: async (fallback?: unknown) => { if (h.choiceThrows) throw new Error("EACCES: permission denied, open 'openclaw.json'"); - return { provider: "openai-compatible", model: "m", baseUrl: h.baseUrl }; + h.placementFallback = fallback; + return h.placement; }, })); vi.mock("@/lib/clawai-cloud-choice", () => ({ noteOwnerChoice: (...a: unknown[]) => h.note(...a) })); @@ -65,7 +70,8 @@ const route = () => import("@/app/setup-api/clawkeep/memory/provider/route"); beforeEach(() => { h.owner = true; h.absent = false; - h.baseUrl = "http://127.0.0.1/setup-api/local-ai/embed/v1"; + h.placement = { source: "local", recorded: true }; + h.placementFallback = undefined; h.choiceThrows = false; h.routeReady = true; h.factsThrow = false; @@ -81,21 +87,42 @@ beforeEach(() => { describe("GET", () => { it("answers the model on this box, the cloud on offer, and whether the GGUF is here", async () => { const res = await (await route()).GET(); - expect(await res.json()).toEqual({ source: "local", cloudSupported: true, cloudAvailable: true, cloudReason: null, localInstalled: false }); + expect(await res.json()).toEqual({ source: "local", recorded: true, cloudSupported: true, cloudAvailable: true, cloudReason: null, localInstalled: false }); }); it("calls an index pointed off the box the cloud", async () => { - h.baseUrl = "https://ai.clawbox.com/v1"; + h.placement = { source: "cloud", recorded: true }; h.installed = true; const body = await (await (await route()).GET()).json(); expect(body).toMatchObject({ source: "cloud", localInstalled: true }); }); - it("offers no cloud on the edition that indexes on the box itself, and does not ask the resolver", async () => { + it("offers the cloud on the edition that indexes on the box itself too", async () => { + // It used to answer `cloudSupported: false, cloudReason: "edition"` here, + // because that index accepted only a loopback endpoint. It now accepts the + // ClawBox AI one as well, so the switch is the same switch on both SKUs. h.absent = true; const body = await (await (await route()).GET()).json(); - expect(body).toEqual({ source: "local", cloudSupported: false, cloudAvailable: false, cloudReason: "edition", localInstalled: false }); - expect(h.facts).not.toHaveBeenCalled(); + expect(body).toEqual({ source: "local", recorded: true, cloudSupported: true, cloudAvailable: true, cloudReason: null, localInstalled: false }); + }); + + it("hands an unpinned box the same verdict as the default, so the card cannot offer what it already uses", async () => { + // The owner's ruling of 2026-09-18: with no choice made, the cloud IS the + // embedder wherever the subscription covers it — not a preselection. + h.absent = true; + h.placement = { source: "cloud", recorded: false }; + const body = await (await (await route()).GET()).json(); + expect(h.placementFallback).toBe("cloud"); + expect(body).toMatchObject({ source: "cloud", cloudAvailable: true }); + }); + + it("leaves an unpinned box on its own model, with the reason, when the cloud cannot serve it", async () => { + h.absent = true; + h.facts.mockImplementation(async () => ({ linked: false, entitlement: null, embeddingsSupported: true, embeddingsRouteReady: false })); + h.placement = { source: "local", recorded: false }; + const body = await (await (await route()).GET()).json(); + expect(h.placementFallback).toBe("local"); + expect(body).toMatchObject({ source: "local", cloudSupported: true, cloudAvailable: false, cloudReason: "not_linked" }); }); it("carries WHY the cloud is not on offer, in the resolver's own vocabulary", async () => { @@ -193,13 +220,14 @@ describe("POST — the ClawBox AI cloud", () => { expect(h.switchLocal).not.toHaveBeenCalled(); }); - it("is refused on the edition that indexes on the box itself", async () => { + it("is written on the edition that indexes on the box itself, like any other", async () => { h.absent = true; const res = await (await route()).POST(post({ source: "cloud" })); - expect(res.status).toBe(409); - expect((await res.json()).kind).toBe("cloud_unsupported"); - expect(h.switchCloud).not.toHaveBeenCalled(); - expect(h.note).not.toHaveBeenCalled(); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ source: "cloud", model: "text-embedding-3-large" }); + expect(h.switchCloud).toHaveBeenCalledWith("https://ai.clawbox.com/v1/embeddings", "claw_test_token"); + expect(h.note).toHaveBeenCalledWith("embeddings"); + expect(h.invalidate).toHaveBeenCalledTimes(1); }); it("is refused where the cloud embedder is not on offer, before any pin is written", async () => { diff --git a/src/tests/routes/memory-shard-search.test.ts b/src/tests/routes/memory-shard-search.test.ts index cee297d44..eedcb7105 100644 --- a/src/tests/routes/memory-shard-search.test.ts +++ b/src/tests/routes/memory-shard-search.test.ts @@ -15,15 +15,28 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; * stated here as a test rather than only in a comment. */ -const { absent, enabled, search } = vi.hoisted(() => ({ +const { absent, enabled, search, MEMORY_SEARCH_DEADLINE_MS } = vi.hoisted(() => ({ absent: { value: true }, enabled: { value: true }, - search: vi.fn(async () => [{ path: "Documents/lease.md", snippet: "The deposit is two months' rent.", score: 0.82 }]), + search: vi.fn(async (_query: string, _limit: number, _signal?: AbortSignal) => [ + { path: "Documents/lease.md", snippet: "The deposit is two months' rent.", score: 0.82 }, + ]), + /** + * The box's OWN bound on an interactive search, which the route combines with + * the caller's `request.signal`. Hoisted with the rest because the mock + * factory runs before this file's own statements do, and named once so the + * deadline case below cannot pass over whatever number the route happened to + * hand `AbortSignal.timeout`. + */ + MEMORY_SEARCH_DEADLINE_MS: 60_000, })); vi.mock("@/lib/openclaw-config", () => ({ openclawIsAbsent: () => absent.value })); vi.mock("@/lib/memory-shard", () => ({ getMemoryShardEnabled: async () => enabled.value })); -vi.mock("@/lib/memory-index-local", () => ({ searchLocalMemory: search })); +vi.mock("@/lib/memory-index-local", () => ({ + searchLocalMemory: search, + MEMORY_SEARCH_DEADLINE_MS, +})); import { NextRequest } from "next/server"; import { GET } from "@/app/setup-api/clawkeep/memory/search/route"; @@ -55,6 +68,36 @@ describe("who may search", () => { }); }); + it("hands the search a deadline of the box's own, not only the caller's signal", async () => { + // A client that simply waits gives the box no bound at all, and ONE embed + // attempt can hold the request for the whole 120 s embed timeout while the + // MCP tool that asked has already abandoned the call at 60 s. The abort is a + // bound on the ANSWER, not a cancellation of the wake it started: that + // carries on and makes the next search warm. + // + // The BOX'S OWN half is what is proven here, and it takes a controlled + // signal to prove it: "an AbortSignal that is not aborted yet" is equally + // true of a route that forwarded `request.signal` alone, and Vitest's fake + // timers do not advance `AbortSignal.timeout`, so waiting the deadline out + // is not an option either. Standing a controller in for the timeout is. + const timeoutController = new AbortController(); + const timeout = vi + .spyOn(AbortSignal, "timeout") + .mockReturnValue(timeoutController.signal); + try { + await call("?q=deposit"); + const signal = search.mock.calls[0]?.[2]; + expect(signal).toBeInstanceOf(AbortSignal); + expect(signal?.aborted).toBe(false); + // The box's own budget, not some other number the route happened to have. + expect(timeout).toHaveBeenCalledWith(MEMORY_SEARCH_DEADLINE_MS); + timeoutController.abort(); + expect(signal?.aborted).toBe(true); + } finally { + timeout.mockRestore(); + } + }); + it("refuses while the owner's switch is off", async () => { enabled.value = false; const res = await call("?q=deposit"); diff --git a/src/tests/unit/clawai-cloud-defaults-apply.test.ts b/src/tests/unit/clawai-cloud-defaults-apply.test.ts index 24116c168..f929cf6ab 100644 --- a/src/tests/unit/clawai-cloud-defaults-apply.test.ts +++ b/src/tests/unit/clawai-cloud-defaults-apply.test.ts @@ -43,11 +43,27 @@ const openclawAbsent = vi.fn(() => false); vi.mock("@/lib/openclaw-config", () => ({ openclawIsAbsent: () => openclawAbsent() })); const switchToCloud = vi.fn(async () => {}); -const embeddingChoice = vi.fn(async () => ({ provider: "openai-compatible", model: "q", baseUrl: "http://127.0.0.1:3000/setup-api/local-ai/embed/v1" })); +/** + * Where the index is embedded, and whether that is WRITTEN DOWN. + * + * The second half is what a box nobody has pinned turns on: there the answer is + * the default rule — the cloud wherever the subscription covers it — and the + * applier still owes it the one write that records it, plus the rebuild. + */ +const placement = vi.fn(async (_fallback?: unknown) => ({ source: "local" as string, recorded: false })); +/** + * THE PIN, on the edition where ClawBox is the indexer: the one thing on such a + * box that records a choice about the embedder. The legacy + * `memory_embeddings_choice_source: "owner"` mark does not, because the wizard + * that wrote it could only ever post one answer. + */ +const embedderPin = vi.fn(async () => null as string | null); +vi.mock("@/lib/memory-embedder", () => ({ readEmbedderPin: () => embedderPin() })); + const shardEnabled = vi.fn(async () => true); vi.mock("@/lib/memory-shard", () => ({ getMemoryShardEnabled: () => shardEnabled(), - readEmbeddingChoice: () => embeddingChoice(), + readEmbeddingPlacement: (...a: unknown[]) => placement(...(a as [])), switchToCloudEmbeddings: (...a: unknown[]) => switchToCloud(...(a as [])), })); @@ -90,7 +106,8 @@ beforeEach(() => { syncChannel.mockResolvedValue(true); startIndex.mockResolvedValue({ accepted: true }); voiceState.mockResolvedValue({ choice: "auto" }); - embeddingChoice.mockResolvedValue({ provider: "openai-compatible", model: "q", baseUrl: "http://127.0.0.1:3000/setup-api/local-ai/embed/v1" }); + placement.mockResolvedValue({ source: "local", recorded: false }); + embedderPin.mockResolvedValue(null); }); describe("applyClawaiCloudDefaults", () => { @@ -150,7 +167,7 @@ describe("applyClawaiCloudDefaults", () => { // Both halves of transcription already say cloud, so the channel sync has // nothing to write — see the loop's note on why it is still asked. syncChannel.mockResolvedValue(false); - embeddingChoice.mockResolvedValue({ provider: "openai-compatible", model: "text-embedding-3-large", baseUrl: "https://clawbox.test/api/ai" }); + placement.mockResolvedValue({ source: "cloud", recorded: true }); voiceProbe.mockResolvedValueOnce({ config: { tts: { provider: "openai", providers: { openai: { apiKey: "claw_test", baseUrl: "https://clawbox.test/api/ai" } } } }, probe: { providerConfigured: false, commandPresent: false, engineInstalled: false, engineNames: [] }, @@ -315,11 +332,99 @@ describe("readCloudDefaultsStatus", () => { }); }); - it("calls the index local on the edition that indexes on the box", async () => { + it("puts the edition that indexes on the box on the cloud too, where it used to say 'edition'", async () => { + // Until 2026-09-18 this answered `{ source: local, target: local, reason: + // "edition" }` and never even probed: ClawBox's own index accepted a + // loopback endpoint and nothing else. It now accepts this box's ClawBox AI + // account as well, so the SKU is no longer a reason to keep a subscriber + // off what they pay for. + openclawAbsent.mockReturnValue(true); + placement.mockResolvedValue({ source: "cloud", recorded: false }); + const status = await readCloudDefaultsStatus(); + expect(status.capabilities.embeddings).toEqual({ source: "cloud", target: "cloud", ownerChoice: false, reason: null }); + }); + + it("records the cloud, and rebuilds, on an unpinned box that is already embedding there", async () => { + // WHO THAT BOX IS: one whose owner has never been asked, whose default + // resolves to the cloud, and whose index — if it has one — was built by the + // model on the box. The pin is what stops this happening again at every + // boot, and the rebuild is what the move needs. A box that finished the + // wizard on the cloud AFTER 2026-09-18 is not in this population: the + // wizard posts, so its choice is recorded and the guard below holds. + openclawAbsent.mockReturnValue(true); + placement.mockResolvedValue({ source: "cloud", recorded: false }); + const applied = await applyClawaiCloudDefaults(); + expect(applied.moved).toContain("embeddings"); + expect(switchToCloud).toHaveBeenCalledWith("https://clawbox.test/api/ai/embeddings", "claw_test"); + expect(startIndex).toHaveBeenCalledWith("full", "manual"); + }); + + it("promotes a Hermes box carrying the LEGACY owner mark, pin and rebuild and all", async () => { + // THE POPULATION THIS UPDATE LANDS ON. Every Hermes box that finished the + // Memory Shard wizard before 2026-09-18 carries + // `memory_embeddings_choice_source: "owner"` and no pin — the wizard's last + // step POSTed the model on this box because it was the only thing the route + // could offer there, and the route marks every pick as the owner's. + // + // Honouring that mark left those boxes in the worst of the three states: + // `resolveMemoryEmbedder` ignores it and started sending their documents to + // the cloud, while this applier honoured it and so never wrote the pin or + // asked for the rebuild — the stored identity stayed the 1,024-dimension + // local one, the card read `mismatched`, and memory search answered nothing + // at all until some later pass happened to rebuild. + openclawAbsent.mockReturnValue(true); + store.set("memory_embeddings_choice_source", "owner"); + embedderPin.mockResolvedValue(null); + placement.mockResolvedValue({ source: "cloud", recorded: false }); + + const applied = await applyClawaiCloudDefaults(); + expect(applied.moved).toContain("embeddings"); + expect(switchToCloud).toHaveBeenCalledWith("https://clawbox.test/api/ai/embeddings", "claw_test"); + expect(startIndex).toHaveBeenCalledWith("full", "manual"); + // …and the stale mark is handed back to the applier, so the card stops + // reporting an owner choice for a move the owner never made. + expect(store.get("memory_embeddings_choice_source")).toBe("auto"); + }); + + it("leaves a box whose owner really picked the model ON THIS BOX exactly where it is", async () => { + // The other half of the same rule. A pick made through the settings card + // writes the mark AND the pin, and a pin is the thing on this edition that + // records a choice. Nothing here may move it. openclawAbsent.mockReturnValue(true); + store.set("memory_embeddings_choice_source", "owner"); + embedderPin.mockResolvedValue("local"); + placement.mockResolvedValue({ source: "local", recorded: true }); + + const applied = await applyClawaiCloudDefaults(); + expect(applied.moved).not.toContain("embeddings"); + expect(switchToCloud).not.toHaveBeenCalled(); + expect(startIndex).not.toHaveBeenCalled(); + expect(store.get("memory_embeddings_choice_source")).toBe("owner"); + }); + + it("still honours the mark on OpenClaw, where the wizard could offer both", async () => { + // The edition-specific twist is exactly that: specific. Where OpenClaw is + // the embedding client there is no pin, the mark is the whole answer, and a + // box whose owner chose the model on it stays on it. + openclawAbsent.mockReturnValue(false); + store.set("memory_embeddings_choice_source", "owner"); + placement.mockResolvedValue({ source: "local", recorded: true }); + const status = await readCloudDefaultsStatus(); - expect(status.capabilities.embeddings).toEqual({ source: "local", target: "local", ownerChoice: false, reason: "edition" }); - // And the probe is never even asked for: the answer could not change it. - expect(routeReady).not.toHaveBeenCalled(); + expect(status.capabilities.embeddings).toEqual({ + source: "local", target: "local", ownerChoice: true, reason: "owner", + }); + const applied = await applyClawaiCloudDefaults(); + expect(applied.moved).not.toContain("embeddings"); + expect(switchToCloud).not.toHaveBeenCalled(); + }); + + it("leaves a box whose cloud embedder is already recorded alone", async () => { + openclawAbsent.mockReturnValue(true); + placement.mockResolvedValue({ source: "cloud", recorded: true }); + const applied = await applyClawaiCloudDefaults(); + expect(applied.moved).not.toContain("embeddings"); + expect(switchToCloud).not.toHaveBeenCalled(); + expect(startIndex).not.toHaveBeenCalled(); }); }); diff --git a/src/tests/unit/clawai-cloud-embeddings.test.ts b/src/tests/unit/clawai-cloud-embeddings.test.ts index d40315328..23c3ff11f 100644 --- a/src/tests/unit/clawai-cloud-embeddings.test.ts +++ b/src/tests/unit/clawai-cloud-embeddings.test.ts @@ -40,17 +40,21 @@ import { } from "@/lib/clawai-cloud-embeddings"; const REAL_URL = process.env.CLAWBOX_AI_EMBEDDINGS_URL; +const REAL_INSECURE = process.env.CLAWBOX_AI_EMBEDDINGS_INSECURE; beforeEach(() => { store.clear(); forgetCloudEmbeddingsProbe(); delete process.env.CLAWBOX_AI_EMBEDDINGS_URL; + delete process.env.CLAWBOX_AI_EMBEDDINGS_INSECURE; }); afterEach(() => { vi.unstubAllGlobals(); if (REAL_URL === undefined) delete process.env.CLAWBOX_AI_EMBEDDINGS_URL; else process.env.CLAWBOX_AI_EMBEDDINGS_URL = REAL_URL; + if (REAL_INSECURE === undefined) delete process.env.CLAWBOX_AI_EMBEDDINGS_INSECURE; + else process.env.CLAWBOX_AI_EMBEDDINGS_INSECURE = REAL_INSECURE; }); function answer(body: unknown, status = 200) { @@ -127,14 +131,40 @@ describe("cloudEmbeddingsUrl", () => { expect(embeddingsBaseUrlOf("https://clawbox.test/api/ai/")).toBe("https://clawbox.test/api/ai"); }); - it("keeps plain http, which is the LAN staging contract", () => { - // Deliberately NOT narrowed to loopback: the override exists so a staging - // image can be pointed at a proxy on a trusted LAN. HTTPS outside that is - // the operator's to honour — see the trust boundary on `usableEndpoint`. + it("refuses plain http off the device, whatever it is pointed at", () => { + // The request carries this box's `claw_` bearer and the owner's document + // text as its body, so cleartext leaving the device is CWE-319. The earlier + // rule accepted any `http:` and left "only on a trusted LAN" to the + // operator — a promise nothing here could check, over the one failure that + // cannot be undone once it has happened. process.env.CLAWBOX_AI_EMBEDDINGS_URL = "http://staging.lan:8080/v1/embeddings"; + expect(cloudEmbeddingsUrl()).toBe("https://clawbox.test/api/ai/embeddings"); + }); + + it("keeps plain http on THIS DEVICE, where it leaves no interface", () => { + process.env.CLAWBOX_AI_EMBEDDINGS_URL = "http://127.0.0.1:8080/v1/embeddings"; + expect(cloudEmbeddingsUrl()).toBe("http://127.0.0.1:8080/v1/embeddings"); + process.env.CLAWBOX_AI_EMBEDDINGS_URL = "http://localhost:8080/v1/embeddings"; + expect(cloudEmbeddingsUrl()).toBe("http://localhost:8080/v1/embeddings"); + }); + + it("keeps the LAN staging lane, but only for an image built to ask for it", () => { + // The staging contract did not go away; it became explicit, and it lives + // where the address lives — root's environment, never the device store, so + // a restored backup or a hand-edited config.json cannot turn cleartext on. + process.env.CLAWBOX_AI_EMBEDDINGS_URL = "http://staging.lan:8080/v1/embeddings"; + process.env.CLAWBOX_AI_EMBEDDINGS_INSECURE = "1"; expect(cloudEmbeddingsUrl()).toBe("http://staging.lan:8080/v1/embeddings"); }); + it("holds the built-in route to the same rule as an override", () => { + // `CLAWBOX_AI_PROXY_URL` is env-overridable too, so exempting the address + // every unconfigured box uses would have been the way round the rule above. + // Nothing usable answers "", which every caller fails closed on. + delete process.env.CLAWBOX_AI_EMBEDDINGS_URL; + expect(cloudEmbeddingsUrl()).toBe("https://clawbox.test/api/ai/embeddings"); + }); + it("refuses an override that is not an http(s) address, and falls back to its own account's route", () => { // The endpoint is the destination of a request carrying this box's bearer, // so a scheme that is not a network fetch, a string that is not a URL, and diff --git a/src/tests/unit/harness-credentials.test.ts b/src/tests/unit/harness-credentials.test.ts index e52018852..023240ce9 100644 --- a/src/tests/unit/harness-credentials.test.ts +++ b/src/tests/unit/harness-credentials.test.ts @@ -358,3 +358,63 @@ describe("a credential the ClawBox AI proxy has refused", () => { expect(mod.clawaiCredentialRefused()).toBeNull(); }); }); + +describe("the statuses the helper accepts", () => { + /** + * THE CONTRACT ITSELF, pinned where the real helper runs. + * + * `proxyRefusedClawaiCredential` had no case of its own, and a suite that + * stood a WIDER stand-in in for it (`memory-index-local.test.ts`, which cannot + * import this module — the proxy-URL re-export drags the whole Hermes adapter + * graph back through it) went on proving a behaviour the product does not + * have. These cases are what that mirror is checked against: change the table + * below and the copy there has to move with it. + */ + async function refused(status: number, body: string): Promise { + const mod = await import("@/lib/harness/credentials"); + return mod.proxyRefusedClawaiCredential(new Response(body, { status })); + } + + const CREDENTIAL = JSON.stringify({ error: { code: "invalid_token" } }); + + it("accepts the two statuses the proxy refuses a credential with, and no others", async () => { + expect(await refused(401, JSON.stringify({ error: { code: "missing_token" } }))).toBe(true); + expect(await refused(403, CREDENTIAL)).toBe(true); + // 402 IS NOT ONE OF THEM. ClawBox's own routes answer 402 + // `paid_plan_required` (`refusePaidPlan`) and so does the portal's device + // poll; the AI proxy does not speak it, and its own plan gate is a 403 + // ("TTS is Max-only on the proxy, which answers 403 to Free and Pro", + // `clawai-cloud-defaults-state.ts`). A caller that read 402 as a dead + // credential would send an owner whose token is fine to re-link the box. + expect(await refused(402, CREDENTIAL)).toBe(false); + // Nor a spent allowance, which belongs to a plan and resets on a clock. + expect(await refused(429, CREDENTIAL)).toBe(false); + expect(await refused(500, CREDENTIAL)).toBe(false); + }); + + it("needs the proxy's OWN envelope, never the status alone", async () => { + // An edge rule, a captive portal or a CDN anti-bot page all answer 403 with + // a page, and the proxy's plan gate answers it with a plan code. + expect(await refused(403, "Access denied")).toBe(false); + expect(await refused(403, JSON.stringify({ error: { code: "paid_plan_required" } }))).toBe(false); + expect(await refused(403, JSON.stringify({ error: {} }))).toBe(false); + }); + + it("fails closed on a body too big to be an error envelope", async () => { + // An interception page is exactly the response a 403 arrives with, on a + // device where memory is the scarce thing. Past the cap it is not the + // envelope we are looking for. + // + // A VALID envelope that is merely TOO BIG, so the byte cap is the only + // thing left that can answer `false`: the body used to be unparseable JSON, + // which the `JSON.parse` guard refuses on its own — the case passed with the + // cap removed. `MAX_REFUSAL_BODY_BYTES` is 8 KiB and is not exported (the + // module is the contract, not its constants), so the bound is written out + // here and asserted rather than imported. + const oversize = JSON.stringify({ + error: { code: "invalid_token", detail: "x".repeat(9 * 1024) }, + }); + expect(oversize.length).toBeGreaterThan(8 * 1024); + expect(await refused(403, oversize)).toBe(false); + }); +}); diff --git a/src/tests/unit/memory-embedder.test.ts b/src/tests/unit/memory-embedder.test.ts new file mode 100644 index 000000000..94b26d7b4 --- /dev/null +++ b/src/tests/unit/memory-embedder.test.ts @@ -0,0 +1,185 @@ +/** + * src/lib/memory-embedder.ts — WHICH embedder Memory Shard's own index uses on + * the edition where ClawBox is the indexer, and the two addresses it may send + * the owner's documents to. + * + * Pinned here: the default is the ClawBox AI cloud wherever the box's + * subscription covers it (the owner's ruling of 2026-09-18) and the model on + * this box everywhere else; a stored pin beats the default in both directions; + * the key holds a WORD, so a hand-edited or restored `data/config.json` can + * never name an endpoint; and the fence passes exactly two addresses. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const store = new Map(); +vi.mock("@/lib/config-store", () => ({ + get: async (key: string) => store.get(key), + set: async (key: string, value: unknown) => { store.set(key, value); }, +})); + +const token = vi.fn(async () => "claw_test" as string | null); +vi.mock("@/lib/harness/credentials", () => ({ + CLAWBOX_AI_PROXY_URL: "https://clawbox.test/api/ai", + resolveClawaiToken: () => token(), +})); + +vi.mock("@/lib/embed-server", () => ({ + getEmbedProxyBaseUrl: () => "http://127.0.0.1/setup-api/local-ai/embed/v1", +})); + +const facts = vi.fn(async () => ({ + linked: true, + entitlement: "pro" as string | null, + embeddingsSupported: true, + embeddingsRouteReady: true, +})); +vi.mock("@/lib/clawai-cloud-defaults", () => ({ readCloudDefaultsFacts: () => facts() })); + +import { + assertEmbedEndpointAllowed, + embedEndpointAllowed, + readEmbedderPin, + resolveMemoryEmbedder, + writeEmbedderPin, +} from "@/lib/memory-embedder"; +import { MEMORY_SHARD_EMBEDDER_KEY } from "@/lib/memory-shard-state"; +import { CLAWAI_CLOUD_EMBEDDINGS_KEY } from "@/lib/clawai-cloud-embeddings"; + +const CLOUD_ENDPOINT = "https://clawbox.test/api/ai/embeddings"; +const PROXY = "http://127.0.0.1/setup-api/local-ai/embed/v1"; + +beforeEach(() => { + store.clear(); + vi.clearAllMocks(); + token.mockResolvedValue("claw_test"); + facts.mockResolvedValue({ linked: true, entitlement: "pro", embeddingsSupported: true, embeddingsRouteReady: true }); +}); + +describe("what a box nobody has pinned embeds with", () => { + it("is the ClawBox AI cloud when the subscription covers it", async () => { + const embedder = await resolveMemoryEmbedder(); + expect(embedder.source).toBe("cloud"); + expect(embedder.requestUrl).toBe(CLOUD_ENDPOINT); + expect(embedder.token).toBe("claw_test"); + // An OpenAI-shaped route: no `input_type`, which is the field the loopback + // proxy reads and this one would 400 on. + expect(embedder.labelInputs).toBe(false); + }); + + it("is the model on this box when nothing links it to a subscription", async () => { + token.mockResolvedValue(null); + facts.mockResolvedValue({ linked: false, entitlement: null, embeddingsSupported: true, embeddingsRouteReady: false }); + const embedder = await resolveMemoryEmbedder(); + expect(embedder.source).toBe("local"); + expect(embedder.requestUrl).toBe(`${PROXY}/embeddings`); + expect(embedder.labelInputs).toBe(true); + }); + + it("is the model on this box when the cloud route does not answer", async () => { + facts.mockResolvedValue({ linked: true, entitlement: "pro", embeddingsSupported: true, embeddingsRouteReady: false }); + expect((await resolveMemoryEmbedder()).source).toBe("local"); + }); + + it("is the model on this box when the facts cannot be read at all", async () => { + facts.mockRejectedValue(new Error("the probe blew up")); + expect((await resolveMemoryEmbedder()).source).toBe("local"); + }); + + it("takes the verdict the caller already paid for rather than reading it again", async () => { + expect((await resolveMemoryEmbedder("local")).source).toBe("local"); + expect(facts).not.toHaveBeenCalled(); + }); +}); + +describe("the pin", () => { + it("beats the default in both directions", async () => { + await writeEmbedderPin("local"); + expect(await readEmbedderPin()).toBe("local"); + expect((await resolveMemoryEmbedder()).source).toBe("local"); + // And the facts are not even read: the owner has said. + expect(facts).not.toHaveBeenCalled(); + + await writeEmbedderPin("cloud"); + facts.mockResolvedValue({ linked: false, entitlement: null, embeddingsSupported: true, embeddingsRouteReady: false }); + expect((await resolveMemoryEmbedder()).source).toBe("cloud"); + }); + + it("is a WORD: anything else in the store is ignored, never used as an address", async () => { + // The state a restored backup or a hand-edited config.json can produce, and + // the reason this key holds a word at all: everything the owner has indexed + // is the body of these requests. + store.set(MEMORY_SHARD_EMBEDDER_KEY, "https://someone-elses-server.example/v1"); + expect(await readEmbedderPin()).toBeNull(); + const embedder = await resolveMemoryEmbedder(); + expect(embedder.requestUrl).toBe(CLOUD_ENDPOINT); + expect(JSON.stringify(embedder)).not.toContain("someone-elses-server"); + }); +}); + +describe("the field switch", () => { + it("takes a PINNED box off the cloud, which is the population the lever exists for", async () => { + // `clawai_cloud_embeddings: "off"` is there so a box already in a + // customer's hands can be taken off the cloud embedder without an update, + // whatever its plan says. It used to be read only inside + // `probeCloudEmbeddings`, i.e. only where nothing had been pinned — and the + // automatic promotion now writes a pin at the first boot of every linked, + // paid box, so the lever no longer reached the boxes it was written for: + // support switched it off and the owner's documents kept going out. + await writeEmbedderPin("cloud"); + store.set(CLAWAI_CLOUD_EMBEDDINGS_KEY, "off"); + const embedder = await resolveMemoryEmbedder(); + expect(embedder.source).toBe("local"); + expect(embedder.requestUrl).toBe(`${PROXY}/embeddings`); + }); + + it("is reversible, and leaves the owner's pin exactly where it was", async () => { + await writeEmbedderPin("cloud"); + store.set(CLAWAI_CLOUD_EMBEDDINGS_KEY, "off"); + await resolveMemoryEmbedder(); + expect(await readEmbedderPin()).toBe("cloud"); + store.delete(CLAWAI_CLOUD_EMBEDDINGS_KEY); + expect((await resolveMemoryEmbedder()).source).toBe("cloud"); + }); + + it("is one word and nothing else", async () => { + await writeEmbedderPin("cloud"); + for (const value of ["on", "OFF ", "", "no", true]) { + store.set(CLAWAI_CLOUD_EMBEDDINGS_KEY, value); + const expected = String(value).trim().toLowerCase() === "off" ? "local" : "cloud"; + expect((await resolveMemoryEmbedder()).source, String(value)).toBe(expected); + } + }); +}); + +describe("the width each embedder answers with", () => { + it("is carried, because the index's memory budget is a function of it", async () => { + // The chunk ceiling and the vector cache are derived from this + // (`maxIndexChunks`): the cloud model is three times as wide as the one on + // the box, and a ceiling written as a flat number next to a comment about + // 1,024 dimensions tripled the budget underneath itself. + expect((await resolveMemoryEmbedder("cloud")).dimensions).toBe(3072); + expect((await resolveMemoryEmbedder("local")).dimensions).toBe(1024); + }); +}); + +describe("the fence", () => { + it("passes this box's own proxy and this box's own ClawBox AI account, and nothing else", () => { + expect(embedEndpointAllowed("local", PROXY)).toBe(true); + expect(embedEndpointAllowed("cloud", "https://clawbox.test/api/ai")).toBe(true); + // Trailing slashes are not part of an address. + expect(embedEndpointAllowed("cloud", "https://clawbox.test/api/ai/")).toBe(true); + + expect(embedEndpointAllowed("local", "https://someone-elses-server.example/v1")).toBe(false); + expect(embedEndpointAllowed("cloud", "https://someone-elses-server.example/v1")).toBe(false); + // The right address on the wrong arm is still refused: each arm knows one. + expect(embedEndpointAllowed("local", "https://clawbox.test/api/ai")).toBe(false); + expect(embedEndpointAllowed("cloud", PROXY)).toBe(false); + }); + + it("refuses in words the switch can show", () => { + expect(() => assertEmbedEndpointAllowed("cloud", "https://someone-elses-server.example/v1")) + .toThrow(/ClawBox AI account/i); + expect(() => assertEmbedEndpointAllowed("local", "https://someone-elses-server.example/v1")) + .toThrow(/not on this device/i); + }); +}); diff --git a/src/tests/unit/memory-index-local.test.ts b/src/tests/unit/memory-index-local.test.ts index 52c8320db..c45a6bed6 100644 --- a/src/tests/unit/memory-index-local.test.ts +++ b/src/tests/unit/memory-index-local.test.ts @@ -19,7 +19,7 @@ import path from "node:path"; * real: a real sqlite store on a real temp DATA_DIR, real files on disk. */ -const { dataDir, embedCalls, embedFail, openclawConfig } = vi.hoisted(() => { +const { dataDir, embedCalls, embedFail, refusalNotes, openclawConfig, boxState } = vi.hoisted(() => { // eslint-disable-next-line @typescript-eslint/no-require-imports const nodeFs = require("node:fs") as typeof import("node:fs"); // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -29,16 +29,37 @@ const { dataDir, embedCalls, embedFail, openclawConfig } = vi.hoisted(() => { return { dataDir: nodeFs.mkdtempSync(nodePath.join(nodeOs.tmpdir(), "memory-index-data-")), /** Every text that reached the embedder, in order, across the run. */ - embedCalls: { texts: [] as string[], types: [] as string[] }, + embedCalls: { texts: [] as string[], types: [] as string[], urls: [] as string[], bearers: [] as string[] }, /** * When `status` is set, an embeddings request answers it instead of a * vector. `after` lets that many requests through first, which is how a * test reaches the embedder dying PART WAY through a rebuild — the one * state where the store really has been emptied. */ - embedFail: { status: 0, after: 0 }, + embedFail: { + status: 0, + after: 0, + retryAfter: null as string | null, + /** The refusal envelope the ClawBox AI proxy sends with a 401 or a 403. */ + body: null as string | null, + /** + * The socket itself failing, which is what a Wi-Fi drop, a DNS blip, a + * reset connection or the embed timeout look like from here. `status` + * cannot express it: those never reach an HTTP status at all. + */ + throwTimes: 0, + }, + /** What `noteClawaiCredentialRefused` was told, if anything. */ + refusalNotes: [] as number[], /** What openclaw.json holds, for the one-time carry-over after a swap. */ openclawConfig: { value: {} as unknown }, + /** The box's ClawBox AI credential, and whether the GGUF is on disk. */ + boxState: { + clawaiToken: "claw_test" as string | null, + gguf: true, + /** The status the ClawBox AI proxy refused this box's credential with. */ + clawaiRefusedStatus: null as number | null, + }, }; }); @@ -55,18 +76,67 @@ vi.mock("@/lib/config-store", async (importOriginal) => { }); vi.mock("@/lib/embed-server", () => ({ getEmbedProxyBaseUrl: () => "http://127.0.0.1/setup-api/local-ai/embed/v1", - getEmbedProvisioningStatus: async () => ({ installed: true, binaryAvailable: true, modelAvailable: true, modelBytes: 1, binPath: "", modelPath: "" }), + getEmbedProvisioningStatus: async () => ({ installed: boxState.gguf, binaryAvailable: boxState.gguf, modelAvailable: boxState.gguf, modelBytes: 1, binPath: "", modelPath: "" }), +})); +// The cloud half of the same index: where the ClawBox AI embedder is, and the +// box's own credential for it. Both are read per request by the real resolver. +vi.mock("@/lib/harness/credentials", () => ({ + CLAWBOX_AI_PROXY_URL: "https://clawbox.test/api/ai", + resolveClawaiToken: async () => boxState.clawaiToken, + // The proxy's own verdict on this box's credential. A lapsed subscription + // leaves the token in place and refuses the request, which is the state the + // status read must not call `semanticAvailable`. + clawaiCredentialRefused: () => boxState.clawaiRefusedStatus, + // The generation guard the real store keeps, so a verdict on a credential the + // box no longer holds is dropped rather than remembered. + clawaiCredentialGeneration: () => 1, + // The PROXY'S OWN identification of the credential as the problem — never the + // status alone, which an edge rule or a plan gate can also send. + // + // A DELIBERATE MIRROR of `proxyRefusedClawaiCredential` in + // `@/lib/harness/credentials`, down to the status table and the envelope it + // parses, and not a convenience stand-in: a mock that accepted one status + // more than the helper does had this suite proving a behaviour the product + // does not have. The real module cannot be imported here — it re-exports from + // `@/lib/hermes-clawai`, which is the whole Hermes adapter graph and imports + // this module's own dependencies back — so the contract is pinned where the + // real helper runs instead (`harness-credentials.test.ts`, "the statuses the + // helper accepts"), and this copy is checked against it there. + proxyRefusedClawaiCredential: async (res: Response) => { + if (res.status !== 401 && res.status !== 403) return false; + const text = await res.text().catch(() => ""); + let payload: unknown; + try { + payload = JSON.parse(text); + } catch { + return false; + } + const code = (payload as { error?: { code?: unknown } } | null)?.error?.code; + return code === "invalid_token" || code === "missing_token"; + }, + noteClawaiCredentialRefused: async (status: number) => { refusalNotes.push(status); }, +})); +vi.mock("@/lib/clawai-cloud-defaults", () => ({ + readCloudDefaultsFacts: async () => ({ + linked: boxState.clawaiToken !== null, + entitlement: "pro", + embeddingsSupported: true, + embeddingsRouteReady: boxState.clawaiToken !== null, + }), })); vi.mock("@/lib/local-ai-token", () => ({ getLocalAiToken: () => "t".repeat(64) })); vi.mock("@/lib/openclaw-config", () => ({ readConfig: async () => openclawConfig.value })); +import { writeEmbedderPin } from "@/lib/memory-embedder"; import { + EMBED_RETRY_MIN_WAIT_MS, LOCAL_INDEX_PATH, _resetLocalMemoryCacheForTests, chunkText, localEmbeddingIdentity, carryMemorySourcesTo, localMemoryStatusJson, + maxIndexChunks, readLocalSources, runLocalIndexPass, searchLocalMemory, @@ -89,18 +159,27 @@ function stubVector(text: string): number[] { } function installFetchStub(): void { - vi.stubGlobal("fetch", vi.fn(async (_url: string, init: { body: string }) => { + vi.stubGlobal("fetch", vi.fn(async (url: string, init: { body: string; headers: Record }) => { + if (embedFail.throwTimes > 0) { + embedFail.throwTimes -= 1; + throw new TypeError("fetch failed"); + } if (embedFail.status) { if (embedFail.after > 0) embedFail.after -= 1; else { const status = embedFail.status; embedFail.status = 0; - return new Response("nope", { status }); + return new Response(embedFail.body ?? "nope", { + status, + headers: embedFail.retryAfter === null ? {} : { "retry-after": embedFail.retryAfter }, + }); } } - const body = JSON.parse(init.body) as { input: string[]; input_type: string }; + const body = JSON.parse(init.body) as { input: string[]; input_type?: string; model: string }; embedCalls.texts.push(...body.input); - embedCalls.types.push(body.input_type); + embedCalls.types.push(body.input_type as string); + embedCalls.urls.push(String(url)); + embedCalls.bearers.push(init.headers.authorization); return Response.json({ data: body.input.map((text, index) => ({ index, embedding: stubVector(text) })), }); @@ -123,8 +202,23 @@ beforeEach(async () => { fs.mkdirSync(source, { recursive: true }); embedCalls.texts = []; embedCalls.types = []; + embedCalls.urls = []; + embedCalls.bearers = []; + // No subscription unless a test says otherwise, so every case below that is + // about the model on this box stays about it. The default-is-cloud rule has + // its own tests, here and in memory-embedder.test.ts. + boxState.clawaiToken = null; + boxState.gguf = true; + boxState.clawaiRefusedStatus = null; + // The store is module-level in the mock, so a pin one case writes would + // otherwise decide the embedder for every case after it. + ((await import("@/lib/config-store")) as unknown as { __store: Map }).__store.clear(); embedFail.status = 0; embedFail.after = 0; + embedFail.retryAfter = null; + embedFail.body = null; + embedFail.throwTimes = 0; + refusalNotes.length = 0; openclawConfig.value = {}; installFetchStub(); _resetLocalMemoryCacheForTests(); @@ -416,6 +510,30 @@ describe("indexing the owner's folders", () => { } }); + it("refuses a REDIRECT on the embedding request rather than following it with the body", async () => { + // THE FENCE CHECKS THE ADDRESS THIS BOX RESOLVED, and `fetch` follows a + // 307/308 by itself — with the METHOD and the BODY intact, and the body on + // this path is the owner's document text. `redirect: "manual"` is what makes + // such an answer land in the `!res.ok` branch below instead of on somebody + // else's server. + const elsewhere: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (url: string, init: { body: string; redirect?: string }) => { + if (init.redirect === "manual") { + return new Response(null, { + status: 307, + headers: { location: "https://elsewhere.example/v1/embeddings" }, + }); + } + // What the DEFAULT `fetch` does with that answer, spelled out: the same + // POST, the same body, at the address the redirect named. + elsewhere.push(...(JSON.parse(init.body) as { input: string[] }).input); + return Response.json({ data: [{ index: 0, embedding: [1, 0, 0, 0, 0.05] }] }); + })); + write("notes.md", "The deposit is two months' rent."); + await expect(runLocalIndexPass("full")).rejects.toThrow(/HTTP 307/); + expect(elsewhere).toEqual([]); + }); + it("ends the pass when the EMBEDDER will not answer, rather than reporting success", async () => { // The difference that matters: a file nobody can read is one file's // problem, and an embedder that is down is every file's. A pass that @@ -427,6 +545,322 @@ describe("indexing the owner's folders", () => { }); }); +describe("the ClawBox AI cloud embedder", () => { + it("embeds in the cloud with the box's own credential, and never through the proxy", async () => { + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + const result = await runLocalIndexPass("full"); + + expect(result.files).toBe(1); + expect(embedCalls.urls.length).toBeGreaterThan(0); + for (const url of embedCalls.urls) expect(url).toBe("https://clawbox.test/api/ai/embeddings"); + for (const bearer of embedCalls.bearers) expect(bearer).toBe("Bearer claw_test"); + // `input_type` is the loopback proxy's field — it restores Qwen3's query + // instruction from it — and an unknown one on an OpenAI-shaped route. + expect(embedCalls.types.every((type) => type === undefined)).toBe(true); + }); + + it("is what a box with a subscription and no pin uses, without being told", async () => { + // The owner's ruling of 2026-09-18: the cloud is the DEFAULT, not a + // preselection in a wizard. Nothing is stored here at all. + boxState.clawaiToken = "claw_test"; + write("notes.md", "The deposit is two months' rent."); + await runLocalIndexPass("full"); + expect(embedCalls.urls).not.toHaveLength(0); + for (const url of embedCalls.urls) expect(url).toBe("https://clawbox.test/api/ai/embeddings"); + }); + + it("indexes on the box itself when nothing links it to a subscription", async () => { + write("notes.md", "The deposit is two months' rent."); + await runLocalIndexPass("full"); + for (const url of embedCalls.urls) expect(url).toBe("http://127.0.0.1/setup-api/local-ai/embed/v1/embeddings"); + }); + + it("ends the pass rather than quietly embedding on the box when the credential is gone", async () => { + // A subscription that lapsed, or a credential the portal revoked. Falling + // back to the model on this box would write vectors from another model into + // an index stamped for the cloud one — a healthy panel over a search that + // ranks nothing. + await writeEmbedderPin("cloud"); + boxState.clawaiToken = null; + write("notes.md", "The deposit is two months' rent."); + await expect(runLocalIndexPass("full")).rejects.toThrow(/ClawBox AI credential/i); + expect(embedCalls.texts).toEqual([]); + }); + + it("searches with the same embedder the index was built by", async () => { + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + await runLocalIndexPass("full"); + embedCalls.urls = []; + embedCalls.types = []; + const hits = await searchLocalMemory("deposit", 3); + expect(hits).not.toHaveLength(0); + expect(embedCalls.urls).toEqual(["https://clawbox.test/api/ai/embeddings"]); + expect(embedCalls.types).toEqual([undefined]); + }); + + it("survives one transient refusal mid-rebuild instead of leaving the box with no index", async () => { + // A full rebuild is ~1,250 requests to a rate-limited endpoint and it + // starts by emptying the store. A single 429 at request 500 used to end the + // pass with the tables already wiped, and memory search then answered + // nothing until some later scheduled pass happened to succeed — on a box + // with no armed slot, indefinitely. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + for (let i = 0; i < 40; i += 1) write(`note-${i}.md`, `The deposit is two months' rent, note ${i}.`); + // Past the rebuild probe and into the owner's own documents. + embedFail.status = 429; + embedFail.after = 2; + embedFail.retryAfter = "0"; + + const result = await runLocalIndexPass("full"); + expect(result.files).toBe(40); + expect(result.chunks).toBeGreaterThan(0); + const hits = await searchLocalMemory("deposit", 3); + expect(hits).not.toHaveLength(0); + }); + + it("survives a dropped CONNECTION mid-rebuild, not only a refused one", async () => { + // `retryable` was set from an HTTP STATUS only, so a reset connection, a DNS + // blip, a TLS error or the 120 s embed timeout ended the pass — with the + // tables already emptied. Over the network those are at least as common as a + // 429 and they are the ones a Wi-Fi box actually sees. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + for (let i = 0; i < 40; i += 1) write(`note-${i}.md`, `The deposit is two months' rent, note ${i}.`); + embedFail.throwTimes = 1; + + const result = await runLocalIndexPass("full"); + expect(result.files).toBe(40); + expect(result.chunks).toBeGreaterThan(0); + expect(await searchLocalMemory("deposit", 3)).not.toHaveLength(0); + }); + + it("remembers a credential the PROXY named as the problem during an embed", async () => { + // L-1's residual. `localMemoryStatusJson` requires `clawaiCredentialRefused() + // === null` on the cloud arm, and nothing on the EMBEDDING path ever armed + // it — only the picture and voice paths did. So a box whose credential the + // proxy refuses — revoked, re-minted elsewhere, corrupted in a migration — + // and whose owner uses neither of those features kept reporting a healthy + // cloud index while every pass failed and every search threw. + // + // 403 AND NOT 402, which is what `proxyRefusedClawaiCredential` accepts and + // what the ClawBox AI proxy actually sends: 402 is spoken by ClawBox's own + // routes (`refusePaidPlan`) and by the portal, never by the proxy. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + embedFail.status = 403; + embedFail.body = JSON.stringify({ error: { code: "invalid_token" } }); + await expect(runLocalIndexPass("full")).rejects.toThrow(/HTTP 403/); + expect(refusalNotes).toEqual([403]); + }); + + it("does not arm the refusal over a status the proxy did not claim as its own", async () => { + // A bare 401/403 on the wire can be an edge rule, a rate-limit page or an + // interception proxy, and remembering one of those would tell a customer + // with a perfectly good credential to re-pair their device. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + embedFail.status = 403; + embedFail.body = "Access denied"; + await expect(runLocalIndexPass("full")).rejects.toThrow(/HTTP 403/); + expect(refusalNotes).toEqual([]); + }); + + it("does not arm the refusal over the proxy's PLAN gate, which is not a credential", async () => { + // THE OTHER 403 THE PROXY SENDS, and the reason the guard may never read a + // status alone: cloud capabilities are sold per tier and the proxy refuses + // an unentitled one with 403 (`clawai-cloud-defaults-state.ts` — "TTS is + // Max-only on the proxy, which answers 403 to Free and Pro"). That is a + // fact about the PLAN, not about the credential, and recording it as a + // refused credential would tell an owner whose token is perfectly good to + // re-link the device. The index pass still ends, which is the honest half: + // what does not happen is the box concluding its credential is dead. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + embedFail.status = 403; + embedFail.body = JSON.stringify({ error: { code: "paid_plan_required" } }); + await expect(runLocalIndexPass("full")).rejects.toThrow(/HTTP 403/); + expect(refusalNotes).toEqual([]); + }); + + it("asks ONCE for a search query, however transient the refusal", async () => { + // The rebuild's retry budget is right for a rebuild and wrong for a person + // waiting: three attempts of up to 120 s with two waits of up to 30 s is + // ~7 minutes inside a search the MCP tool abandons at 60. A query that + // cannot be embedded now is answered now. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + await runLocalIndexPass("full"); + embedCalls.urls = []; + embedFail.status = 429; + embedFail.retryAfter = "0"; + await expect(searchLocalMemory("deposit", 3)).rejects.toThrow(/HTTP 429/); + expect(embedCalls.urls).toHaveLength(0); + }); + + it("waits a real interval when the far side asks for `Retry-After: 0`", async () => { + // `err.retryAfterMs ?? wait` reads `0` as a number, not as absent, so a + // rate-limited endpoint answering `Retry-After: 0` was asked three times + // with no pause at all — the hammer the backoff exists to prevent. + // + // ON A FAKE CLOCK, and a partly fake one on purpose. Measuring the pause by + // the wall clock put a real second on every `test:unit` run for one + // assertion, and answered a weaker question besides — "at least a second + // passed" is true of a pass that slept for any reason. `setTimeout` alone is + // faked (`pauseBeforeRetry` is the only timer this pass arms), so the sqlite + // and filesystem work either side of it still completes on the real event + // loop, which `setImmediate` is kept real to pump. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + embedFail.status = 429; + embedFail.retryAfter = "0"; + // What is asserted is the REQUEST, not that the pass is unfinished: a pass + // released early still has its sqlite and filesystem work to do, so "not + // settled yet" is true of a retry that has already gone out. + const pump = async (turns: number) => { + for (let i = 0; i < turns; i += 1) { + await new Promise((resolve) => { setImmediate(resolve); }); + } + }; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + let settled = false; + const pass = runLocalIndexPass("full").finally(() => { settled = true; }); + // Let the pass get as far as arming its retry pause. Bounded, so a pass + // that never arms one fails on the assertion below rather than hanging. + for (let i = 0; i < 200 && vi.getTimerCount() === 0 && !settled; i += 1) { + await new Promise((resolve) => { setImmediate(resolve); }); + } + expect(vi.getTimerCount()).toBe(1); + const asked = embedCalls.urls.length; + // A tick SHORT of the floor, and time for a request released early to + // actually reach the stub: nothing may have been asked again yet. + await vi.advanceTimersByTimeAsync(EMBED_RETRY_MIN_WAIT_MS - 1); + await pump(5); + expect(embedCalls.urls.length).toBe(asked); + expect(settled).toBe(false); + // …and the floor itself releases it. + await vi.advanceTimersByTimeAsync(1); + const result = await pass; + expect(embedCalls.urls.length).toBeGreaterThan(asked); + expect(result.files).toBe(1); + } finally { + vi.useRealTimers(); + } + }); + + it("gives up on a refusal that is not transient rather than asking three times", async () => { + // A 403 is a lapsed plan or a bad credential: the next attempt is refused + // the same way, and retrying would make the pass three times as slow to say + // so. The pass ends, and the index it was going to replace is still there. + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + write("notes.md", "The deposit is two months' rent."); + embedFail.status = 403; + await expect(runLocalIndexPass("full")).rejects.toThrow(/HTTP 403/); + }); + + it("calls semantic search available in the cloud without the 639 MB model on disk", async () => { + boxState.clawaiToken = "claw_test"; + boxState.gguf = false; + await writeEmbedderPin("cloud"); + const status = await localMemoryStatusJson() as { + status: { provider: string; model: string; vector: { semanticAvailable: boolean }; custom: { providerState: { mode: string } } }; + }; + expect(status.status.model).toBe("text-embedding-3-large"); + expect(status.status.provider).toBe("openai-compatible"); + expect(status.status.vector.semanticAvailable).toBe(true); + expect(status.status.custom.providerState.mode).toBe("active"); + }); + + it("does NOT call semantic search available when the proxy has refused this box's credential", async () => { + // FALSE SUCCESS, the exact shape. A credential the proxy has rejected — + // revoked, re-minted on another device, lost in a migration — sits in the + // store looking exactly like a working one, so "a token is present" + // reported a healthy cloud index while every pass failed and every search + // threw. The refusal the rest of the box already records for the picture + // and microphone paths is the fact that answers. + boxState.clawaiToken = "claw_test"; + boxState.clawaiRefusedStatus = 403; + await writeEmbedderPin("cloud"); + const status = await localMemoryStatusJson() as { + status: { vector: { semanticAvailable: boolean }; custom: { providerState: { mode: string } } }; + }; + expect(status.status.vector.semanticAvailable).toBe(false); + expect(status.status.custom.providerState.mode).not.toBe("active"); + }); + + it("reports the index as mismatched the moment the embedder moves, and valid again after the rebuild", async () => { + write("notes.md", "The deposit is two months' rent."); + await runLocalIndexPass("full"); + const read = async () => ((await localMemoryStatusJson()) as { status: { custom: { indexIdentity: { status: string } } } }) + .status.custom.indexIdentity.status; + expect(await read()).toBe("valid"); + + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + expect(await read()).toBe("mismatched"); + await runLocalIndexPass("full"); + expect(await read()).toBe("valid"); + }); + + it("never stamps the new embedder over an index the other one built", async () => { + // The stamp is for an index about to be built — the wizard's provisioning + // step. Over one that HOLDS vectors it would report `valid` for rows every + // query misses, which is the one lie this whole identity exists to stop. + write("notes.md", "The deposit is two months' rent."); + await runLocalIndexPass("full"); + boxState.clawaiToken = "claw_test"; + await writeEmbedderPin("cloud"); + await stampLocalEmbeddingIdentity(); + const status = await localMemoryStatusJson() as { status: { custom: { indexIdentity: { status: string } } } }; + expect(status.status.custom.indexIdentity.status).toBe("mismatched"); + }); + + it("leaves no database behind on a box whose owner never switched the feature on", async () => { + // `openIndexForRead`'s stated invariant, and the boot promotion is what + // started reaching this path unattended: on beta the stamp was owner- + // initiated only, and it opens the store FOR WRITE, which creates it. With + // no index there is nothing to stamp either — the identity reads `missing` + // and the first pass stamps what it wrote. + expect(fs.existsSync(LOCAL_INDEX_PATH)).toBe(false); + await stampLocalEmbeddingIdentity(); + expect(fs.existsSync(LOCAL_INDEX_PATH)).toBe(false); + }); +}); + +describe("the chunk ceiling, which is a memory budget", () => { + it("is the same ~78 MiB of vectors whichever embedder the box uses", () => { + // The flat 20,000 was sized against Qwen3's 1,024 dimensions — 4 KB a + // chunk, ~80 MB of float32 that fits beside the agent on an Orin. The cloud + // model is 3,072 dimensions, so the same 20,000 chunks would have been + // ~234 MiB, allocated contiguously by `loadVectors` on the agent's first + // search after a restart and pinned for ten minutes after every search. + const bytes = (chunks: number, dim: number) => chunks * dim * 4; + expect(maxIndexChunks(1024)).toBe(20_000); + expect(maxIndexChunks(3072)).toBe(6_666); + expect(bytes(maxIndexChunks(3072), 3072)).toBeLessThanOrEqual(bytes(20_000, 1024)); + // Within one chunk's worth of the budget, not merely under it. + expect(bytes(maxIndexChunks(3072), 3072)).toBeGreaterThan(bytes(20_000, 1024) - 3072 * 4); + }); + + it("falls back to the on-device ceiling for a width it cannot use, never to zero", () => { + // A ceiling of zero is an index that refuses every file. + for (const bad of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(maxIndexChunks(bad), String(bad)).toBe(20_000); + } + }); +}); + describe("finding things again", () => { it("ranks the document that actually answers the question first", async () => { write("lease.md", "The deposit is two months' rent, returned within 30 days."); @@ -571,7 +1005,7 @@ describe("the index knows what it was built for", () => { expect(status.status.custom.indexIdentity.status).toBe("missing"); expect(status.status.files).toBe(0); expect(status.status.chunks).toBe(0); - expect(localEmbeddingIdentity()).toHaveLength(16); + expect(await localEmbeddingIdentity()).toHaveLength(16); }); it("keeps the index it was going to replace when the embedder will not answer", async () => { diff --git a/src/tests/unit/memory-shard-embedding-keys.test.ts b/src/tests/unit/memory-shard-embedding-keys.test.ts index 261927f12..960303224 100644 --- a/src/tests/unit/memory-shard-embedding-keys.test.ts +++ b/src/tests/unit/memory-shard-embedding-keys.test.ts @@ -14,14 +14,17 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; const OPENCLAW_PACKAGE_JSON = "/home/clawbox/.npm-global/lib/node_modules/openclaw/package.json"; const PROXY_URL = "http://127.0.0.1/setup-api/local-ai/embed/v1"; +const CLOUD_ENDPOINT = "https://clawbox.test/api/ai/embeddings"; const TOKEN = "t".repeat(64); -const { runOpenclawConfigSetBatch, readFile, openclawIsAbsent, stampLocalEmbeddingIdentity, readConfig } = vi.hoisted(() => ({ +const { runOpenclawConfigSetBatch, readFile, openclawIsAbsent, stampLocalEmbeddingIdentity, readConfig, writeEmbedderPin } = vi.hoisted(() => ({ runOpenclawConfigSetBatch: vi.fn(async () => ""), openclawIsAbsent: vi.fn(() => false), stampLocalEmbeddingIdentity: vi.fn(async () => {}), readFile: vi.fn<(path: string, encoding: string) => Promise>(), readConfig: vi.fn<() => Promise>>(), + /** The word ClawBox's own store keeps on the SKU where it is the indexer. */ + writeEmbedderPin: vi.fn(async (_source: string) => {}), })); vi.mock("fs/promises", () => ({ readFile })); @@ -50,8 +53,44 @@ vi.mock("@/lib/embed-server", () => ({ vi.mock("@/lib/local-ai-token", () => ({ getLocalAiToken: () => TOKEN, })); +vi.mock("@/lib/memory-embedder", async (importOriginal) => ({ + // The fence is the real one: this file is about what each edition WRITES, and + // a stubbed fence would let a cloud switch record an endpoint it must refuse. + ...(await importOriginal()), + writeEmbedderPin, + readEmbedderPin: vi.fn(async () => null), + defaultEmbedderSource: vi.fn(async () => "local" as const), +})); +/** The support kill switch, `clawai_cloud_embeddings: "off"`. */ +const cloudEmbeddingsOff = vi.hoisted(() => ({ value: false })); +vi.mock("@/lib/clawai-cloud-embeddings", () => ({ + CLOUD_EMBEDDING_MODEL: "text-embedding-3-large", + CLOUD_EMBEDDING_PROVIDER: "openai-compatible", + cloudEmbeddingsUrl: () => CLOUD_ENDPOINT, + embeddingsBaseUrlOf: (endpoint: string) => endpoint.replace(/\/+$/, "").replace(/\/embeddings$/, ""), + // The real rule, narrow enough to stand in for it here: an address that could + // be an embedding endpoint at all. `readEmbeddingPlacement` asks it before it + // calls a configured endpoint a recorded cloud placement, and the endpoint it + // is asking about is OPENCLAW'S client's, not ClawBox's — so ClawBox's own + // cleartext fence has no jurisdiction over it. + embeddingEndpointParseable: (raw: string | undefined) => { + if (!raw || raw.length > 2048) return false; + try { + const parsed = new URL(raw); + return (parsed.protocol === "https:" || parsed.protocol === "http:") && parsed.hostname !== ""; + } catch { + return false; + } + }, + cloudEmbeddingsSwitchedOff: async () => cloudEmbeddingsOff.value, +})); +vi.mock("@/lib/harness/credentials", () => ({ + CLAWBOX_AI_PROXY_URL: "https://clawbox.test/api/ai", + resolveClawaiToken: async () => "claw_test", +})); -import { embeddingConfigHome, readEmbeddingChoice, switchToLocalEmbeddings } from "@/lib/memory-shard"; +import { embeddingConfigHome, readEmbeddingChoice, readEmbeddingPlacement, switchToCloudEmbeddings, switchToLocalEmbeddings } from "@/lib/memory-shard"; +import { defaultEmbedderSource, readEmbedderPin } from "@/lib/memory-embedder"; import { LOCAL_EMBEDDING_MODEL, LOCAL_EMBEDDING_PROVIDER } from "@/lib/memory-shard-state"; /** Everything the embedder needs, and the provider LAST — the switch itself. */ @@ -77,6 +116,7 @@ beforeEach(() => { readFile.mockReset(); readConfig.mockReset().mockResolvedValue({}); stampLocalEmbeddingIdentity.mockClear(); + writeEmbedderPin.mockClear(); openclawIsAbsent.mockReturnValue(false); }); @@ -215,13 +255,146 @@ describe("switchToLocalEmbeddings", () => { it("writes NOTHING into openclaw.json on the edition that has no OpenClaw", async () => { // There is no external client to point at the embedder on that SKU — - // ClawBox is the client — so the only thing left to record is which model - // the vectors about to be written belong to. Spawning the CLI there would - // be spawning a binary that is not installed. + // ClawBox is the client — so what is recorded is where ClawBox should embed + // and which model the vectors about to be written belong to. Spawning the + // CLI there would be spawning a binary that is not installed. openclawIsAbsent.mockReturnValue(true); await switchToLocalEmbeddings(); + expect(writeEmbedderPin).toHaveBeenCalledWith("local"); expect(stampLocalEmbeddingIdentity).toHaveBeenCalledTimes(1); expect(runOpenclawConfigSetBatch).not.toHaveBeenCalled(); expect(readFile).not.toHaveBeenCalled(); }); }); + +describe("switchToCloudEmbeddings on the edition that has no OpenClaw", () => { + beforeEach(() => openclawIsAbsent.mockReturnValue(true)); + + it("records the WORD and nothing else — no address, no copy of the credential", async () => { + await switchToCloudEmbeddings(CLOUD_ENDPOINT, "claw_test"); + expect(writeEmbedderPin).toHaveBeenCalledWith("cloud"); + expect(stampLocalEmbeddingIdentity).toHaveBeenCalledTimes(1); + expect(runOpenclawConfigSetBatch).not.toHaveBeenCalled(); + for (const [[stored]] of writeEmbedderPin.mock.calls.map((call) => [call])) { + expect(stored).toBe("cloud"); + } + }); + + it("refuses an endpoint that is not this box's own ClawBox AI account", async () => { + // The fence, checked where the choice is RECORDED as well as where the + // socket is opened: everything the owner has indexed is the body of those + // requests. + await expect(switchToCloudEmbeddings("https://someone-elses-server.example/v1/embeddings", "claw_test")) + .rejects.toThrow(/ClawBox AI account/i); + expect(writeEmbedderPin).not.toHaveBeenCalled(); + expect(stampLocalEmbeddingIdentity).not.toHaveBeenCalled(); + }); + + it("refuses an empty credential rather than recording a cloud it cannot reach", async () => { + await expect(switchToCloudEmbeddings(CLOUD_ENDPOINT, " ")).rejects.toThrow(/credential/i); + expect(writeEmbedderPin).not.toHaveBeenCalled(); + }); +}); + +/** + * `readEmbeddingPlacement` — the ONE reader of where the index is embedded, and + * the only one that can say whether that is written down. + * + * Two arms because the thing that INDEXES owns the setting, and `recorded` is + * what stops the automatic promotion rebuilding the index at every boot. + */ +describe("readEmbeddingPlacement", () => { + const pin = vi.mocked(readEmbedderPin); + const fallback = vi.mocked(defaultEmbedderSource); + + beforeEach(() => { + pin.mockReset().mockResolvedValue(null); + fallback.mockReset().mockResolvedValue("local"); + cloudEmbeddingsOff.value = false; + installedCore("2026.9.1"); + }); + + it("reads openclaw.json where the core is the embedding client", async () => { + readConfig.mockResolvedValue({ memory: { search: { remote: { baseUrl: PROXY_URL } } } }); + expect(await readEmbeddingPlacement()).toEqual({ source: "local", recorded: true }); + + readConfig.mockResolvedValue({ memory: { search: { remote: { baseUrl: "https://clawbox.test/api/ai" } } } }); + expect(await readEmbeddingPlacement()).toEqual({ source: "cloud", recorded: true }); + }); + + it("calls an OpenClaw box with no endpoint at all unwritten, so the default still has its write to make", async () => { + readConfig.mockResolvedValue({}); + expect(await readEmbeddingPlacement()).toEqual({ source: "local", recorded: false }); + }); + + it("never calls an address nothing can embed through a recorded cloud placement", async () => { + // `memory.search` is a file a restored backup, a hand edit or a + // half-finished migration can leave holding a truncated URL, a `file:` + // scheme or cleartext off the device. Reading any of those as + // `recorded: cloud` is the false-success shape on the one reader the + // automatic default asks before deciding whether it still owes this box a + // write: it skipped the write, and the box kept a `memory.search` it could + // not embed with and no surface saying so. Unrecorded hands it back to the + // rule that writes a configuration which works. + for (const bad of [ + "https://", + "not-a-url", + "file:///etc/passwd", + `https://long.test/${"a".repeat(2100)}`, + ]) { + readConfig.mockResolvedValue({ memory: { search: { remote: { baseUrl: bad } } } }); + expect(await readEmbeddingPlacement(), bad).toEqual({ source: "local", recorded: false }); + } + }); + + it("still calls a genuine third-party https endpoint the cloud, because it IS one", async () => { + // The owner pointing OpenClaw at their own account is a real recorded + // placement, and the promotion leaves a recorded cloud alone. Refusing it + // here would have had the applier overwrite their configuration. + readConfig.mockResolvedValue({ memory: { search: { remote: { baseUrl: "https://api.openai.com/v1" } } } }); + expect(await readEmbeddingPlacement()).toEqual({ source: "cloud", recorded: true }); + }); + + it("leaves the owner's own LAN embedding endpoint alone", async () => { + // ClawBox's cleartext fence is about where CLAWBOX sends the owner's + // documents. On this arm the client is OpenClaw's own, pointed at a + // llama.cpp on another machine by a person editing openclaw.json, and + // ClawBox has no jurisdiction over it. Read as unset, the boot promotion + // replaced that endpoint with the ClawBox AI one and started a full reindex + // — a configuration change the owner never asked for. + readConfig.mockResolvedValue({ memory: { search: { remote: { baseUrl: "http://192.168.1.50:8080/v1" } } } }); + expect(await readEmbeddingPlacement()).toEqual({ source: "cloud", recorded: true }); + }); + + it("reads the pin where ClawBox is the indexer", async () => { + openclawIsAbsent.mockReturnValue(true); + pin.mockResolvedValue("cloud"); + expect(await readEmbeddingPlacement()).toEqual({ source: "cloud", recorded: true }); + expect(fallback).not.toHaveBeenCalled(); + }); + + it("says LOCAL over a cloud pin the support kill switch has switched off", async () => { + // M-6's other half. `resolveMemoryEmbedder` honours `clawai_cloud_embeddings: + // "off"` over a stored pin, so every embed and every search goes to the + // loopback proxy — while this reader answered "cloud" from the pin alone, so + // the card drew the cloud hint and preselected the cloud segment, and the + // provider GET answered `source: "cloud"` beside `cloudAvailable: false`. + // The pin is NOT rewritten: the lever is a support action and reversible. + openclawIsAbsent.mockReturnValue(true); + pin.mockResolvedValue("cloud"); + cloudEmbeddingsOff.value = true; + expect(await readEmbeddingPlacement()).toEqual({ source: "local", recorded: true }); + }); + + it("answers the default rule for a box nobody has pinned — the cloud where the subscription covers it", async () => { + openclawIsAbsent.mockReturnValue(true); + fallback.mockResolvedValue("cloud"); + expect(await readEmbeddingPlacement()).toEqual({ source: "cloud", recorded: false }); + }); + + it("takes a verdict the caller already worked out rather than paying for the probe twice", async () => { + openclawIsAbsent.mockReturnValue(true); + expect(await readEmbeddingPlacement("cloud")).toEqual({ source: "cloud", recorded: false }); + expect(fallback).not.toHaveBeenCalled(); + }); +}); From 3885f73376c0a349df8bf2585bbafb321ab4ec6b Mon Sep 17 00:00:00 2001 From: Krasimir Kralev <263465593+KrasimirKralev@users.noreply.github.com> Date: Fri, 18 Sep 2026 13:46:43 +0300 Subject: [PATCH 2/2] feat: load a Hermes plugin installed after boot without a sudoers grant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hermes scans for plugins once per process — discover_plugins(force=True) at start, _ensure_plugins_discovered() returning early ever after, and nothing reachable over the dashboard socket passes its force flag. Skills have /reload-skills and MCP has reload.mcp; plugins have only Hermes' own instruction, printed at the end of every install: restart the gateway. On this SKU the process serving chat is clawbox-hermes-dashboard.service, so a plugin installed after boot reached no chat at all, new sessions included, while hermes plugins list said "enabled" and a fresh hermes chat -q proved it working. The assistant then reached for systemctl restart and was refused. THE GRANT IS NOT THE ANSWER, and is deliberately not added. bounceHermesDashboard() already restarts this exact unit with no privilege: hermes dashboard --stop is upstream's own SIGTERM path over a process the clawbox user owns, Restart=always brings it back, and a new main PID plus :9119 answering is what "restarted" means. A restart grant would also START a stopped unit, which is how an OpenClaw box could resurrect the dashboard its foreign-edition teardown had just stopped — the invariant install-sudoers-migration and install-foreign-edition-teardown own. config/clawbox-sudoers gains that reasoning as comments and no rule. - hermes-plugin-set.ts reads DECLARED against LOADED: a content hash of the install ledger and the top-level plugins: block (the block alone — every Settings save rewrites config.yaml and the dashboard's own ExecStartPre rewrites it per start, which an mtime watcher would turn into a restart loop), and the running unit's own journal for what it registered. "Could not be asked" answers null, never an empty list. - hermes-plugin-reload.ts restarts and tells the owner, and watches ~/.hermes: first look is a baseline, one restart per change burst, a backoff doubling to five minutes over a failure that never moves the baseline forward. - POST/GET /setup-api/hermes/plugins/reload, owner cookie or the MCP bearer — the agent is the intended caller — 404 off the editions with a dashboard, 502 rather than a 200 with a flag when the restart could not be taken. - hermes_plugins_reload and the field-guide section that tells the assistant to call it instead of sudo, and that hermes chat -q proves nothing about the chat. The owner's open chat window drops on the restart; the notice says to open a new one. (cherry picked from commit f2a7eb13b506e4194d3680bc2564d96de09ad118) --- CLAUDE.md | 1 + Clawbox.md | 7 + config/clawbox-sudoers | 28 + mcp/README.md | 58 ++ mcp/clawbox-mcp.ts | 6 + mcp/tools/hermes-plugins.ts | 146 +++ src/app/setup-api/clawkeep/restore/route.ts | 20 +- .../setup-api/hermes/plugins/reload/route.ts | 154 +++ src/instrumentation.ts | 19 + src/lib/hermes-bounce-claim.ts | 250 +++++ src/lib/hermes-dashboard-control.ts | 137 ++- src/lib/hermes-image-refresh.ts | 11 +- src/lib/hermes-plugin-reload.ts | 617 ++++++++++++ src/lib/hermes-plugin-set.ts | 778 ++++++++++++++++ src/lib/yaml-block-edit.ts | 105 ++- .../routes/clawkeep-restore-restart.test.ts | 7 +- .../routes/hermes-plugins-reload.test.ts | 205 ++++ .../unit/hermes-dashboard-control.test.ts | 102 +- src/tests/unit/hermes-image-refresh.test.ts | 2 +- src/tests/unit/hermes-plugin-reload.test.ts | 881 ++++++++++++++++++ src/tests/unit/hermes-plugin-set.test.ts | 786 ++++++++++++++++ 21 files changed, 4291 insertions(+), 29 deletions(-) create mode 100644 mcp/tools/hermes-plugins.ts create mode 100644 src/app/setup-api/hermes/plugins/reload/route.ts create mode 100644 src/lib/hermes-bounce-claim.ts create mode 100644 src/lib/hermes-plugin-reload.ts create mode 100644 src/lib/hermes-plugin-set.ts create mode 100644 src/tests/routes/hermes-plugins-reload.test.ts create mode 100644 src/tests/unit/hermes-plugin-reload.test.ts create mode 100644 src/tests/unit/hermes-plugin-set.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index d2bfed9b8..3f98fd24e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,6 +49,7 @@ Next.js rewrites in `next.config.ts` proxy gateway paths (`/api/*`, `/assets/*`, - **Files**: `files/` — file list, read, write, upload, mkdir, delete. Its `resolve` action answers `relPath` beside `absPath` — the browse-relative path the Files app navigates by — which is how the Coding Agent's "Open in Files" turns a project folder's absolute path into a Files window opened there (`dispatchOpenApp("files", { forceNew, meta: { path } })` → `FilesApp initialPath`). Uploads are bounded by what the box can spare, not by a size cap (large uploads are the point of the PUT path): `src/lib/bounded-body.ts` meters the body against a budget that re-measures free space on the DESTINATION filesystem every 64 MiB minus `DISK_FREE_RESERVE_BYTES` (`src/lib/disk-reserve.ts`, 512 MiB — the in-app update's build is what a full disk kills; project-import's reserve is the same constant), a chunked or understated body is cut at the reserve with 507 `disk_full` and the partial file unlinked, and the multipart POST carries Busboy limits (`files: 1`, `parts: 4`) with the attachments route's teardown discipline — busboy's `fileSize` TRUNCATES rather than errors, so the `limit` handler destroys the write and answers 507 instead of `ok: true` over a cut file; count limits answer 413 `too_many_parts`; a statfs that fails skips the reserve for that request rather than refusing every upload. The `[...path]` PUT (rename) and DELETE refuse a PROTECTED CONTAINER with 400 `protected_container` (`isProtectedContainer` in `src/lib/file-guard.ts`, judged on the typed path AND its `canonicalPath`): `data/` itself, the checkout, `~/.config` and the browse root — `isProtectedFilePath(DATA_DIR)` stays false so the listing keeps filtering entry by entry, but a rename of `data/` to `data-copy` used to move every credential store out from under the guard in one request. - **Browser**: `browser/` — Chromium automation via CDP (launch, navigate, click, type, screenshot). WHICH Chromium it drives is the owner's setting, read for EVERY session: with the Coding Agent's real browser on (`coding_agent_real_browser`, absent means on) it uses the device's own window, brought up the way `browser/manage`'s open does, and falls back to a headless Chromium of its own only when that is impossible (a foreign Chromium on port 18800, a launch that failed); with it off it goes straight to the headless one and never touches the owner's screen. Every answer carries `browser: "desktop" | "headless"` — a screenshot from the two is identical, and a run that verified its work where nobody could watch must not report that the owner saw it. The page handles it hands out live in `src/lib/browser-sessions.ts` rather than in the route, because a delegated coding run drives this Chromium too: a run gets a page of its OWN (borrowing `context.pages().at(-1)` steered the tab the owner was reading, and the ten-minute idle sweep then closed it), every session records the run that opened it, and the runner closes exactly those pages when the run settles. `browser/manage` is the device half — install Chromium, the agent link, open/close the desktop window — and answers `serviceSafe: false` for a snap build `clawbox-browser.service` cannot start, refusing the launch at once instead of "check /tmp/clawbox-browser.log" ten seconds later; the same GET also carries `setupComplete`, `autoOpen` and `startUrl`, so the app's face and its state cannot come from two different moments. `browser/setup` (owner-only AND same-origin) writes those three: the auto-open switch exists because `ui_open_app("browser")` is a tool call, and a tool that could switch it back on would make the owner's "no" temporary. systemd starts Chromium, not the web server, so the start page reaches it on disk — `writeBrowserLaunchEnv` (`src/lib/browser-setup.ts`) puts it in `~/.cache/clawbox/browser.env`, which `scripts/launch-browser.sh` sources the way it already sources the VNC display — and only `about:blank` (the default since 2026-09-15 — it loads nothing) or an `http(s)` address survives `normalizeStartUrl`, because Chromium would happily open `file:///etc/shadow` on the screen the agent screenshots. - **Harness**: `harness/active`, `harness/select`, `harness/status`, `harness/swap` (see `harness-swap.ts` under Server Libraries), and `harness/mcp` — the owner's on/off switch for the ClawBox MCP server, the assistant's device tools (Settings → Harness → Device tools, owner's request 2026-09-15). GET `{ enabled, registered: { openclaw, hermes } }` — the switch (`clawbox_mcp_enabled` in the config store; ABSENT MEANS ON, it is the box's default capability and not a consent, `src/lib/clawbox-mcp-switch.ts`) and whether each harness's config lists the server right now (null = not on this edition, or its config could not be read). POST `{ enabled }` is owner cookie AND same origin, the `harness/swap` guard (403 `owner_only` for the MCP bearer, 403 `cross_origin`), because a tool that could switch itself back on would make the owner's "off" temporary. It writes the key FIRST (500 `store_write_failed` touches nothing) and then, on every harness this edition runs (`src/lib/clawbox-mcp-registration.ts`, both on `dual`): OFF removes `mcp.servers.clawbox` from openclaw.json with the module's read-strict/write pattern and `restartGateway()`, and removes `mcp_servers.clawbox` from `~/.hermes/config.yaml` through `patchHermesConfig`; ON is `restartGateway()` alone on OpenClaw (`gateway-pre-start.sh` is the gateway's ExecStartPre and reconciles the entry from the switch) and `scripts/register-mcp.sh` run the way `production-server.js` runs it on Hermes; either way Hermes is then asked to `reload.mcp` (`hermes-mcp-reload.ts`, the harness/select mechanism) and a messaging gateway that is UP is restarted so a Telegram turn follows the switch too (never installed, never started). Both boot scripts honour the key — `false` makes each REMOVE its entry instead of writing it, one journal line either way — so a reboot cannot put the tools back. The answer is the re-read GET shape plus `applied`; a half that failed after the key landed is a 502 carrying that state beside `{ error, code }` (`openclaw_config_unreadable`, `openclaw_unregister_failed`, `gateway_restart_failed`, `hermes_unregister_failed`, `hermes_register_failed`, `hermes_still_registered` (the unset was saved over by a straddling boot-script run, checked by reading the file back), `openclaw_not_registered` (ON restarted the gateway but its pre-start skipped the entry)), because the switch IS saved and the panel draws that. The MCP server's own code is unchanged. +- **Hermes plugins**: `hermes/plugins/reload` — GET reports what `~/.hermes` DECLARES beside what the running dashboard LOADED (and `stale`: the box declares a plugin as enabled that the running registry does not have on); POST restarts the chat backend so it re-scans. Hermes scans for plugins once per process (`discover_plugins(force=True)` at start, `_ensure_plugins_discovered()` returning early ever after) and offers no runtime reload — skills have `/reload-skills`, MCP has `reload.mcp`, plugins have only Hermes' own instruction to restart — so a plugin installed after boot reached no chat at all, new sessions included, while `hermes plugins list` said `enabled` and a fresh `hermes chat -q` proved it working. The restart is `bounceHermesDashboardShared()` (`src/lib/hermes-bounce-claim.ts`) — the ONE door every caller uses (the watcher, this route and the MCP tool behind it, a ClawKeep restore, the image refresh), wrapping `bounceHermesDashboard()` in state the callers share (`process-store.ts`, `PluginReloadState`): a CLAIM taken before the first await and cleared only by the caller that set it, so a second caller is answered `in_flight` — its own fact, never a failure — instead of stopping a dashboard that is already coming back; the declaration signature read BEFORE the stop recorded as the watcher's BASELINE, so a restart the route performed is not repeated eight seconds later; the outgoing MainPID recorded beside it when the bounce came back `pending`, which is what lets the reconcile tell a replacement from the process that never went away; and a PENDING-RESTART mark set when a caller arrives while the claim is held — that caller's mutation (a restore that rewrote `~/.hermes` after the running bounce had read it) may not be in what the replacement loaded, so the baseline is left BEHIND on the way out and the watcher takes exactly one more bounce for it. It adds NO privilege: `hermes dashboard --stop` over a process the clawbox user owns, `Restart=always` bringing it back, a new main PID and :9119 answering before `ready` is reported. A sudoers grant is deliberately NOT added — `systemctl restart` also STARTS a stopped unit, which is how an OpenClaw box could resurrect the dashboard its foreign-edition teardown stopped and disabled (`install-sudoers-migration.test.ts`, `install-foreign-edition-teardown.test.ts`), and `config/clawbox-sudoers` carries that reasoning where the next request for the grant will look. `requireSession` admits the owner's cookie OR the MCP bearer, because the AGENT is the intended caller: it is what installs a plugin and what tried `sudo systemctl restart` and was refused. `restarted && !ready` is a 200 (systemd owns it, acting again makes it worse); a restart that could not be taken is a 502, never a 200 with a flag, since the caller is about to tell the owner the plugin is live. LOADED is asked of HERMES ITSELF — `plugins.list` on the dashboard socket ClawBox already dials for `reload.mcp`, built from `get_plugin_manager()._plugins`, the running process's own registry — with the `journalctl _SYSTEMD_INVOCATION_ID=… | Plugin '' registered` scrape kept only as the fallback for a dashboard that cannot be reached: that scrape matched NOTHING on the owner's own box (a plugin registering only tools logs it below the level the read sees), so the feature's primary proof was permanently unavailable on the hardware it was built for. `loaded` is null — never `[]` — when the process could not be asked; an answer with rows and none of them enabled IS `[]`, because "the process answered no" and "the process did not answer" are different facts. `stale` follows from that registry and NEVER from an mtime: `max(mtime(ledger), mtime(config.yaml))` made it true on every Settings save, since config.yaml is rewritten by each one, so the MCP tool warned "the agent is still behind the files" about a plugin that had been loaded for hours. It is one-directional — a name the box declares as enabled that the registry does not have on — so a bundled plugin the process carries and the config never names cannot make a box report work outstanding for ever. Both verbs are owner cookie OR the MCP bearer, and the POST is SAME-ORIGIN on top of that (403 `cross_origin`): it drops the owner's chat window, and `isSameOriginRequest` waves the header-less MCP caller through, which is what keeps the intended caller working. `src/lib/hermes-plugin-reload.ts` also runs the automatic half: a watcher started from `src/instrumentation.ts` on `hasHermesHarness()`, whose first look is a BASELINE unless TWO facts both hold — `stale` (the running registry is missing something the box declares) AND `changedAfterStart` (the declaration's files were touched after the dashboard started). Neither alone: `stale` is the honest answer and is not self-limiting, so a box whose declaration and registry can never agree would be bounced at every web-server boot for good (measured on the owner's box, where the registry answers PATH-DERIVED keys — `image_gen/clawai`, `superpowers/.hermes-plugin` — that a whole-string comparison called missing, which is why `registryHas` mirrors Hermes' own matcher — `names = {manifest name, registry key}` (`_plugin_status`), with the name↔key map read from `plugins.manage {action:"list"}`, the one method that carries both; a declared name it cannot resolve answers `null`, never `true`, because `stale: true` is what arms an unattended restart of the owner's chat. `plugins.disabled` WINS over `plugins.enabled` there too, the way `_plugin_status` gives the deny-list precedence: `hermes plugins disable superpowers` discards the resolved key and its leaf from the allow-list and adds the key to the deny-list, leaving the bare name a person wrote behind in `enabled` — read from `enabled` alone, `stale` was permanently true on any box whose owner had deliberately switched a plugin off (measured on the owner's box, 2026-09-18)); `changedAfterStart` is an mtime and every Settings save makes it true, but once the dashboard has restarted its start is newer than those files. Together they are the `register-mcp.sh` case — that script runs at EVERY web-server boot and can append a plugin to `plugins.enabled`, leaving it invisible for the rest of that dashboard's life — and true at most once per dashboard. An unknown answer on either side seeds. That baseline lives in `process-store.ts`, NOT in the watcher's closure: `instrumentation.ts` reaches the module with `require` while the route `import`s it, so the two copies each kept their own — the assistant doing what `hermes_plugins_reload` tells it to do bounced the dashboard, the watcher saw the same change and bounced it again eight seconds later, and the owner's chat dropped twice in fifteen seconds. A bounce in flight (any caller's) makes the watcher open no window at all, and a bounce that came back `pending` — the stop took, the replacement is not serving yet — does NOT move the baseline: the next window RECONCILES only on PROOF that the process was replaced — systemd's MainPID differs from the one recorded before the stop, or, where the pid cannot settle it (the pre-bounce read failed, or systemd reported the valid `MainPID=0`), a plugin query that answers `stale: false`, which is the same question the baseline exists to record. A port that answers proves nothing on its own: `pending` covers "systemd could not be asked, twice", where the OLD process is still serving and answering :9119 perfectly well, and reconciled on the probe alone the watcher recorded the change as loaded and never tried again. The same process still serving, or a query that says the running registry is still `stale`, bounces AGAIN; anything unproven waits with a widening backoff while systemd says it is still coming, and bounces a second time only once the unit reads `down`. It compares a hash of the PARSED plugin set — the sorted `plugins.enabled` and `plugins.disabled` names and the install ledger's name→revision rows — read out of the top-level `plugins:` block of `config.yaml` and `~/.hermes/plugins/.install-metadata.json`. The block alone, because every Settings save rewrites that file and the dashboard's own ExecStartPre rewrites it on each start, which an mtime watcher would turn into a restart loop; and the PARSED set rather than the block's text, because `scripts/register-mcp.sh` re-serialises the whole config through `yaml.safe_dump` whenever anything changed — spawned at every web-server boot and again from the ClawBox-MCP toggle — so a hash of the raw text read a re-indentation as a plugin change and dropped the owner's chat for it. Both block layouts are read: Hermes' dumper forces `indentless=False` (` - name`) while PyYAML's default in a mapping context is an indentless sequence (` - name`), and the old rule accepted only the first, so on every box `register-mcp.sh` had written `plugins.enabled` parsed as EMPTY and `stale` was `false` for good. A trailing `# comment` is a comment and `- "weird # name"` is a name, quote-aware through `splitYamlComment` in `yaml-block-edit.ts`. It debounces `HERMES_PLUGIN_DEBOUNCE_MS` so one `hermes plugins install` is ONE restart, backs off doubling to five minutes over a restart that failed without ever moving the baseline forward, and pushes an owner notice naming the plugin and saying to open a new chat. The owner's open chat window DOES drop on the restart; that is the feature, and the notice is what explains it. There is no turn-in-flight gate: the dashboard socket exposes no active-run signal (`reload.mcp`, `reload.env`, `commands.catalog`, `image.generate`, the session methods and the turn transport), and inferring one from a slow reply would bounce a box that was merely busy or defer for ever on one that was merely slow. - **Gateway**: `gateway/`, `gateway/health`, `gateway/ws-config` — gateway proxying with HTML injection - **Telegram**: `telegram/configure`, `telegram/status` — Telegram bot config - **Email**: `email/configure`, `email/status`, `email/test`, `email/send`, `email/messages`, `email/pending` — a mail account (any provider; Gmail app-password guide in the UI) with ONE of three modes: send only, read on demand (`email/messages` backs the `email_list`/`email_read` MCP tools over ClawBox's own IMAP client — no polling, EXAMINE + BODY.PEEK so reading never marks mail seen), or answer senders (Hermes' native adapter, allowlist-only). A separate "ask me before sending" gate turns `email/send` into a queued draft; `email/pending` approves or deletes it and refuses the MCP bearer, because the agent is the party it gates (`coding-agent/enable` refuses it for the same reason). Its `approve_batch` action is what the chat's batch card posts: one consent for a NAMED set of drafts, each carrying the content fingerprint it was shown with, so nothing queued while the owner was reading can ride along — and it answers per draft, 207 unless every one of them reached somebody — a copy an identical message already covered, and a draft another surface SENT while the card sat on screen, are counted apart from the failures, because neither is anything gone wrong; a draft that was deleted, refused or left unconfirmed elsewhere stays among them, because under all three the message reached nobody. A row for a send that failed carries the RECEIPT's ending (`failed` for a refusal the mail server spoke, `unconfirmed` for a silence this box cannot read), never the SMTP kind, so no surface can turn a message that may have gone out into a definite "not sent". Its `reject_batch` is the mirror: one gesture deletes a NAMED SET, each entry carrying the fingerprint it was shown with, so "send nothing" reaches the store instead of hiding the card until the next poll re-offered the same drafts. Its GET answers `{ pending, outcomes }` in one response — the queue and the receipts for what has left it — so no surface can catch a draft in neither list and render a live Approve button over a message that is already sent. `email/chat-approval` (owner-only, off by default) connects a SECOND Telegram bot that ClawBox polls itself and never hands to the harness, so a queued draft can be approved with a button in the owner's chat: the tap arrives from Telegram, not from the agent. `email/chat-reply` is the same consent with no BotFather token, and unlike `chat-approval` it is **ON by default** (owner's ruling 2026-09-04: "when I tell it in Telegram to send, it will send" — a switch he has to find first would not be that; and unlike the second bot it asks him for no credential, so there is nothing to configure): the owner replies "send " in the conversation he already has with the box, and each harness's OWN inbound hook — `before_dispatch` on OpenClaw, `pre_gateway_dispatch` on Hermes, both registered by the plugin ClawBox already ships — claims that message before the model sees it and posts it here. The consent it rides on is the one already given: it only ever asks about a draft, and a draft only exists because "ask me before sending" is on. The gate is the sender id the harness reports, checked against the harness's own owner allowlist, exactly as the button path checks who pressed; the code names ONE draft and its content fingerprint, so "send" can never mean "whatever is queued now". There is still no approve verb on the tool surface (`src/lib/owner-session.ts`), and the code is deliberately withheld from the `email_send` tool result. diff --git a/Clawbox.md b/Clawbox.md index 7aba111a6..b314f80cd 100644 --- a/Clawbox.md +++ b/Clawbox.md @@ -147,6 +147,13 @@ ClawBox adds no web tools on this harness. Reach the web with whatever your Herm - `skill_install(id)` / `skill_uninstall(name)` — add or remove one. You can do this yourself; it is the Hermes answer to the app store. - The owner sees the same catalogue in the **Hermes Skills** app on the desktop. +### 🧩 Plugins (they need a restart, and you can order it) + +- A **plugin** is not a skill. A skill is read again on every turn; a plugin is scanned **once, when your process starts** — so a plugin installed or enabled while you are running does not reach you, and does not reach a new chat session either. +- After `hermes plugins install`, `enable`, `disable` or `remove`, call **`hermes_plugins_reload()`**. That is the whole step. Do not try `sudo systemctl restart …` — it is refused, and this tool is the supported way. +- Verifying a plugin by running `hermes chat -q` proves nothing about **this** chat: that is a separate, fresh process. Only the reload makes the plugin live here. +- The restart closes the owner's open chat window. Tell them to open a new chat; the box shows them a notice saying the same thing. + ### 🧠 Your own model - `ai_list_models()` / `ai_set_provider(...)` / `ai_set_model(...)` — the device default in `~/.hermes/config.yaml`. diff --git a/config/clawbox-sudoers b/config/clawbox-sudoers index 670b828e4..e96c7cada 100644 --- a/config/clawbox-sudoers +++ b/config/clawbox-sudoers @@ -103,6 +103,34 @@ clawbox ALL=(root) NOPASSWD: /usr/bin/systemctl disable clawbox-tunnel clawbox ALL=(root) NOPASSWD: /usr/bin/systemctl restart hermes-gateway.service clawbox ALL=(root) NOPASSWD: /usr/bin/systemctl restart hermes-gateway +# clawbox-hermes-dashboard DELIBERATELY HAS NO GRANT, and this note is here +# because the request for one keeps arriving with a real problem behind it. +# +# The problem is genuine: Hermes scans for plugins once per process, so a plugin +# installed after boot never reaches the chat that dashboard serves, and Hermes' +# own instruction is to restart. The assistant on a box hits this, reaches for +# `sudo systemctl restart clawbox-hermes-dashboard`, and is refused. +# +# The answer is not a grant, for two reasons. +# +# * IT IS NOT NEEDED. `bounceHermesDashboard()` +# (src/lib/hermes-dashboard-control.ts) already restarts this exact unit with +# no privilege at all: `hermes dashboard --stop` is upstream's own +# SIGTERM-grace-SIGKILL path over a process the clawbox user owns — the unit's +# own ExecStartPre runs the same command — and `Restart=always` brings it +# back. It then PROVES the restart: a new MainPID from systemd and :9119 +# answering again. src/lib/hermes-plugin-reload.ts and +# /setup-api/hermes/plugins/reload are built on it and contain no sudo. +# * IT WOULD BE UNSAFE. `systemctl restart` STARTS a stopped unit, so this +# grant would let anything with clawbox-level access on an OPENCLAW box +# resurrect the Hermes dashboard that step_edition_foreign_teardown had just +# stopped and disabled — the state that had two harnesses long-polling one +# Telegram token and the box unable to receive a message for hours. +# +# install-foreign-edition-teardown.test.ts owns that invariant and +# install-sudoers-migration.test.ts asserts it from the installed file +# ("still grants nothing over a Hermes dashboard unit"). Both must stay. + # Ollama, for Settings -> Local Models. Ollama is a system unit and it holds # real RAM on an 8 GB box, so the owner has to be able to stop it and have it # stay stopped; until this grant existed nothing in the UI could. `--now` is diff --git a/mcp/README.md b/mcp/README.md index f8e4e8e10..495053209 100644 --- a/mcp/README.md +++ b/mcp/README.md @@ -83,6 +83,7 @@ chronically-failing tool takes *every* ClawBox tool offline for the agent. | | OpenClaw | Hermes | |---|---|---| | Capability store | `app_search`, `app_install` | `skill_search`, `skill_info`, `skill_install`, `skill_list`, `skill_uninstall` | +| Plugin reload (`hermes_plugins_reload`) | **no** — no plugin system | yes | | AI configuration | in Settings (gateway-owned) | `ai_list_models`, `ai_set_provider`, `ai_set_model` | | Coding family (`bash`, file tools, web tools) | yes | **no** — Hermes ships its own, and a second unguarded shell doubles the attack surface for no gain | | Coding agent (`coding_agent_run/status/stop`, `coding_secret_list`) | when the owner switched it on | when the owner switched it on | @@ -102,6 +103,63 @@ chronically-failing tool takes *every* ClawBox tool offline for the agent. | `clawbox_health` | Is the device API reachable and is our token accepted. Separates auth from connectivity. | | `clawbox_context` | The device field guide, the webapp storage/styling rules, and whose screen the browser tools drive (`BROWSER_GUIDE` — the desktop's window while the owner's real-browser setting is on, an invisible one when it is off). The guide is one file, `Clawbox.md`, filtered before it is served: `` blocks follow the ACTIVE HARNESS (tool sets) and `` blocks follow the INSTALL (what the device has), so a `dual` box is told about both harnesses and a Hermes agent is never handed the OpenClaw toolbelt. | +### Hermes plugins (Hermes only) +`hermes_plugins_reload` + +A PLUGIN is not a skill, and the difference is the whole reason this tool +exists. A skill is re-read per turn; a plugin is scanned **once, when the agent's +process starts** — `discover_plugins(force=True)` at start, and +`_ensure_plugins_discovered()` returning early ever after, with nothing reachable +over the dashboard socket passing its `force` flag. Hermes knows: every +`plugins install` ends with *"Restart the gateway for the plugin to take +effect."* + +On this SKU the process serving chat is `clawbox-hermes-dashboard.service`, so +that instruction means "restart the dashboard" — and the assistant cannot, +because `sudo systemctl restart` is refused (agent shells run with +`no_new_privs`). Measured on the owner's box: a plugin installed at 12:59 into a +dashboard up since 10:52 was proven working in a fresh `hermes chat -q`, listed +as `enabled` by `hermes plugins list`, and invisible to every chat the owner +opened, new sessions included. + +This tool is the supported way to ask, and **it is not a privilege**. It posts to +`/setup-api/hermes/plugins/reload`, whose restart is `bounceHermesDashboard()`: +`hermes dashboard --stop`, upstream's own SIGTERM path over a process the clawbox +user already owns, with the unit's `Restart=always` bringing it back and the +route waiting for a new main PID and for :9119 to answer before it reports +`ready`. **No sudoers grant is added or needed**, and one must not be: `systemctl +restart` also STARTS a stopped unit, which would let an OpenClaw box resurrect +the dashboard its foreign-edition teardown had just stopped and disabled +(`install-sudoers-migration.test.ts`, `install-foreign-edition-teardown.test.ts`). + +The device does this by itself as well — a watcher in the web server bounces the +dashboard when `~/.hermes` really declares a different plugin set — so the tool +is for the deliberate case: call it once, right after `hermes plugins +install/enable/disable/remove`. The two share one baseline (`process-store.ts`), +so a reload asked for here does not earn a second bounce from the watcher eight +seconds later. + +What the answer's `loaded` and `stale` are worth: `loaded` is HERMES' OWN +registry, read with `plugins.list` over the dashboard socket — `null` means the +process could not be asked and never "it loaded nothing", while `[]` means it +answered and has none of them on. `stale` says the box declares a plugin as +enabled that the running registry does not have; it is derived from that +registry rather than from a file's mtime, so an unrelated Settings save cannot +make it true. + +Two answers that must not be collapsed. `restarted` without `serving_again` means +systemd owns the restart and it is on its way; that is not a failure and calling +again would stop a dashboard in the middle of coming back. And `loaded` is +`"could not be established on this device"` rather than an empty list whenever +the running agent could not be asked what it registered — a plugin whose only +registrations are tools logs them below the level that read can see, so an empty +list would have the assistant tell an owner their plugin is missing from a device +that is serving it. + +**The owner's open chat window closes with the restart.** That is the feature, not +a fault: the box shows them a notice saying to open a new chat, and the tool's +answer repeats it so the assistant says the same thing. + ### Hermes skills (Hermes only) `skill_search` · `skill_info` · `skill_install` · `skill_list` · `skill_uninstall` diff --git a/mcp/clawbox-mcp.ts b/mcp/clawbox-mcp.ts index dd572c674..c0d29eb54 100644 --- a/mcp/clawbox-mcp.ts +++ b/mcp/clawbox-mcp.ts @@ -63,6 +63,7 @@ import { import { registerImprovementTools } from "./tools/improvement"; import { registerMediaTools } from "./tools/media"; import { registerOrientationTools } from "./tools/orientation"; +import { registerHermesPluginTools } from "./tools/hermes-plugins"; import { registerSkillTools } from "./tools/skills"; import { registerMemoryTools } from "./tools/memory"; import { registerSystemTools } from "./tools/system"; @@ -195,6 +196,11 @@ export async function buildServer( // (see `watchEmailReadability` in main()). Nothing else here is re-asked. registerOrientationTools(reg, ctx); registerSkillTools(reg); + // Hermes-only, and the registrar drops it on OpenClaw. It is next to the skill + // family because the two are how a Hermes box gains abilities — but a skill is + // re-read per turn and a PLUGIN is scanned once per process, which is the whole + // reason this tool has to exist. + registerHermesPluginTools(reg); registerMemoryTools(reg); registerAiTools(reg, ctx); registerSystemTools(reg, ctx); diff --git a/mcp/tools/hermes-plugins.ts b/mcp/tools/hermes-plugins.ts new file mode 100644 index 000000000..5aae28c48 --- /dev/null +++ b/mcp/tools/hermes-plugins.ts @@ -0,0 +1,146 @@ +// Making a Hermes plugin the assistant just installed actually reach the chat. +// +// THE FAILURE THIS TOOL REMOVES, from the owner's own transcript (2026-09-18). +// The assistant installed and enabled the `superpowers` plugin, verified it +// worked by running `hermes chat -q` — a FRESH process — and reported success. +// The chat the owner was looking at is served by a long-lived process that had +// started hours earlier, and Hermes scans for plugins exactly once per process. +// So the plugin was installed, provably working, and invisible to every chat the +// owner opened, including new sessions. +// +// The assistant then did the only sensible thing and tried `sudo systemctl +// restart clawbox-hermes-dashboard`. It was refused — agent shells run with +// `no_new_privs`, and no such grant exists or should — and it stopped there. +// +// This tool is what it should have reached for instead. What it asks for is not +// privileged: the route behind it stops a process the clawbox user already owns +// and lets systemd's `Restart=always` bring it back. + +import { apiPost, type ApiOptions } from "../lib/api"; +import { type ErrorRule } from "../lib/errors"; +import { json, type Registrar } from "../lib/register"; + +/** What POST /setup-api/hermes/plugins/reload answers. */ +interface ReloadBody { + restarted?: unknown; + inFlight?: unknown; + ready?: unknown; + plugins?: unknown; + loaded?: unknown; + stale?: unknown; + detail?: unknown; +} + +/** + * The restart takes seconds, not milliseconds: a stop, systemd's `RestartSec=5`, + * two `ExecStartPre` steps and the process binding its socket. This waits out + * the route's whole worst case plus the round trip rather than abandoning a + * restart that is working — timing out here is a FALSE FAILURE on the one call + * whose purpose is to say whether the plugin is live. + * + * THE ARITHMETIC, spelled out because it was wrong once and drifted: + * + * - the bounce waits `DASHBOARD_RESPAWN_WAIT_MS` (45 s) for a new main pid and + * the socket, and may spend one more `systemctl show` (5 s) deciding whether + * anything is still coming — 50 s (`src/lib/hermes-dashboard-control.ts`); + * - the post-bounce read is capped at `POST_BOUNCE_READ_BUDGET_MS` (10 s) + * rather than the ~33 s its three RPC budgets allow + * (`src/lib/hermes-plugin-reload.ts`); + * - plus the owner notice and the round trip. + * + * 60 s used to sit under that sum, so a slow box answered the agent "the restart + * was started but did not report back in time" over a dashboard that was already + * serving the plugin. 90 s clears it with room, and the note below is why + * waiting is the right trade in the first place. + */ +const RELOAD_TIMEOUT_MS = 90_000; + +const RELOAD_RULES: ErrorRule[] = [ + { + status: 404, + code: "NOT_SUPPORTED_HERE", + message: "This ClawBox does not run the Hermes agent, so it has no plugins to reload.", + next: "Do not retry. Report that this device has no Hermes plugins.", + }, + { + status: 502, + code: "ENDPOINT_DOWN", + message: "The Hermes agent could not be restarted, so the plugin change is not active yet.", + next: "Tell the owner the plugin change was saved but could not be applied to the running agent, and that a reboot from Settings will apply it.", + }, +]; + +/** + * Timing out is NOT a reason to call this again. The restart is already under + * way on the box, and a second call would stop a dashboard that is in the middle + * of coming back — turning one bounce into two outages for the owner. + */ +const RELOAD_TIMEOUT_NOTE = { + message: "The restart was started but did not report back in time.", + next: "Do not call this again. Wait about a minute, then use clawbox_health or simply tell the owner to open a new chat.", +}; + +export function registerHermesPluginTools(reg: Registrar): void { + reg.tool( + "hermes_plugins_reload", + "Restart this device's Hermes agent so it loads plugins installed or enabled since it started. " + + "Hermes only scans for plugins when its process starts, so a plugin you add with " + + "`hermes plugins install`, `enable`, `disable` or `remove` does NOT reach the chat until this " + + "is called — not even in a new chat session. Call it once, right after any of those commands. " + + "Never try `sudo systemctl restart` for this; it is refused, and this tool is the supported way. " + + "The owner's open chat window closes when the agent restarts, so tell them to open a new chat. " + + "Answers which plugins the device now declares and, where it can be established, which ones the " + + "restarted agent actually loaded.", + {}, + // HERMES ONLY. On OpenClaw there is no dashboard and no plugin system, and a + // tool that 404s for ever there trips the per-server circuit breaker that + // takes every ClawBox tool offline for the agent. + { editions: ["hermes"], destructive: true, profile: "core" }, + async () => { + const options: ApiOptions = { + timeoutMs: RELOAD_TIMEOUT_MS, + rules: RELOAD_RULES, + onTimeout: RELOAD_TIMEOUT_NOTE, + }; + const body = await apiPost("/setup-api/hermes/plugins/reload", {}, options); + const plugins = Array.isArray(body.plugins) ? body.plugins.map(String) : []; + // `loaded` is null when the running agent could not be asked what it + // registered, and that is NOT "it loaded nothing" — passing an empty list + // on would have the assistant tell the owner their plugin is missing from a + // device that is serving it. The null is carried through as an explicit + // "could not be established". + const loaded = Array.isArray(body.loaded) ? body.loaded.map(String) : null; + // On most devices `loaded` cannot be read at all — the dashboard publishes + // no plugin-registration lines — so THIS is the fact that proves the + // restart took: the process now serving chat is no longer behind the files. + const stale = typeof body.stale === "boolean" ? body.stale : null; + // Another caller — the owner pressing the card's own reload — already owns + // this restart. Reported as a distinct fact rather than as a failure, and + // with the same instruction the timeout note carries: the restart IS + // happening, and calling again would stop a dashboard mid-recovery. + if (body.inFlight === true) { + return json({ + restarted: false, + restart_already_under_way: true, + plugins, + tell_the_owner: "The assistant is already restarting — open a new chat in a moment to use the plugin.", + next: "Do not call this again.", + }); + } + return json({ + restarted: body.restarted === true, + // `restarted` without `ready` means systemd owns the restart and it is on + // its way back. That is not a failure and must not be retried. + serving_again: body.ready === true, + plugins, + ...(loaded ? { loaded } : { loaded: "could not be established on this device" }), + ...(stale === null + ? {} + : stale + ? { warning: "the agent is still behind the files — the plugin is NOT loaded yet" } + : { up_to_date: "the agent now serving chat has read the current plugin set" }), + tell_the_owner: "Open a new chat to use the plugin — the previous chat window closed with the restart.", + }); + }, + ); +} diff --git a/src/app/setup-api/clawkeep/restore/route.ts b/src/app/setup-api/clawkeep/restore/route.ts index a4c81e1a0..5f0d73d7e 100644 --- a/src/app/setup-api/clawkeep/restore/route.ts +++ b/src/app/setup-api/clawkeep/restore/route.ts @@ -11,7 +11,7 @@ import { } from "@/lib/clawkeep"; import { getEdition } from "@/lib/harness"; import { HERMES_DASHBOARD_UNIT } from "@/lib/hermes-dashboard-auth"; -import { bounceHermesDashboard } from "@/lib/hermes-dashboard-control"; +import { bounceHermesDashboardShared } from "@/lib/hermes-bounce-claim"; import { GATEWAY_PORT, gatewayReadyWaitMs } from "@/lib/openclaw-config"; import { hasOwnerSession } from "@/lib/owner-session"; import { waitForPortOpen } from "@/lib/port-probe"; @@ -83,12 +83,20 @@ interface RestartOutcome { async function restartStateHolder(edition: string): Promise { if (edition === "hermes") { - // No exception to quote: bounceHermesDashboard() never throws, it answers - // which of the three things happened. Say the one the owner can act on - // rather than inventing a detail we do not have. - const outcome = await bounceHermesDashboard(); + // No exception to quote: the bounce never throws, it answers which of the + // things happened. Say the one the owner can act on rather than inventing a + // detail we do not have. + // + // THROUGH THE SHARED CLAIM, not `bounceHermesDashboard()` directly. This + // restore has just rewritten the whole of `~/.hermes` — the plugin + // directory, the install ledger and config.yaml — so the plugin watcher is + // about to see a new declaration; going round the claim left its gate down + // (a window opened over this very bounce) and its baseline behind (a second + // bounce eight seconds later). One restore was two chat outages and two + // "open a new chat" notices. + const outcome = await bounceHermesDashboardShared("a ClawKeep restore replaced ~/.hermes"); if (outcome === "restarted") return { errors: [], pending: [] }; - if (outcome === "pending") { + if (outcome === "pending" || outcome === "in_flight") { const detail = "was restarted and is not serving the restored state yet"; console.warn(`[clawkeep/restore] ${HERMES_DASHBOARD_UNIT} ${detail}`); return { errors: [], pending: [`${HERMES_DASHBOARD_UNIT}: ${detail}`] }; diff --git a/src/app/setup-api/hermes/plugins/reload/route.ts b/src/app/setup-api/hermes/plugins/reload/route.ts new file mode 100644 index 000000000..b902fb17d --- /dev/null +++ b/src/app/setup-api/hermes/plugins/reload/route.ts @@ -0,0 +1,154 @@ +export const dynamic = "force-dynamic"; + +import { NextResponse } from "next/server"; +import { hasHermesHarness } from "@/lib/edition-source"; +import { reloadHermesPlugins } from "@/lib/hermes-plugin-reload"; +import { readHermesPluginState } from "@/lib/hermes-plugin-set"; +import { requireSession } from "@/lib/route-auth"; +import { isSameOriginRequest } from "@/lib/same-origin"; + +/** + * POST /setup-api/hermes/plugins/reload — make the running agent re-scan for + * plugins, on purpose. + * + * The watcher in `src/lib/hermes-plugin-reload.ts` does this by itself when + * `~/.hermes` changes. This is the same thing, asked for rather than noticed, and + * it exists because of what the assistant on the owner's box did next after + * installing a plugin: it tried `sudo systemctl restart + * clawbox-hermes-dashboard`, was refused (agent shells run with + * `no_new_privs`, and no such grant exists or should), and stopped there — with + * the plugin installed, proven to work in a fresh CLI process, and invisible to + * every chat the owner opened. + * + * THE AGENT IS AN INTENDED CALLER, which is unusual for a route that changes + * device state and is deliberate here. `requireSession` admits the owner's + * session cookie OR the MCP bearer. Refusing the bearer would leave the + * assistant exactly where it was: able to install a plugin and unable to make it + * work. What it is being handed is not privilege — the restart underneath is + * `bounceHermesDashboard()`, which stops a process the clawbox user already owns + * and lets `Restart=always` bring it back. This file names no unit, takes no + * argument, and contains neither `sudo` nor `systemctl`; its own suite asserts + * that last part, because the moment it did it would need a grant that + * `install-sudoers-migration.test.ts` refuses to give — and that grant would let + * an OpenClaw box START the dashboard its foreign-edition teardown had stopped. + * + * Answers `{ restarted, inFlight, ready, plugins, loaded, detail }`: + * - `restarted` — the restart was TAKEN. + * - `inFlight` — nothing was done because another caller's restart is + * already under way. A 200, not a 502: the restart asked for + * is happening, and a second bounce would add an outage. + * - `ready` — and the replacement is SERVING. `restarted && !ready` means + * systemd owns it and it is on its way; acting again makes it + * worse, so that is a 200, not an error. + * - `plugins` — what the box declares now. + * - `loaded` — what the REPLACEMENT proved it registered, or null when this + * box could not be asked. Never an empty array standing in for + * "unknown", and on most boxes it IS null: the dashboard + * publishes no plugin-registration lines (measured on the + * owner's device). + * - `stale` — whether the process now serving chat is STILL behind the + * files. `false` is the proof the restart took, and it is the + * one that holds on a box where `loaded` cannot be read. + * + * A restart that could not be taken is a 502 rather than a 200 with a flag: the + * caller is an agent about to tell the owner their plugin is live, and a 200 is + * the one thing that would make it say so. + */ + +export async function POST(request: Request) { + const unauthorized = await requireSession(request); + if (unauthorized) return unauthorized; + + // AND OUR PAGE ONLY, on top of that. The owner's browser attaches the session + // cookie to a POST any other site's page fires at the box, and this one drops + // their chat window — a page they merely visited could restart the assistant + // as often as it liked. `isSameOriginRequest` waves a header-less caller + // through, which is exactly what keeps the intended agent caller working: the + // MCP server sends neither `Origin` nor `Sec-Fetch-Site`, and its credential + // is what the gate above already decided on. + if (!isSameOriginRequest(request)) { + return NextResponse.json( + { + error: "Reloading the Hermes plugins only works from this ClawBox's own pages.", + code: "cross_origin", + }, + { status: 403 }, + ); + } + + // THE EDITION, NOT THE ACTIVE HARNESS. On the premium `dual` SKU the dashboard + // runs whichever agent is serving the owner — `install.sh` enables the unit for + // hermes AND dual — so asking which harness is active would refuse this on a + // box that has the very process it restarts. An OpenClaw box has no dashboard + // by design, and reaching for one there would be this route starting a unit the + // foreign-edition teardown deliberately stopped and disabled. + if (!hasHermesHarness()) { + return NextResponse.json( + { + error: "This ClawBox does not run the Hermes agent, so it has no plugins to reload.", + code: "no_hermes_dashboard", + }, + { status: 404 }, + ); + } + + const result = await reloadHermesPlugins("a reload was requested through /setup-api"); + const body = { + restarted: result.restarted, + // A restart ANOTHER caller already owns — the owner's card and the agent's + // tool pressed together. Not a failure: the restart the caller asked for is + // happening, so this is a 200 with a flag rather than the 502 a bounce that + // could not be taken earns. + inFlight: result.inFlight, + ready: result.ready, + plugins: result.plugins, + loaded: result.loaded, + stale: result.stale, + detail: result.detail, + }; + return NextResponse.json(body, { + status: result.restarted || result.inFlight ? 200 : 502, + headers: { "Cache-Control": "no-store" }, + }); +} + +/** + * GET — the same two facts, without restarting anything. + * + * "Installed" and "loaded" are different questions (see + * `src/lib/hermes-plugin-set.ts`), and until this existed nothing on the box + * could answer the second. `stale` is the answer from the RUNNING REGISTRY — the + * box declares a plugin as enabled that the process serving chat does not have + * on — and it holds even for a plugin whose registrations Hermes logs below the + * level the journal read can see. It is deliberately NOT an mtime: "the + * declaration changed after the dashboard started" is a weaker and different + * fact (`changedAfterStart`), true on every Settings save because config.yaml is + * rewritten by each one, which had the MCP tool warn "the agent is still behind + * the files" about a plugin that had been loaded for hours. `null` is "could not + * be established", never "no". + */ +export async function GET(request: Request) { + const unauthorized = await requireSession(request); + if (unauthorized) return unauthorized; + + if (!hasHermesHarness()) { + return NextResponse.json( + { + error: "This ClawBox does not run the Hermes agent, so it has no plugins to report.", + code: "no_hermes_dashboard", + }, + { status: 404 }, + ); + } + + const state = await readHermesPluginState(); + return NextResponse.json( + { + plugins: state.declared, + loaded: state.loaded, + stale: state.stale, + dashboardStartedAt: state.dashboardStartedAt, + }, + { headers: { "Cache-Control": "no-store" } }, + ); +} diff --git a/src/instrumentation.ts b/src/instrumentation.ts index bddf1062c..d075ca903 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -422,6 +422,25 @@ export async function register() { } catch (err) { console.error('[instrumentation] Could not resume email chat approvals:', err instanceof Error ? err.message : err) } + try { + // A Hermes plugin the owner installs after boot never reaches the chat on + // its own: Hermes scans for plugins once per process, and the process + // serving chat is a long-lived service. Nothing else on the box notices, so + // this watches ~/.hermes and bounces the dashboard when the plugin set + // really changes — see src/lib/hermes-plugin-reload.ts for why that restart + // needs no privilege and must not be given one. + // + // Starts nothing on a box with no Hermes dashboard (`hasHermesHarness()`). + // Its first look is a BASELINE unless the running dashboard is demonstrably + // BEHIND the files AND those files were touched after it started — the + // `register-mcp.sh` case, true at most once per dashboard — so an ordinary + // web-server restart never restarts the owner's chat backend. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { startHermesPluginWatcher } = require('./lib/hermes-plugin-reload') + startHermesPluginWatcher() + } catch (err) { + console.error('[instrumentation] Could not start the Hermes plugin watcher:', err instanceof Error ? err.message : err) + } try { // On the OpenClaw arm the memory-status probe boots a whole OpenClaw // process (~8 s on a Jetson), so it is paid once, after the boot rush diff --git a/src/lib/hermes-bounce-claim.ts b/src/lib/hermes-bounce-claim.ts new file mode 100644 index 000000000..dbc3e9464 --- /dev/null +++ b/src/lib/hermes-bounce-claim.ts @@ -0,0 +1,250 @@ +import { + bounceHermesDashboard, + hermesDashboardMainPid, + type HermesBounceOutcome, +} from "@/lib/hermes-dashboard-control"; +import { readHermesPluginDeclaration } from "@/lib/hermes-plugin-set"; +import { logSafe } from "@/lib/log-safe"; +import { processStore } from "@/lib/process-store"; + +/** + * WHO IS ALLOWED TO STOP THE HERMES DASHBOARD RIGHT NOW, and what the + * replacement will have read when it comes back. + * + * A module of its own, and a small one on purpose. The bounce has three callers + * in the product — the plugin watcher and its route, a ClawKeep restore, and the + * image refresh — and the two that are not the watcher used to call + * `bounceHermesDashboard()` directly, so neither raised the watcher’s gate nor + * moved its baseline: one ClawKeep restore was TWO chat outages, its own and the + * watcher’s eight seconds later, with two "open a new chat" notices about a + * plugin set nobody had touched. The claim belongs to the BOUNCE rather than to + * one of its callers, so it lives here, where every caller can reach it without + * dragging in the owner-notice machinery the watcher needs. + */ + +export interface PluginReloadState { + /** The signature the running dashboard is believed to have read. */ + baseline: string | null; + /** A bounce is in flight, from the watcher, the route, or anything else. */ + bouncing: boolean; + /** + * A bounce was TAKEN for this signature and the replacement was not serving + * by the end of the budget. The baseline must not move on that — "the stop + * took" is not "the new process is up" — but the answer is to WAIT and look + * again, never to stop it once more. See the watcher's reconcile step. + */ + awaiting: string | null; + /** + * The main pid that bounce tried to stop, so the reconcile can tell a + * REPLACEMENT from the process that never went away. + * + * `bounceHermesDashboard` answers `pending` for any unit state that is not + * `running`/`down`, and "systemd could not be asked, twice" is one of those — + * with the old process still serving. The port probe then succeeds, because it + * is the OLD process answering, and the reconcile recorded the change as + * loaded and stopped trying. Null when it could not be read, which is not a + * fact about anything and leaves the probe alone. + */ + awaitingPid: number | null; + /** + * WHAT THE BOUNCE THAT HOLDS THE CLAIM IS FOR, so a second caller can be + * told apart from a duplicate of the one already running. Null while nothing + * is in flight. + */ + activeKind: HermesBounceKind | null; + /** + * The declaration signature the in-flight PLUGIN RELOAD is acting on, when it + * is known. Null for every other kind of bounce and whenever the caller could + * not read one — and null is "cannot be compared", never "the same". + */ + activeSignature: string | null; + /** + * A caller arrived while the claim was held, so ONE more bounce is owed. + * + * `"in_flight"` says another restart owns the dashboard. It does NOT say the + * replacement will have read what THAT caller just wrote: every caller here + * mutates `~/.hermes` and then asks for the restart, and the running bounce + * read the files at the moment IT started — a ClawKeep restore that rewrites + * the whole directory a second later is served by a process that never saw + * it, and nothing tried again. The declaration signature cannot catch it + * either: a restore can put back a byte-identical plugin set beside a + * completely different `state.db`. + * + * Recorded rather than acted on inside the running bounce, because the + * running bounce is somebody's REQUEST — the route budgets 45 s and the MCP + * tool 60 — and a second stop chained onto it would answer that request + * ~80 s late over a restart that had already succeeded. + */ + owed: boolean; +} + +/** + * The baseline of a box the first look found BEHIND the files: a value no + * declaration signature can take, so every comparison with it says "changed". + */ +export const BASELINE_BEHIND = "behind"; + +/** + * WHAT A CALLER IS BOUNCING FOR, and the only thing that decides whether a + * second one arriving mid-bounce owes another restart. + * + * `"plugin_reload"` is the one mutation this module can COMPARE: it is entirely + * described by the declaration signature, so two of them for the same signature + * are the same request and the replacement already coming up serves both. + * `"mutation"` is everything else — a ClawKeep restore, the image refresh — and + * it is the DEFAULT on purpose: a caller that has not said what it changed is a + * caller whose change the signature cannot represent. + */ +export type HermesBounceKind = "plugin_reload" | "mutation"; + +export function hermesBounceState(): PluginReloadState { + return processStore("clawbox.hermes-plugin-reload", () => ({ + baseline: null, + bouncing: false, + awaiting: null, + awaitingPid: null, + activeKind: null, + activeSignature: null, + owed: false, + })); +} + +/** Exported for the suites, which build several watchers in one process. */ +export function _resetHermesPluginReloadStateForTests(): void { + const state = hermesBounceState(); + state.baseline = null; + state.bouncing = false; + state.awaiting = null; + state.awaitingPid = null; + state.activeKind = null; + state.activeSignature = null; + state.owed = false; +} + + +/** + * Restart the dashboard under the shared claim, and record what the replacement + * will have read. NEVER THROWS — `bounceHermesDashboard` does not either. + * + * `"in_flight"` is its own answer and not a failure: another caller owns this + * restart, its replacement reads the files as they are NOW, and a second stop + * would add an outage and prove nothing. + * + * @param reason why, in the caller's words, for the journal line. + */ +export async function bounceHermesDashboardShared( + reason: string, + opts: { signature?: string; kind?: HermesBounceKind } = {}, +): Promise { + const shared = hermesBounceState(); + // UNSAID IS "MUTATION": a caller that has not named its kind has changed + // something this module cannot compare, and it keeps the debt. + const kind: HermesBounceKind = opts.kind ?? "mutation"; + // THE CLAIM IS TAKEN BEFORE THE FIRST AWAIT. It used to be set AFTER reading + // the declaration, so two callers arriving together — the owner's card and the + // agent's `hermes_plugins_reload` — both got past the check and both stopped + // the dashboard, and the first `finally` then cleared a flag the second bounce + // was still relying on. That cleared flag is the window the watcher SIGTERMs a + // restarting dashboard through. Only the caller that SET it clears it. + // + // A CLAIM AND NOT A QUEUE, deliberately — `createSerialLock` is the other tool + // this codebase has for "one at a time" and it is the wrong one here: the + // second caller would WAIT and then stop the dashboard a second time for a + // plugin set the first bounce's replacement has already read, and pay the + // whole of a second bounce inside its own request to do it. + // + // WHAT IT OWES INSTEAD is one more bounce, recorded and handed to the watcher + // (see `owed`): this caller's own mutation landed after the running bounce + // read the files, so the replacement coming up may not carry it. + // + // UNLESS IT IS THE SAME REQUEST. A debt for EVERY in-flight caller made the + // commonest pair of all — the owner's card and the agent's + // `hermes_plugins_reload`, pressed within a second of each other — cost the + // owner's chat a SECOND restart: two plugin reloads for one declaration, the + // replacement already coming up carrying it, and `BASELINE_BEHIND` on its way + // out telling the watcher to bounce again for nothing. Two plugin reloads with + // the same signature ARE one request; anything else is not, and an unknown + // signature on either side is not a match. A restore or an image refresh keeps + // the debt whatever the signature says — see `HermesBounceKind`. + if (shared.bouncing) { + const duplicate = + kind === "plugin_reload" && + shared.activeKind === "plugin_reload" && + typeof opts.signature === "string" && + shared.activeSignature !== null && + opts.signature === shared.activeSignature; + if (!duplicate) shared.owed = true; + return "in_flight"; + } + shared.bouncing = true; + shared.activeKind = kind; + // From the caller when it read one, and filled in below from this module's own + // reading when it did not. Never left over from the previous bounce. + shared.activeSignature = kind === "plugin_reload" ? opts.signature ?? null : null; + try { + // WHAT THE REPLACEMENT WILL HAVE READ, captured BEFORE the stop and never + // after it. A bounce is up to 45 s long and `hermes plugins install` can + // land another write inside that window; recording the post-bounce + // signature would mark a change the new process never saw as loaded, and + // nothing would ever try again. The pre-bounce reading is the conservative + // one: a write that landed during the bounce simply stays outstanding and + // the watcher catches it. The caller passes the signature it already read + // rather than paying for a second reading of the same two files. + const acted = + opts.signature ?? (await readHermesPluginDeclaration().then((d) => d.signature).catch(() => null)); + // The reading above is this bounce's answer to "which plugin set is the + // replacement getting", so a caller arriving from here on can be compared + // against it. Before it lands `activeSignature` is null, and null refuses + // the comparison rather than guessing at it. + if (kind === "plugin_reload" && shared.activeSignature === null) shared.activeSignature = acted; + // One local `systemctl show` on a path about to spend up to 45 s, so a + // `pending` outcome can be reconciled against the process it tried to stop. + const outgoing = await hermesDashboardMainPid().catch(() => ({ read: false, pid: null })); + const outcome = await bounceHermesDashboard(); + // THE BASELINE IS THIS MODULE'S, not the watcher's, so a reload asked for + // through the route, the MCP tool, a ClawKeep restore or the image refresh + // moves it too — see `PluginReloadState`. Only a bounce whose replacement is + // SERVING moves it: `pending` means the stop took and systemd owns the unit, + // which is a reason to look again rather than to call the change loaded. + if (acted !== null) { + if (outcome === "restarted") { + shared.baseline = acted; + shared.awaiting = null; + shared.awaitingPid = null; + } else if (outcome === "pending") { + shared.awaiting = acted; + shared.awaitingPid = outgoing.read ? outgoing.pid : null; + } + } + if (outcome === "failed") { + console.error(`[hermes/bounce] ${logSafe(reason, 120)}: the Hermes dashboard could not be restarted`); + } + return outcome; + } finally { + // THE DEBT IS PAID BY THE WATCHER, not by a second stop chained onto this + // caller's request. `BASELINE_BEHIND` is the sentinel that already means + // "whatever the running dashboard loaded, it is not what is on disk" — no + // 32-hex signature can equal it — so the next tick opens a window and takes + // exactly ONE more bounce, debounced with anything else that arrives. That + // is what makes this work for a restore whose plugin set did not move: the + // watcher's own comparison is on the signature, and this is not. + // + // `awaiting` goes with it: a `pending` recorded above would otherwise have + // the next tick reconcile the very restart we have just decided is not + // enough. + // + // Wholly SYNCHRONOUS, with no await between the read and the clear, so a + // caller arriving mid-way cannot have its request dropped here — it either + // set the flag before this runs, or finds `bouncing` false and takes the + // claim itself. + if (shared.owed) { + shared.owed = false; + shared.baseline = BASELINE_BEHIND; + shared.awaiting = null; + shared.awaitingPid = null; + } + shared.bouncing = false; + shared.activeKind = null; + shared.activeSignature = null; + } +} diff --git a/src/lib/hermes-dashboard-control.ts b/src/lib/hermes-dashboard-control.ts index 465c1b81c..40ed71647 100644 --- a/src/lib/hermes-dashboard-control.ts +++ b/src/lib/hermes-dashboard-control.ts @@ -52,6 +52,8 @@ const STOP_TIMEOUT_MS = 15_000; const DASHBOARD_RESPAWN_WAIT_MS = 45_000; /** Gap between systemd queries while waiting for the replacement process. */ const RESPAWN_POLL_MS = 500; +/** How long to wait before re-asking systemd for the baseline main pid. */ +const BASELINE_RETRY_DELAY_MS = 250; /** * Read per call so a box that needs longer — or a test that cannot spend 45 s @@ -206,7 +208,7 @@ export function classifyUnitState(unit: { } /** - * The PID systemd currently considers the unit's main process, or null. + * The PID systemd currently considers the unit's main process. * * THE IDENTITY OF THE PROCESS, which a socket cannot give. `Type=simple` means * :9119 says only "something is listening", and between our stop and @@ -215,17 +217,54 @@ export function classifyUnitState(unit: { * killed answers, wait for the port to close first and a fast respawn beats the * first probe. `MainPID` changes exactly once, when systemd starts the * replacement. Read through the same by-name `systemctl show` as every other - * property here, and 0 (no running main process) reads as null. + * property here. + * + * TWO NULLS, AND THEY ARE NOT THE SAME ANSWER. `MainPID=0` is systemd saying + * "no running main process" — a fact, and the state a stopped unit is in. A + * MISSING property is systemd not having been asked successfully at all: no + * systemctl, a timeout, a `show` that produced nothing. Folded together, the + * second read as a valid baseline of "nothing was running", and the very next + * pid — the process this bounce was supposed to stop — then satisfied + * `waitForReplacement` and the bounce reported `restarted` over a dashboard it + * had not replaced. So `read` is carried beside the value and the caller acts + * on the difference. + */ +export interface MainPidRead { + /** systemd answered the question at all. */ + read: boolean; + /** The main pid, or null for a valid `MainPID=0`. */ + pid: number | null; +} + +async function mainPid(): Promise { + const raw = (await showUnit(["MainPID"])).MainPID; + if (raw === undefined) return { read: false, pid: null }; + const value = Number(raw); + // A property that is present but not a whole non-negative number is not an + // answer either — the same "could not be asked" as an absent one. + if (!Number.isInteger(value) || value < 0) return { read: false, pid: null }; + return { read: true, pid: value > 0 ? value : null }; +} + +/** + * Read the baseline pid, with one retry: this single read decides what the + * whole bounce is measured against, and a transient `systemctl show` failure + * must not become a baseline. A second attempt costs one local read on a path + * that is about to spend up to 45 s. */ -async function mainPid(): Promise { - const value = Number((await showUnit(["MainPID"])).MainPID); - return Number.isInteger(value) && value > 0 ? value : null; +async function readBaselinePid(): Promise { + const first = await mainPid(); + if (first.read) return first; + await new Promise((resolve) => setTimeout(resolve, BASELINE_RETRY_DELAY_MS)); + return await mainPid(); } async function waitForReplacement(previousPid: number | null, deadline: number): Promise { for (;;) { - const pid = await mainPid(); - if (pid !== null && pid !== previousPid) return true; + const current = await mainPid(); + // A read that FAILED establishes nothing: it is neither the replacement nor + // proof there is none, so the wait simply continues. + if (current.read && current.pid !== null && current.pid !== previousPid) return true; const remaining = deadline - Date.now(); if (remaining <= 0) return false; await new Promise((resolve) => setTimeout(resolve, Math.min(RESPAWN_POLL_MS, remaining))); @@ -247,6 +286,35 @@ async function waitForReplacement(previousPid: number | null, deadline: number): */ export type HermesBounceOutcome = "restarted" | "pending" | "failed"; +/** + * Is the dashboard SERVING right now? One probe, no stop, no waiting. + * + * The question a caller has after a bounce came back `pending`: the stop took + * and systemd owns the unit, so the thing to do is find out whether it has + * finished coming up — never to stop it again, which would turn one outage into + * two. `Type=simple` means the unit reads `active` the instant ExecStart forks, + * so the SOCKET is the fact and the unit state is only good for telling "still + * on its way" from "nothing is coming" (`hermesDashboardUnitState`). + */ +export async function hermesDashboardServing(): Promise { + // A non-positive budget is "one probe, then give up" — see `waitForPortOpen`. + return await waitForPortOpen(DASHBOARD_PORT, DASHBOARD_HOST, { timeoutMs: 0 }); +} + +/** + * WHICH PROCESS is serving the dashboard right now, for a caller that has to + * tell a replacement from the process a bounce tried to stop. + * + * The same two nulls {@link MainPidRead} keeps apart, and for the same reason: + * `pid: null` with `read: true` is systemd saying "no running main process", + * while `read: false` is systemd not having been asked successfully at all. + * A caller that folded them would read a failed `systemctl show` as proof the + * old process had gone. + */ +export async function hermesDashboardMainPid(): Promise { + return await mainPid(); +} + /** * Stop the dashboard, wait for systemd to bring it back, and say which of the * three things happened. @@ -265,15 +333,41 @@ export type HermesBounceOutcome = "restarted" | "pending" | "failed"; export async function bounceHermesDashboard(): Promise { if (!(await restartsItself())) return "failed"; // Read BEFORE the stop: this is the process the answer is measured against. - const outgoing = await mainPid(); - const result = await runHermesCli(["dashboard", "--stop"], { timeoutMs: STOP_TIMEOUT_MS }).catch( - () => null, - ); - if (result?.code !== 0) return "failed"; + // NOTHING IS STOPPED UNTIL IT IS READ. Without a baseline this function + // cannot tell the replacement from the process it killed, so it would stop + // the owner's chat backend and then report `restarted` on the first pid it + // saw — which is the outgoing one. Refusing before the stop leaves a working + // dashboard working, and "failed" is the honest answer: nothing was done. + const baseline = await readBaselinePid(); + if (!baseline.read) { + console.error( + `[hermes] ${HERMES_DASHBOARD_UNIT} was not stopped — systemd could not be asked which process is serving it`, + ); + return "failed"; + } + const outgoing = baseline.pid; + await runHermesCli(["dashboard", "--stop"], { timeoutMs: STOP_TIMEOUT_MS }).catch(() => null); - // Past this line the restart HAS been taken: the stop exited 0 over a unit - // that restarts itself. The clock running out below is therefore "pending" — - // with one exception, asked of systemd rather than assumed, immediately after. + // THE STOP'S EXIT CODE IS NOT THE OUTCOME, and reading it as one was a false + // failure on every bounce this box performs. + // + // MEASURED on the owner's Hermes device (2026-09-18): `hermes dashboard + // --stop` exits **143** — SIGTERM, 128+15 — after 764 ms, printing + // "Terminated", while stopping the dashboard perfectly well; the unit's + // MainPID went 17768 → 17877 across that call. The CLI signals the process + // group it is itself in, so it kills its own process on the way out. The old + // `if (result?.code !== 0) return "failed"` therefore reported a failure over + // a restart that had just happened, on EVERY caller of this helper: a ClawKeep + // restore told the owner its restored state.db was not being served, the image + // refresh told them the box could not draw, and the plugin watcher sent no + // "open a new chat" notice for a chat window it had just dropped — then armed + // a retry to do it again. + // + // So the stop is VERIFIED rather than believed, which this function already + // knew how to do: the answer is systemd's NEW MainPID and a socket that + // answers, and neither of those can be faked by an exit code. That is also why + // no branch is lost — a stop that genuinely did not take is still caught, by + // the `outgoing` comparison below rather than by the CLI's word for it. // // ONE deadline across both halves — the doc block above budgets them together // because they are the same restart, and spending it twice would put the @@ -295,6 +389,19 @@ export async function bounceHermesDashboard(): Promise { // already spent the whole budget: the ceiling moves 45 s → 50 s, still far // inside the 100 s edge cut the budget above is sized against. const state = await hermesDashboardUnitState(); + // THE STOP NEVER TOOK — the branch the exit-code check used to cover, now + // asked of the thing that knows. The very same process is still the unit's + // main one and systemd calls the unit running, so nothing was stopped and + // nothing is on its way back. `pending` here would be the worst of the three + // answers: it means "leave it alone", over a dashboard that will stay stale + // until somebody acts. + const current = await mainPid(); + if (current.read && current.pid !== null && current.pid === outgoing && state === "running") { + console.error( + `[hermes] ${HERMES_DASHBOARD_UNIT} was not stopped — the same process is still serving`, + ); + return "failed"; + } console.error( `[hermes] ${HERMES_DASHBOARD_UNIT} did not come back after its stop (unit is ${state})`, ); diff --git a/src/lib/hermes-image-refresh.ts b/src/lib/hermes-image-refresh.ts index 612c0014e..7904f2129 100644 --- a/src/lib/hermes-image-refresh.ts +++ b/src/lib/hermes-image-refresh.ts @@ -1,4 +1,4 @@ -import { bounceHermesDashboard } from "@/lib/hermes-dashboard-control"; +import { bounceHermesDashboardShared } from "@/lib/hermes-bounce-claim"; import { dashboardRpc } from "@/lib/hermes-dashboard-rpc"; import { MCP_RELOAD_ASKED, reloadMcpServers, reportMcpReloadRefused } from "@/lib/hermes-mcp-reload"; @@ -182,7 +182,14 @@ export async function refreshHermesImageTools(before: boolean, after: boolean): // its way back and this call cannot prove it, which is the same as a failure // HERE: the next family pays for its own `reload.mcp`, a redundant respawn // rather than anything the owner sees. - if ((await bounceHermesDashboard()) === "restarted") { + // Through the SHARED CLAIM rather than the bouncer directly: while this + // restart is in flight the plugin watcher must open no window over it, and the + // declaration signature this replacement reads has to become the watcher's + // baseline — otherwise it bounces the owner's chat a second time for a plugin + // set the new process has already loaded. "in_flight" is not "restarted": a + // bounce somebody else owns may have started before the backend was + // installed, and this function may not claim a picture it cannot prove. + if ((await bounceHermesDashboardShared("the image backend needs the agent to re-scan")) === "restarted") { console.log("[hermes/image-refresh] bounced the Hermes dashboard so it picks up the image backend"); // The bounce takes the MCP children down with the dashboard and brings them // back, so every family's tool list is rebuilt — the same effect a reload diff --git a/src/lib/hermes-plugin-reload.ts b/src/lib/hermes-plugin-reload.ts new file mode 100644 index 000000000..b584e359b --- /dev/null +++ b/src/lib/hermes-plugin-reload.ts @@ -0,0 +1,617 @@ +import { hasHermesHarness } from "@/lib/edition-source"; +import { notifyOwner } from "@/lib/email-notify"; +import { + hermesDashboardMainPid, + hermesDashboardServing, + hermesDashboardUnitState, +} from "@/lib/hermes-dashboard-control"; +import { + BASELINE_BEHIND, + bounceHermesDashboardShared, + hermesBounceState, + _resetHermesPluginReloadStateForTests, +} from "@/lib/hermes-bounce-claim"; +import { logSafe } from "@/lib/log-safe"; +import { + readHermesPluginDeclaration, + readHermesPluginState, + type HermesPluginState, +} from "@/lib/hermes-plugin-set"; + +// Re-exported from the module the CLAIM lives in, so the route, the watcher and +// the suites keep one import path for the whole mechanism. +export { bounceHermesDashboardShared, _resetHermesPluginReloadStateForTests }; + +/** + * Make a Hermes plugin installed after boot reach the chat. + * + * ── THE MECHANISM IS HERMES' OWN, AND IT IS A RESTART ──────────────────────── + * + * Hermes has NO runtime plugin reload, and this was checked before anything here + * was written rather than assumed. Skills have `/reload-skills`; MCP has + * `reload.mcp` on the dashboard socket, which ClawBox already drives + * (`src/lib/hermes-mcp-reload.ts`); the dashboard even has + * `/api/dashboard/plugins/rescan` — but that one re-scans the dashboard's own UI + * extensions, not the agent's plugin registry. For the agent's plugins there is + * exactly one instruction, and Hermes prints it itself at the end of every + * install (`hermes_cli/plugins_cmd.py:857`): + * + * Restart the gateway for the plugin to take effect: + * hermes gateway restart + * + * On this SKU the process that serves chat is not `hermes-gateway` — that unit + * carries Telegram and friends and is inactive on the owner's box. It is + * ClawBox's own `clawbox-hermes-dashboard.service`. So the instruction reads: + * restart the dashboard. + * + * ── THERE IS NO NEW PRIVILEGE HERE, AND THERE MUST NOT BE ──────────────────── + * + * The obvious fix is a sudoers grant for `systemctl restart + * clawbox-hermes-dashboard`, and it is the wrong one twice over. + * + * It is unnecessary: `bounceHermesDashboard()` already restarts this exact unit + * with no root at all. It runs `hermes dashboard --stop`, which is upstream's + * own SIGTERM-grace-SIGKILL path over a process the clawbox user owns — the + * unit's own `ExecStartPre` runs the same command — and the unit's + * `Restart=always` brings it back. It then proves the restart rather than + * assuming it: a NEW main PID from systemd, and :9119 answering again. + * + * And it is unsafe: `systemctl restart` STARTS a stopped unit, so the grant + * would let anything with clawbox-level access on an OPENCLAW box resurrect the + * Hermes dashboard that `step_edition_foreign_teardown` had just stopped and + * disabled — the state that had two harnesses long-polling one Telegram token + * and the box unable to receive a message for hours. + * `src/tests/unit/install-foreign-edition-teardown.test.ts` owns that invariant + * and `src/tests/unit/install-sudoers-migration.test.ts` asserts it from the + * installed file ("still grants nothing over a Hermes dashboard unit"). + * + * So this module adds no grant, changes no sudoers file, and the route above it + * contains the string `sudo` nowhere — which its own suite asserts. + * + * ── WHAT IS ACTUALLY NEW ───────────────────────────────────────────────────── + * + * Knowing WHEN to restart, doing it once per burst, proving what the replacement + * loaded, and telling the owner — because the restart drops their open chat + * window. That is not a side effect to be minimised; it is the visible half of + * the feature, and the notice is what turns it from a glitch into an answer. + * + * ── WHAT IS NOT GUARDED, SAID OUT LOUD ─────────────────────────────────────── + * + * A turn in flight is interrupted. The dashboard exposes no active-run signal + * this could ask — the socket's method surface has `reload.mcp`, `reload.env`, + * `commands.catalog`, `image.generate`, the session methods and the turn + * transport, and none of them reports "a turn is running". Inferring it from a + * slow reply would be a guess, and a guess here either bounces a box that was + * merely busy or defers for ever on a box that is merely slow. The debounce + * window below is what keeps this to one interruption per action, and the notice + * is what explains it. + */ + +/** + * How long the declaration must hold still before a restart. + * + * `hermes plugins install` is several writes, not one: the plugin directory, the + * install ledger, then `plugins.enabled` in config.yaml if `--enable` was + * passed, and an `enable` afterwards is another. Each is a change, and one + * restart per write is four outages for one action — and the first of them would + * restart into a half-declared set and have to do it again. + * + * Sized against the action rather than the file system: eight seconds is longer + * than any single `hermes plugins` invocation spends between its writes, and + * short enough that an owner who installs a plugin and opens the chat finds it + * there. + */ +export const HERMES_PLUGIN_DEBOUNCE_MS = 8_000; + +/** How often the declaration is read. Two small local file reads. */ +export const HERMES_PLUGIN_POLL_MS = 5_000; + +/** + * The ceiling on retrying a restart that failed. + * + * A bounce spends up to its whole 45 s budget before answering "failed", and the + * commonest reason for that answer is a unit systemd has GIVEN UP on — a + * crash-looping dashboard past its start limit, which this module is + * unprivileged to `reset-failed`. Retrying such a box every debounce window is a + * SIGTERM every few seconds against a process that is already failing to start, + * for the life of the web server. The same shape, and the same answer, as the + * two child supervisors in `src/instrumentation-node.ts`: double the wait, cap + * it, and keep trying — because the plugin really is still not loaded. + */ +export const HERMES_PLUGIN_RETRY_MAX_MS = 5 * 60_000; + +/** + * WHAT THE RUNNING DASHBOARD IS BELIEVED TO HAVE LOADED, once per PROCESS. + * + * In `process-store.ts` and not in a module-level `let`, for the reason that + * module exists: `src/instrumentation.ts` reaches this file through + * `require('./lib/hermes-plugin-reload')` while the route `import`s it, and Next + * compiles those into two different module instances inside the one web server. + * Kept per copy, the watcher's baseline could not be moved by a reload the ROUTE + * performed — so the assistant calling `hermes_plugins_reload` right after + * `hermes plugins install` (which is exactly what the tool tells it to do) + * bounced the dashboard at once, and the watcher, which had seen the same + * signature change, bounced it again eight seconds later, inside the first + * bounce's 45 s budget: the owner's chat dropped twice in fifteen seconds and + * they were told to open a new one twice. `POST /setup-api/clawkeep/restore` has + * the same shape — it rewrites `~/.hermes` and bounces the dashboard itself. + * + * `bouncing` closes the other half of that: while ANY caller is mid-bounce the + * watcher opens no window, so it cannot SIGTERM a dashboard that is in the + * middle of coming back. + */ +/** What one `reloadHermesPlugins` call did. */ +export interface HermesPluginReloadResult { + /** The restart was TAKEN — the stop landed over a unit that restarts itself. */ + readonly restarted: boolean; + /** + * Nothing was done because ANOTHER caller's bounce is already under way. + * + * Its own field beside `restarted: false`, because the two need opposite + * things said: a bounce that could not be taken is a failure the caller must + * report, while this one means the restart the caller asked for is happening + * and asking again would add an outage. + */ + readonly inFlight: boolean; + /** …and the replacement is SERVING again. `restarted && !ready` is "on its way". */ + readonly ready: boolean; + /** What the box declares now. Always answered. */ + readonly plugins: readonly string[]; + /** What the replacement proved it registered, or null when it could not be asked. */ + readonly loaded: readonly string[] | null; + /** + * Is the process now serving chat STILL behind the files? False is the proof a + * restart actually took, and on most boxes it is the ONLY proof available — + * measured on the owner's device, the dashboard publishes no registration + * lines at all, so `loaded` is null there and this is what answers. Null means + * even this could not be established. + */ + readonly stale: boolean | null; + /** One sentence for a journal line and for the route's body. */ + readonly detail: string; +} + +/** + * The owner-facing sentence. ClawBox's words, never a plugin's own: the names + * come from a directory a publisher chose, so they are bounded and never used as + * anything but text. + */ +function noticeFor(names: readonly string[]): string { + const named = names.slice(0, 3).map((n) => logSafe(n, 40)).join(", "); + const subject = named ? `the plugin ${named}` : "a new plugin"; + return `The assistant restarted to load ${subject} — open a new chat to use it.`; +} + +/** + * Restart the dashboard so it re-scans for plugins, and report what came back. + * + * NEVER THROWS. Every caller is either a poll loop that must not die or a route + * that has to answer something; `bounceHermesDashboard` never throws either, and + * the readers below answer null rather than raising. + * + * @param reason why, in the caller's words, for the journal line. + */ +/** + * How long the POST-BOUNCE read may take before the answer goes out without it. + * + * THE RESTART IS THE OUTCOME; `loaded` and `stale` are a courtesy beside it, and + * they are the only thing this read produces. It is not cheap — `plugins.list` + * is budgeted at 8 s, its journal fallback at 10 s and `plugins.manage` at 15 s, + * and the deny-list branch makes that last one ordinary on any box whose owner + * has switched a plugin off — so unbounded it stacked up to ~33 s on top of the + * bounce's own 45-50 s ceiling. The MCP tool waits 60 s and would then have told + * the agent "the restart was started but did not report back in time" over a + * restart that had succeeded and was serving: a false failure, on the one call + * whose whole purpose is to say whether the plugin is live. + * + * Cut short, the read answers what a read that could not be made answers — + * `loaded: null`, `stale: null`, "could not be asked" — which every caller here + * already handles and none reads as "not loaded". The RPCs left running carry + * their own timeouts and end on their own. + * + * The WATCHER's reads are deliberately not bounded by this: nobody is waiting on + * a tick, and its reconcile read is proof rather than a courtesy. + */ +const POST_BOUNCE_READ_BUDGET_MS = 10_000; + +/** + * `readHermesPluginState()` within the budget above, or the caller's fallback. + * + * The `catch` is attached to the read itself rather than to the race, so a + * rejection that lands after the budget has expired is still handled. + */ +async function readStateWithin(fallback: T): Promise { + const read = readHermesPluginState().catch(() => fallback); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + read, + new Promise((resolve) => { + timer = setTimeout(() => resolve(fallback), POST_BOUNCE_READ_BUDGET_MS); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +export async function reloadHermesPlugins( + reason: string, + opts: ReloadOptions = {}, +): Promise { + const names = opts.names; + // READ BEFORE THE CLAIM IS ASKED FOR, not inside it. The claim compares this + // caller's declaration signature with the one the in-flight bounce is acting + // on, and that comparison has to be made without an await — a debt recorded + // after the running bounce's `finally` is a debt nothing will ever pay. The + // watcher passes the signature it has already read; the route and the MCP tool + // behind it pass none, and this is where theirs comes from. A read that failed + // leaves it undefined, which the claim treats as "cannot be compared". + const signature = + opts.signature ?? (await readHermesPluginDeclaration().then((d) => d.signature).catch(() => undefined)); + const outcome = await bounceHermesDashboardShared(reason, { signature, kind: "plugin_reload" }); + + if (outcome === "in_flight") { + // ANOTHER CALLER OWNS THIS RESTART, and its replacement reads the files as + // they are NOW — so a second bounce would add an outage and prove nothing. + // Reported as its own fact rather than as a failure: `restarted: false` + // alone would have the route answer 502 and the agent tell the owner the + // plugin could not be loaded, over a restart that is happening. + const running = await readStateWithin(null); + return { + restarted: false, + inFlight: true, + ready: false, + plugins: running?.declared ?? names ?? [], + loaded: running?.loaded ?? null, + stale: running?.stale ?? null, + detail: "a restart of the Hermes dashboard is already under way", + }; + } + + // Read AFTER the bounce, so `plugins` is what the box declares now and + // `loaded` is what the REPLACEMENT registered — the process whose answer is + // the only one that ever mattered here. + const state: HermesPluginState = await readStateWithin({ + declared: names ?? [], + loaded: null, + stale: null, + dashboardStartedAt: null, + changedAfterStart: null, + signature: signature ?? "", + }); + + if (outcome === "failed") { + // NOT a restart, and the caller must not be told otherwise: the plugin change is + // still not applied to the running agent and the owner's chat still works. + const detail = + "the Hermes dashboard could not be restarted, so the plugin change is not active yet"; + console.error(`[hermes/plugin-reload] ${logSafe(reason, 120)}: ${detail}`); + return { restarted: false, inFlight: false, ready: false, plugins: state.declared, loaded: state.loaded, stale: state.stale, detail }; + } + + const ready = outcome === "restarted"; + const detail = ready + ? "the Hermes dashboard restarted and is serving again" + : "the Hermes dashboard was stopped and systemd is bringing it back"; + console.log(`[hermes/plugin-reload] ${logSafe(reason, 120)}: ${detail}`); + + // The owner's open chat window has just gone. Best effort and never awaited + // for its own sake — a notice that fails to appear must not turn a restart + // that worked into a failure — but awaited here so a route's answer and the + // toast cannot race on a box that is about to be told the plugin is live. + await notifyOwner(noticeFor(names ?? state.declared)).catch(() => undefined); + + return { restarted: true, inFlight: false, ready, plugins: state.declared, loaded: state.loaded, stale: state.stale, detail }; +} + +export interface ReloadOptions { + /** The plugins the notice should name. Read from the declaration when absent. */ + readonly names?: readonly string[]; + /** + * The declaration signature this reload is FOR, when the caller has already + * read it. Absent, it is read here — before the CLAIM, which is the only + * moment at which it both means "what the replacement will have loaded" and + * is in time to tell this reload apart from the one already in flight. + */ + readonly signature?: string; +} + +/** What one `tick()` decided. Named so a test — and a journal — can read it. */ +export type HermesPluginTick = + | "baseline" + | "unchanged" + | "waiting" + | "restarted" + /** A bounce was taken, the replacement has come up, and nothing was stopped. */ + | "reconciled" + /** A bounce was taken and the replacement is not serving yet. */ + | "pending" + | "failed" + | "unreadable"; + +export interface HermesPluginWatcher { + /** Look once and act if it is time. Never throws. */ + tick(): Promise; +} + +export interface HermesPluginWatcherOptions { + /** The clock, so a test can move it instead of waiting. */ + readonly now?: () => number; + readonly debounceMs?: number; +} + +/** + * The watcher, as a thing that can be ticked. + * + * A TIMER IS NOT THE LOGIC, which is why the loop is somewhere else. Everything + * that can be got wrong here — a boot that must not count as a change, a burst + * that must collapse, a rewrite whose content is identical, a failed restart + * that must not be recorded as done — is a decision about two signatures and a + * clock, and all four are exercised by moving that clock by hand. + */ +export function createHermesPluginWatcher( + opts: HermesPluginWatcherOptions = {}, +): HermesPluginWatcher { + const now = opts.now ?? Date.now; + const debounceMs = opts.debounceMs ?? HERMES_PLUGIN_DEBOUNCE_MS; + + // The set the RUNNING dashboard is assumed to have loaded lives in + // `PluginReloadState`, shared with every other caller that bounces the + // dashboard — see that type for what kept it here cost. + const shared = hermesBounceState(); + /** The signature the current debounce window is waiting on, and since when. */ + let pending: { signature: string; since: number } | null = null; + /** + * How long the NEXT window must be, after a restart that failed. Zero while + * nothing has failed, so an ordinary change waits only the debounce. + */ + let retryDelayMs = 0; + + /** The window this tick is measuring against: the debounce, or the backoff. */ + const windowMs = () => Math.max(debounceMs, retryDelayMs); + + return { + async tick(): Promise { + let declaration; + try { + declaration = await readHermesPluginDeclaration(); + } catch (err) { + // A read that failed establishes nothing, so nothing is decided from it + // — the baseline stands and the window, if one is open, stays open. The + // poll loop must survive this: it runs for the life of the web server. + console.warn( + `[hermes/plugin-watch] could not read the plugin declaration: ${ + err instanceof Error ? logSafe(err.message, 160) : "unknown error" + }`, + ); + return "unreadable"; + } + + // A BOUNCE IS ALREADY IN FLIGHT — this one's or another caller's. Opening + // a window over it is how one action became two outages: the route bounces + // at once, the watcher sees the same change and SIGTERMs a dashboard that + // is in the middle of coming back. + if (shared.bouncing) return "waiting"; + + if (shared.baseline === null) { + // THE FIRST LOOK IS NOT AUTOMATICALLY A BASELINE, which is the whole of + // what this asks. Adopting whatever is on disk is right when the running + // dashboard has already read it, and wrong when it has not — and the + // dashboard OUTLIVES the web server, so an update or a `clawbox-setup` + // restart re-seeds from a file the chat backend has never seen. + // `scripts/register-mcp.sh` makes that concrete: it runs at EVERY + // web-server boot and can append the EMAIL-directive hook plugin to + // `plugins.enabled`, and whether the plugin stayed invisible for the + // rest of that dashboard's life was a race between its write and this + // tick five seconds later. + // + // TWO FACTS, AND NEITHER ALONE. `stale` is about the RUNNING registry — + // the box declares a plugin as enabled that the process does not have — + // and it is the honest answer, but it is not self-limiting: a box whose + // declaration and registry can never agree (a name that resolves to + // nothing, a plugin that fails to load) would be behind for ever, and a + // seed rule that read it alone would bounce the owner's chat at EVERY + // web-server boot, for good. `changedAfterStart` is an mtime, so every + // Settings save makes it true and it may not stand alone either — but it + // IS self-limiting: once the dashboard has restarted, its start is newer + // than those files. Together: bounce only when the process is + // demonstrably missing something AND the files were touched after it + // started, which is exactly the `register-mcp.sh` case and is true at + // most once per dashboard. Unknown on either side seeds, because a + // watcher that bounced the chat over what it could not establish would + // be worse than one that waits for the next real change. + const state = await readHermesPluginState().catch(() => null); + if (state?.stale !== true || state.changedAfterStart !== true) { + shared.baseline = declaration.signature; + return "baseline"; + } + console.log("[hermes/plugin-watch] the running dashboard is behind ~/.hermes; a restart is owed"); + // A baseline IS recorded — the sentinel, which no content hash can be + // (`readHermesPluginDeclaration` answers 32 hex characters). Without one + // this branch would be re-entered on every tick and the window it opens + // would never close. What it says is exactly true: whatever the running + // dashboard loaded, it is not what is on disk. + shared.baseline = BASELINE_BEHIND; + pending = { signature: declaration.signature, since: now() }; + return "waiting"; + } + + if (declaration.signature === shared.baseline) { + // Includes a change that was UNDONE inside the window — installed and + // removed again. Nothing is different from what the dashboard is running, + // so there is nothing to restart for, and a backoff earned by the change + // that has just gone away is not owed by the next one. + pending = null; + retryDelayMs = 0; + shared.awaiting = null; + shared.awaitingPid = null; + return "unchanged"; + } + + if (!pending || pending.signature !== declaration.signature) { + // Either the first sight of this change or another write on top of it. + // Restarting the window rather than the clock is what makes a burst ONE + // restart: `hermes plugins install` writes the directory, the ledger and + // the config within a second or two of each other. + pending = { signature: declaration.signature, since: now() }; + return "waiting"; + } + + if (now() - pending.since < windowMs()) return "waiting"; + + const acted = pending.signature; + + // A BOUNCE ALREADY TAKEN FOR THIS SIGNATURE IS RECONCILED, NOT REPEATED. + // `pending` from `bounceHermesDashboard` means the stop landed and + // `Restart=always` owns the unit — the replacement, whenever it arrives, + // reads the files as they are now. What is unproven is that it ARRIVED, so + // the answer is to look, not to stop a process that is coming up. Only + // once recovery is established as FAILED — systemd says the unit is down, + // nothing is coming on its own — does this fall through to a new bounce. + if (shared.awaiting === acted) { + // IS THE PROCESS ANSWERING THE PORT A REPLACEMENT, or the one the bounce + // tried to stop? `pending` covers "systemd could not be asked, twice", + // where nothing was stopped at all and the OLD process answers the probe + // perfectly well — reconciled on the probe alone, the watcher recorded + // the change as loaded and never tried again. `false` here is that case + // and is a reason to bounce AGAIN, not to wait for ever. + const current = await hermesDashboardMainPid().catch(() => ({ read: false, pid: null })); + const replaced = + shared.awaitingPid !== null && current.read && current.pid !== null + ? current.pid !== shared.awaitingPid + : null; + // A PORT THAT ANSWERS IS NOT A REPLACEMENT, and `null` is not a pid that + // moved. `awaitingPid` is null whenever the pre-bounce read failed or + // systemd reported the perfectly valid `MainPID=0`, and `current` is + // null whenever this read fails — so a bounce that stopped nothing could + // still be reconciled here on the probe alone, which is the same false + // success the pid was added to close, one step further out. + // + // The other proof is the one the baseline is actually ABOUT: ask the + // process that is serving whether it has the set the box declares. + // `stale === false` is that answer and is stronger than a pid; `true` is + // the running registry saying it has NOT read the change, which makes + // this the same case as a pid that never moved. A `null` from either + // establishes nothing and falls through to the wait below, where systemd + // is asked whether anything is still coming. + let loaded: boolean | null = replaced; + if (replaced !== false && (await hermesDashboardServing())) { + if (replaced === null) { + const running = await readHermesPluginState().catch(() => null); + loaded = running?.stale === false ? true : running?.stale === true ? false : null; + } + if (loaded === true) { + shared.baseline = acted; + shared.awaiting = null; + shared.awaitingPid = null; + pending = null; + retryDelayMs = 0; + return "reconciled"; + } + } + if (loaded === false) { + // Nothing was replaced — either the same process is still the unit's + // main one, or it is serving a registry that has not read the change. + // Waiting longer changes nothing in either case, so the bounce is + // retaken. + console.error("[hermes/plugin-watch] the dashboard serving now has not loaded the change; bouncing again"); + shared.awaiting = null; + shared.awaitingPid = null; + } else { + const unitState = await hermesDashboardUnitState().catch(() => "unknown" as const); + if (unitState !== "down") { + // Still on its way, in the RestartSec gap, or a state that cannot be + // read. Wait longer each time rather than asking every poll. + pending = { signature: acted, since: now() }; + retryDelayMs = widened(retryDelayMs, debounceMs); + return "waiting"; + } + console.error("[hermes/plugin-watch] the dashboard did not come back after its stop; bouncing again"); + shared.awaiting = null; + shared.awaitingPid = null; + } + } + + const result = await reloadHermesPlugins("the box's Hermes plugin set changed", { + names: declaration.names, + signature: acted, + }); + if (!result.restarted) { + // THE BASELINE DOES NOT MOVE. A bounce that failed left the dashboard on + // the OLD plugin set, so recording this signature as done would mean + // nothing ever tried again and the plugin stayed invisible until the next + // reboot — with the box reporting no work outstanding. + // + // The window re-opens rather than the retry firing on the next poll, and + // it widens each time: a dashboard systemd has given up on would + // otherwise take a SIGTERM every few seconds for ever. `pending` is kept + // (not re-created) so a further write during the backoff is still seen as + // a new change and resets it. + pending = { signature: acted, since: now() }; + retryDelayMs = widened(retryDelayMs, debounceMs); + return "failed"; + } + if (!result.ready) { + // TAKEN BUT NOT SERVING. `reloadHermesPlugins` has recorded this + // signature as awaited rather than loaded; the next window reconciles it + // above. The baseline deliberately does not move — a replacement that + // never came up has read nothing. + pending = { signature: acted, since: now() }; + retryDelayMs = widened(retryDelayMs, debounceMs); + return "pending"; + } + pending = null; + retryDelayMs = 0; + // NOT over a baseline the claim marked BEHIND on its way out. A caller + // arrived while THIS bounce was in flight — a ClawKeep restore is the + // worked example — so one more restart is owed, and writing the signature + // this tick acted on would erase the only record of that debt: the + // restore's own plugin set can be byte-identical, so nothing else here + // would ever see a change again. + if (shared.baseline !== BASELINE_BEHIND) shared.baseline = acted; + return "restarted"; + }, + }; +} + +/** Double the wait, from the debounce, and never past the ceiling. */ +function widened(current: number, debounceMs: number): number { + return Math.min(current ? current * 2 : debounceMs * 2, HERMES_PLUGIN_RETRY_MAX_MS); +} + +/** The live loop's handle, held so a dev hot-reload can replace it. */ +let watcherTimer: ReturnType | null = null; + +/** + * Start watching, on the editions that have a dashboard to restart. + * + * `hasHermesHarness()` — edition `hermes` or `dual` — is the predicate, and it is + * the same one `install.sh` uses to decide whether to enable the dashboard unit + * at all. Asking which harness is ACTIVE would be the wrong question on the dual + * SKU, where the dashboard runs whichever agent is serving the owner. + */ +export function startHermesPluginWatcher(opts: HermesPluginWatcherOptions = {}): boolean { + if (!hasHermesHarness()) return false; + if (watcherTimer) clearInterval(watcherTimer); + const watcher = createHermesPluginWatcher(opts); + let running = false; + watcherTimer = setInterval(() => { + // One tick at a time. A restart holds a tick for up to the bounce's whole + // budget (45 s), which is nine poll intervals, and a second tick inside it + // would open a window over a box that is mid-restart. + if (running) return; + running = true; + void watcher + .tick() + .catch(() => undefined) + .finally(() => { + running = false; + }); + }, opts.debounceMs ? Math.min(HERMES_PLUGIN_POLL_MS, opts.debounceMs) : HERMES_PLUGIN_POLL_MS); + // Never hold the process open for a poll: this is a background reconciliation, + // not work anything is waiting on. + watcherTimer.unref?.(); + console.log("[hermes/plugin-watch] watching ~/.hermes for a plugin change"); + return true; +} diff --git a/src/lib/hermes-plugin-set.ts b/src/lib/hermes-plugin-set.ts new file mode 100644 index 000000000..14e4877da --- /dev/null +++ b/src/lib/hermes-plugin-set.ts @@ -0,0 +1,778 @@ +import { execFile } from "child_process"; +import crypto from "crypto"; +import fs from "fs/promises"; +import os from "os"; +import path from "path"; +import { promisify } from "util"; +import { HERMES_DASHBOARD_UNIT } from "@/lib/hermes-dashboard-auth"; +import { hermesHome } from "@/lib/hermes-env"; +import { processStore } from "@/lib/process-store"; +import { parseYamlFlowSequence, parseYamlScalar, splitYamlComment } from "@/lib/yaml-block-edit"; + +const execFileAsync = promisify(execFile); + +/** + * What this box DECLARES as its Hermes plugin set, and what the RUNNING + * dashboard actually LOADED. They are different questions, and the whole defect + * this module exists for is that every surface on the box answered the first one + * while the owner was asking the second. + * + * THE MEASUREMENT (owner's box, 2026-09-18). The assistant installed and enabled + * the `superpowers` plugin at 12:59 — `~/.hermes/plugins/superpowers/`, a row in + * `.install-metadata.json`, its name under `plugins.enabled` in `config.yaml` — + * and proved it worked by running `hermes chat -q`, which is a FRESH PROCESS. + * The chat the owner was looking at is served by + * `clawbox-hermes-dashboard.service`, up since 10:52. Hermes scans for plugins + * once per process (`discover_plugins(force=True)` at start; + * `_ensure_plugins_discovered()` returns early ever after, and nothing reachable + * over the dashboard socket passes its `force` flag), so that process had never + * heard of the plugin and never would. `hermes plugins list` said "enabled". The + * chat had no such tool. + * + * So a reader that answers from `~/.hermes` is answering a question nobody + * asked. This module answers both, and keeps them apart in its own vocabulary: + * DECLARED is what the files say, LOADED is what the running process proved it + * registered, and `null` for the latter means THIS BOX COULD NOT BE ASKED — + * never "it loaded nothing". + */ + +/** Where `hermes plugins install` records what it put on the box. */ +function installLedgerPath(): string { + return path.join(hermesHome(), "plugins", ".install-metadata.json"); +} + +function hermesConfigPath(): string { + return path.join(hermesHome(), "config.yaml"); +} + +/** + * The top-level `plugins:` block of config.yaml, verbatim, or "". + * + * WHY A BLOCK AND NOT THE FILE. `config.yaml` is rewritten by every Settings + * save on this box — a provider key, a model change, a voice toggle — and the + * dashboard's own `ExecStartPre` re-provisions `dashboard.basic_auth` into it on + * every start. A watcher keyed on the whole file would bounce the box's chat + * backend on each of those, and the re-provisioning one would make it a loop: + * the dashboard restarts, rewrites the file, and is restarted for having + * restarted. + * + * WHY COLUMN ZERO. `dashboard.hidden_plugins` and `plugins.entries.` both + * put the word further in, and a substring match on "plugins" would take the + * former for the latter — a UI preference then reads as a plugin change. Only a + * key at indent 0 opens the block, and the block ends at the next line at indent + * 0 that is neither blank nor a comment. + * + * Both halves that decide loading are inside it: `plugins.enabled` (Hermes' + * opt-in allow-list) and `plugins.disabled` (the deny-list `_plugin_status` + * gives precedence to). A plugin the owner disables has changed the set as + * surely as one they install. + */ +export function hermesPluginsBlock(yamlText: string): string { + const lines = yamlText.split(/\r?\n/); + const kept: string[] = []; + // Blank lines and COMMENTS at column zero belong to whatever block encloses + // them and end nothing — YAML has no rule that a `#` closes a mapping. They + // are held back rather than kept outright, because the same lines sitting + // AFTER the block belong to whatever comes next: held, they are committed + // only when a genuine in-block line follows, and dropped at the break. A + // comment at column zero used to end the extraction, which silently cut + // `disabled:` — the deny-list `_plugin_status` gives precedence to — out of + // the signature on any box whose config.yaml carries one. + let pending: string[] = []; + let inside = false; + for (const line of lines) { + if (/^plugins\s*:/.test(line)) { + inside = true; + kept.push(line); + continue; + } + if (!inside) continue; + if (!line.trim() || /^\s*#/.test(line)) { + pending.push(line); + continue; + } + // A line that starts with whitespace is inside the block. + if (/^\s/.test(line)) { + kept.push(...pending, line); + pending = []; + continue; + } + // A key back at column zero: the block is over, and anything held since the + // last real line belongs to what follows it. + break; + } + // Trailing blank lines say nothing about the plugin set; keeping them would + // make a stray newline elsewhere in the file look like a change. + return kept.join("\n").replace(/\s+$/, ""); +} + +/** What `~/.hermes` says this box's plugin set is. */ +export interface HermesPluginDeclaration { + /** Every plugin name the box declares, installed or merely enabled, sorted. */ + readonly names: readonly string[]; + /** The names under `plugins.enabled`, which is what Hermes will load… */ + readonly enabled: readonly string[]; + /** + * …unless `plugins.disabled` names them, which WINS: `_plugin_status` is + * `"disabled" if names & disabled else "enabled" if names & enabled`. + * + * The two lists overlap in ordinary use, which is why this is carried rather + * than folded into `enabled` at the parse. `hermes plugins disable superpowers` + * discards the resolved KEY and its leaf from the allow-list and adds the key + * to the deny-list (`cmd_disable` → `_discard_key_and_leaf`), so the bare name + * a person originally wrote is left sitting in `enabled` — measured on the + * owner's box. + */ + readonly disabled: readonly string[]; + /** + * A hash of the PARSED plugin set — never an mtime, and never the raw text. + * + * The dashboard rewrites `config.yaml` on every start, so two reads a restart + * apart have different mtimes and identical meaning. Only a hash of what the + * file SAYS can tell "the owner installed something" from "the thing we + * restarted has restarted", and getting that wrong is an endless restart loop + * over the box's chat. + * + * AND A HASH OF THE TEXT IS NOT THAT HASH, which cost a chat window before it + * was noticed: `scripts/register-mcp.sh` re-serialises the whole config + * through `yaml.safe_dump` whenever anything at all changed, normalising the + * plugins block's indentation, quoting and comments — and it is spawned + * fire-and-forget at every web-server boot (`production-server.js`) and again + * from the ClawBox-MCP toggle in Settings. Its write lands well after the + * watcher's first tick, so the owner toggled "expose ClawBox tools to the + * assistant", and eight seconds later their chat closed with "The assistant + * restarted to load the plugin …" over a set that had not moved by one name. + * So the sorted `enabled` and `disabled` names and the install ledger's + * name→revision rows are what is hashed: a re-serialisation of the same set + * is the same signature, and a real change to any of the three is not. + */ + readonly signature: string; + /** Newest mtime of the inputs, for reporting only. Null when unreadable. */ + readonly changedAt: number | null; +} + +/** A file's text and mtime, or nulls. NEVER THROWS: see `readHermesPluginDeclaration`. */ +async function readIfPresent(file: string): Promise<{ text: string; mtimeMs: number | null }> { + try { + const [text, stat] = await Promise.all([fs.readFile(file, "utf-8"), fs.stat(file)]); + return { text, mtimeMs: stat.mtimeMs }; + } catch { + return { text: "", mtimeMs: null }; + } +} + +/** + * The install ledger as `name → recorded revision`, or {} for anything + * unreadable. + * + * THE REVISION IS PART OF THE SET, not decoration: `hermes plugins update` + * pulls the plugin's git checkout and writes the new HEAD here + * (`plugins_cmd.py:_pull_plugin_update`). The name has not moved and the code + * has, and the process serving chat is holding the old one — which is the same + * "restart to pick it up" as an install. + * + * A HALF-WRITTEN FILE IS NOT AN EMPTY BOX. `hermes plugins install` writes + * this, and a watcher polling every few seconds will catch it mid-write — + * reproduced by writing `{"superpowers":` in the suite. Answering {} for that + * one read is right (nothing can be established from it) and, because the + * signature is built from what was PARSED, it is also the answer the settled + * file gave a moment earlier: no window opens over a partial write, and the + * next poll reads the finished file. What must not happen is a throw, which + * would take the poll loop down for the life of the web server. + */ +function ledgerEntries(text: string): Record { + if (!text.trim()) return {}; + try { + const parsed: unknown = JSON.parse(text); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {}; + const out: Record = {}; + for (const [name, row] of Object.entries(parsed as Record)) { + const revision = row && typeof row === "object" ? (row as { revision?: unknown }).revision : undefined; + out[name] = typeof revision === "string" ? revision : ""; + } + return out; + } catch { + return {}; + } +} + +/** + * The names under `enabled:` — or `disabled:` — inside an already-extracted + * `plugins:` block. + * + * BOTH BLOCK LAYOUTS, because this box has two writers and only one of them was + * ever read. Hermes' own dumper forces `indentless=False`: + * + * plugins: + * enabled: + * - superpowers + * + * `scripts/register-mcp.sh` re-serialises the whole file with plain + * `yaml.safe_dump`, and PyYAML's default in a mapping context is an INDENTLESS + * block sequence: + * + * plugins: + * enabled: + * - superpowers + * + * The old rule accepted an item only when it was indented FURTHER than the key, + * so on every box that script had ever had something to change — which is every + * box that got the EMAIL-directive hook — `plugins.enabled` read as EMPTY, and + * `stale`, being `enabled.some(...)`, was `false` for ever. The MCP tool then + * told the owner "the agent now serving chat has read the current plugin set" + * about a plugin that had never loaded. An item belongs to the list when it sits + * at the key's indent or deeper; anything shallower is a level up. + * + * COMMENTS AND QUOTES ARE THE READER'S JOB, not a regex's: + * `- superpowers # installed 2026-09-18` is one name and a note, while + * `- "weird # name"` is one name with a hash in it. Read the wrong way round, + * the first invents a name no registry can ever match — a permanent `stale` and + * a chat restart that cannot fix it. {@link splitYamlComment} is the one place + * that rule lives. + * + * NOT READ, and said out loud: a `plugins:` written as a FLOW MAPPING at column + * zero (`plugins: {enabled: [a]}`). Neither writer on the box produces it — + * PyYAML is called with `default_flow_style=False`, Hermes' dumper likewise, and + * ClawBox's own editor refuses flow shapes outright — so it can only arrive by + * hand, and the previous reader did not understand it either. + */ +/** + * The indent a DIRECT child of `plugins:` sits at, or -1 for a block with none. + * + * The shallowest indented line in the block, which is what `plugins:`' own + * children share — an indentless block sequence puts its items at the key's + * indent too, so items never make this smaller than the key above them. Blanks + * and comments say nothing about structure and are skipped, and the `plugins:` + * line itself sits at column zero and is not a child. + */ +function childIndentOf(block: string): number { + let indent = -1; + for (const line of block.split("\n")) { + if (!line.trim() || /^\s*#/.test(line)) continue; + const leading = /^(\s+)\S/.exec(line); + if (!leading) continue; + if (indent < 0 || leading[1].length < indent) indent = leading[1].length; + } + return indent; +} + +function listedNames(block: string, wanted: "enabled" | "disabled"): string[] { + const names: string[] = []; + let inList = false; + let listIndent = -1; + // WHOSE `enabled:` IS IT? The key used to match at any depth inside the + // block, so a plugin's own settings under `plugins.entries.` — the one + // thing the block extraction exists to keep out of the set — could put its + // members into the box's declaration when its schema happened to have a + // list-valued key called `enabled` or `disabled`. A ghost name is a permanent + // `stale`/`null` and a signature that moves when a preference does. Only a + // DIRECT child of `plugins:` is the box's plugin list. + const childIndent = childIndentOf(block); + for (const line of block.split("\n")) { + const key = /^(\s+)(enabled|disabled)\s*:(.*)$/.exec(line); + if (key && key[1].length !== childIndent) { + // A nested key of the same name. It is not the list, and — unlike a line + // this loop does not recognise at all — it is not part of one either, so + // whatever list was open ends here. + inList = false; + continue; + } + if (key) { + inList = key[2] === wanted; + listIndent = key[1].length; + const inline = splitYamlComment(key[3]).value.trim(); + // Nothing after the colon: a block sequence follows on the next lines. + if (!inline) continue; + // An inline list — `enabled: [a, b]`, which is what `hermes config set` + // writes when it is handed JSON — with or without a note after it. + if (inList) { + for (const member of parseYamlFlowSequence(inline) ?? []) { + const name = parseYamlScalar(member)?.trim(); + if (name) names.push(name); + } + } + // Whatever it was, the value was on this line: no block sequence follows. + inList = false; + continue; + } + if (!inList) continue; + // A comment or a blank line inside the list is not the end of it, for the + // same reason it is not the end of the block. + if (!line.trim() || /^\s*#/.test(line)) continue; + const item = /^(\s*)-(?:\s+(.*))?$/.exec(line); + if (item && item[1].length >= listIndent) { + const raw = splitYamlComment(item[2] ?? "").value.trim(); + const name = raw ? parseYamlScalar(raw)?.trim() : ""; + if (name) names.push(name); + continue; + } + inList = false; + } + return names; +} + +/** + * Read what `~/.hermes` declares, cheaply enough to poll. + * + * NEVER THROWS, and answers an empty set for a box with no plugins at all — + * which is most boxes, and is not an error. Two small local file reads; nothing + * here spawns Hermes, because the caller is a loop that runs for the life of the + * web server and a Python CLI cold start is seconds. + */ +export async function readHermesPluginDeclaration(): Promise { + const [ledger, config] = await Promise.all([ + readIfPresent(installLedgerPath()), + readIfPresent(hermesConfigPath()), + ]); + const block = hermesPluginsBlock(config.text); + const enabled = [...new Set(listedNames(block, "enabled"))].sort(); + const disabled = [...new Set(listedNames(block, "disabled"))].sort(); + const installed = ledgerEntries(ledger.text); + const names = [...new Set([...Object.keys(installed), ...enabled])].sort(); + // WHAT THE SET IS, not how it happens to be spelled today — see `signature` on + // the type above for the chat window that bought this back. `disabled` is + // hashed beside `enabled` because `_plugin_status` gives the deny-list + // precedence, so a plugin the owner disables has changed the set as surely as + // one they install; the ledger's revisions are hashed because an update + // changes the code under an unchanged name. A key inside the block that is + // neither — a `plugins.entries.` preference — deliberately is not: it does + // not decide what loads, and hashing it is what made a whole-file + // re-serialisation read as a plugin change. + const signature = crypto + .createHash("sha256") + .update( + JSON.stringify({ + enabled, + disabled, + installed: Object.keys(installed) + .sort() + .map((name) => [name, installed[name]]), + }), + ) + .digest("hex") + .slice(0, 32); + const mtimes = [ledger.mtimeMs, config.mtimeMs].filter((m): m is number => m !== null); + return { + names, + enabled, + disabled, + signature, + changedAt: mtimes.length ? Math.max(...mtimes) : null, + }; +} + +/** + * What systemd knows about the dashboard's CURRENT run. + * + * The invocation id is the one key that names exactly this activation — + * `journalctl -u` sees every run the unit has ever had, and a plugin registered + * by yesterday's process must never count as loaded by today's. Same idiom, and + * the same reason, as `readSwapInvocationId` in `src/lib/harness-swap.ts`. + */ +async function dashboardRun(): Promise<{ invocationId: string | null; startedAtMs: number | null }> { + const { stdout } = await execFileAsync( + "/usr/bin/systemctl", + ["show", HERMES_DASHBOARD_UNIT, "--property=InvocationID,ExecMainStartTimestampMonotonic"], + { timeout: 5_000 }, + ).catch(() => ({ stdout: "" })); + const props: Record = {}; + for (const line of stdout.split(/\r?\n/)) { + const eq = line.indexOf("="); + if (eq > 0) props[line.slice(0, eq).trim()] = line.slice(eq + 1).trim(); + } + const id = props.InvocationID ?? ""; + // Monotonic microseconds since boot, turned into wall clock through the same + // boot this process shares. These boxes have no RTC and step the clock when + // NTP first lands, so a stored wall-clock start would be wrong across exactly + // that window; the monotonic value is not, and the conversion is only ever + // used for reporting. + const monotonicUsec = Number(props.ExecMainStartTimestampMonotonic); + const startedAtMs = + Number.isFinite(monotonicUsec) && monotonicUsec > 0 + ? Date.now() - (os.uptime() * 1_000 - monotonicUsec / 1_000) + : null; + return { + invocationId: /^[0-9a-f]{8,}$/i.test(id) ? id : null, + startedAtMs, + }; +} + +/** + * Hermes' own "I have this plugin" line, as the plugin manager writes it. + * + * `hermes_cli/plugins.py` logs `Plugin '' registered : ` at + * INFO for every registry a plugin lands in — tools, a context engine, a memory + * provider, a dashboard-auth provider, an approval transport — with the id + * unquoted in two of them. Both spellings are matched. + */ +const REGISTERED_RE = /\bPlugin\s+'?([A-Za-z0-9._-]+)'?\s+registered\b/g; + +/** How much of the run's journal to read. Bounded: this is a request path. */ +const JOURNAL_LINES = 500; + +/** + * The plugins the RUNNING dashboard proved it registered, or null. + * + * NULL IS THE IMPORTANT ANSWER and it must never be folded into `[]`. A box with + * no journalctl, a unit systemd cannot name, a run whose id could not be read — + * none of those says anything about what loaded, and answering "nothing loaded" + * over them is the false-failure shape on the one reader whose job is to be + * believed. It would have the route tell the owner their plugin is missing from + * a dashboard that is serving it. + * + * AN ABSENT NAME IS NOT A NO EITHER, and the caller is told so in the type it + * gets back: a plugin whose only registrations are tools logs them at DEBUG + * (`plugins.py:500`), so it can be loaded and leave no INFO line. What a name + * present here proves is the positive — this process has it — which is exactly + * what a restart has to demonstrate. + * + * WHICH IS WHY AN EMPTY RESULT IS ALSO NULL. Measured on the owner's box + * (2026-09-18): the running dashboard's whole journal for its current invocation + * is 187 lines of `sessions.changed` events and the readiness banner, with not + * one `Plugin … registered` among them — Hermes writes those on its Python + * logger and this process does not route them out. Hermes always registers the + * bundled `basic` dashboard-auth plugin on a gated bind, so ZERO registration + * lines cannot mean "nothing loaded"; it can only mean this box does not publish + * them. Returning `[]` there would have the route report every plugin on a + * working device as missing, which is the false-failure this reader exists to + * prevent — and it would do it on the exact box the feature was built for. + * + * `stale` (see `readHermesPluginState`) is the answer that still holds on such a + * box: it is a fact about WHEN, not about what was logged. + */ +export async function readLoadedHermesPlugins(): Promise { + // HERMES' OWN ANSWER FIRST — leverage the harness before re-deriving it. + // `plugins.list` is a method on the dashboard socket ClawBox already dials + // (`tui_gateway/methods_tools.py`, verified against the pinned checkout), and + // it is built from `get_plugin_manager()._plugins` — the running process's own + // registry rather than a log of what it once printed. That is exactly the + // question this function exists to answer, and unlike the journal scrape it + // works on the owner's box. + const fromHarness = await loadedFromDashboard(); + // `[]` is an ANSWER here (rows, none enabled) and truthiness would drop it. + if (fromHarness !== null) return fromHarness; + const { invocationId } = await dashboardRun(); + if (!invocationId) return null; + const { stdout } = await execFileAsync( + "/usr/bin/journalctl", + [ + `_SYSTEMD_INVOCATION_ID=${invocationId}`, + "-o", + "cat", + "--no-pager", + "-n", + String(JOURNAL_LINES), + ], + { timeout: 10_000, maxBuffer: 4 * 1024 * 1024 }, + ).catch(() => ({ stdout: null as string | null })); + if (stdout === null) return null; + const names = new Set(); + for (const match of stdout.matchAll(REGISTERED_RE)) names.add(match[1]); + // Nothing at all means this dashboard does not publish those lines — not that + // it loaded no plugins. See the doc block above; this is the one branch that + // was measured wrong on hardware before it shipped. + return names.size ? [...names].sort() : null; +} + +/** How long `plugins.list` may take. It is a dict comprehension over a registry + * the process already holds — not a reload — so seconds, not tens of them. */ +const PLUGINS_LIST_TIMEOUT_MS = 8_000; + +/** + * What the RUNNING dashboard's own plugin registry says, or null. + * + * NULL FOR AN ANSWER WITH NO ROWS AT ALL, by the same rule the journal scrape + * follows: the dashboard runs `discover_plugins(force=True)` at start and Hermes + * ships bundled plugins, so a registry with nothing in it is far more likely to + * be a shape this build does not understand than a box with none — and "nothing + * is loaded" is the one answer that would have the route tell an owner their + * working plugin is missing. + * + * AN ANSWER WITH ROWS, NONE OF THEM ENABLED, IS NOT THAT. It is the registry + * saying exactly what this feature is about — the process HAS the manifests and + * has none of them on — so it comes back as `[]`, established, and the staleness + * that follows from it is a fact rather than a guess. The difference is "the + * process did not answer" against "the process answered no". + * + * The ENABLED ones, because that is what "loaded" means to the person asking: + * `_plugins` holds every manifest discovery found, bundled ones included, each + * with the verdict the config gave it. + */ +async function loadedFromDashboard(): Promise { + const { dashboardRpc } = await import("@/lib/hermes-dashboard-rpc"); + const result = await dashboardRpc("plugins.list", {}, { timeoutMs: PLUGINS_LIST_TIMEOUT_MS }).catch(() => null); + const rows = (result as { plugins?: unknown } | null)?.plugins; + if (!Array.isArray(rows) || rows.length === 0) return null; + const names = new Set(); + for (const raw of rows) { + const row = raw as { name?: unknown; enabled?: unknown }; + // `enabled` absent is treated as on, the same default the RPC itself uses + // (`getattr(i, "enabled", True)`): an older Hermes that answers with names + // alone still tells us what the process has. + if (typeof row?.name === "string" && row.name.trim() && row.enabled !== false) { + names.add(row.name.trim()); + } + } + return [...names].sort(); +} + +/** How long `plugins.manage` may take. It re-discovers from disk and consults + * the live plugin catalogue (5 s upstream timeout, 6 h cache), so it is bounded + * wider than `plugins.list` and asked only when something needs resolving. */ +const PLUGINS_MANAGE_TIMEOUT_MS = 15_000; + +/** + * Registry key → manifest name, from Hermes' OWN plugin discovery, or null. + * + * THE HALF `plugins.list` CANNOT ANSWER. That RPC is built from + * `get_plugin_manager()._plugins`, whose keys are `manifest_key(manifest)` = + * `manifest.key or manifest.name` (`hermes_cli/plugins_manifest.py`), and a + * nested plugin's key is `/` — which says nothing about the + * name a person enables it under. `plugins.manage {action:"list"}` is the one + * method that carries BOTH (`tui_gateway/methods_tools.py:_plugin_rows`), so it + * is what the match below is built from rather than a rule re-derived here. + * + * Null for a box that could not be asked — an older Hermes with no such method, + * a dashboard that did not answer — and the caller turns that into "could not + * establish", never into "no". Asked LAZILY, because a box whose declared names + * are all registry keys needs none of it. + */ +/** + * How long a name↔key map is reused before it is asked for again. + * + * Short, and a memo rather than a cache: the map changes only when a plugin is + * INSTALLED or REMOVED, and `plugins.manage {action:"list"}` is the expensive + * one of the two RPCs here — `_plugin_rows` re-discovers every manifest from + * disk and consults the catalogue, which is why it is budgeted at 15 s. Nothing + * polls the GET today, so this is not hot; what the memo buys is that it cannot + * BECOME hot, which is the difference the deny-list branch made when it turned + * "asked only when a name will not resolve" into "asked on every read for any + * box whose owner has switched a plugin off". + * + * Deliberately far shorter than the thing it describes is stable for. The first + * call in a process always asks — which is the watcher's first look, the one + * read whose answer arms an unattended restart — and a plugin installed while a + * memo stands resolves within the window. Only a SUCCESSFUL map is kept: a box + * that could not be asked must be asked again, never remembered as "no". + */ +const PLUGIN_NAME_MEMO_MS = 30_000; + +/** + * The memo lives in the process store, not in a module-level `let`. + * + * This file is reached both by `src/instrumentation.ts`'s `require` and by the + * routes' `import`, which Next compiles as two different modules in the one web + * server — see `process-store.ts`. Two memos would not be wrong here, only + * wasteful, but the rule is cheaper to keep than to reason about each time. + */ +function pluginNameMemo(): { at: number; byKey: Map } | null { + return processStore<{ value: { at: number; byKey: Map } | null }>( + "clawbox.hermes-plugin-name-memo", + () => ({ value: null }), + ).value; +} + +function rememberPluginNames(byKey: Map): void { + processStore<{ value: { at: number; byKey: Map } | null }>( + "clawbox.hermes-plugin-name-memo", + () => ({ value: null }), + ).value = { at: Date.now(), byKey }; +} + +/** Test seam: forget the memo, so each case asks for itself. */ +export function _resetHermesPluginNameMemoForTests(): void { + processStore<{ value: unknown }>("clawbox.hermes-plugin-name-memo", () => ({ value: null })).value = null; +} + +async function pluginNameByKey(): Promise | null> { + const memo = pluginNameMemo(); + if (memo && Date.now() - memo.at < PLUGIN_NAME_MEMO_MS) return memo.byKey; + const { dashboardRpc } = await import("@/lib/hermes-dashboard-rpc"); + const result = await dashboardRpc( + "plugins.manage", + { action: "list" }, + { timeoutMs: PLUGINS_MANAGE_TIMEOUT_MS }, + ).catch(() => null); + const rows = (result as { plugins?: unknown } | null)?.plugins; + if (!Array.isArray(rows) || rows.length === 0) return null; + const byKey = new Map(); + for (const raw of rows) { + const row = raw as { name?: unknown; key?: unknown }; + if (typeof row?.key === "string" && typeof row?.name === "string" && row.key.trim() && row.name.trim()) { + byKey.set(row.key.trim(), row.name.trim()); + } + } + if (!byKey.size) return null; + rememberPluginNames(byKey); + return byKey; +} + +/** + * Does the running registry carry the plugin the box declares by this name? + * True, false, or NULL for "this box could not be asked". + * + * HERMES' OWN RULE, mirrored rather than approximated: `_plugin_status` builds + * `names = {manifest name, registry key}` and intersects it with the enabled set + * (`hermes_cli/plugins_cmd.py`). So a declared name matches a running plugin + * when it IS that plugin's registry key, or when it is the manifest name behind + * it — and never in any other way. + * + * WHAT WAS HERE BEFORE was a path-segment match (`image_gen/clawai`.split("/") + * contains `clawai`), which happened to cover the two plugins measured on the + * owner's box and is both looser and stricter than the real rule. Stricter is + * the expensive direction: a plugin installed into a directory whose name + * differs from its manifest `name`, enabled under that name, has a key carrying + * neither — `stale` was then true for ever, the MCP tool warned "the plugin is + * NOT loaded yet" about a box that was serving it, and the next config write + * bounced the owner's chat for a restart that could not change the answer. + * + * NULL RATHER THAN FALSE when the name↔key map could not be read, because + * "this build cannot resolve a nested key" is not evidence that a plugin is + * missing, and `stale: true` is what arms an unattended restart of the chat. + */ +/** + * Do these two declared strings name the SAME plugin? + * + * Hermes' identity for a plugin is the pair `{manifest name, registry key}`, and + * a person may write either into `plugins.enabled` or `plugins.disabled`. With + * no name↔key map only the literal answer can be given, which is the + * conservative direction here: an unmatched pair leaves the enabled entry in + * play and its verdict comes back as "could not establish". + */ +function samePlugin(a: string, b: string, nameByKey: ReadonlyMap | null): boolean { + if (a === b) return true; + if (!nameByKey) return false; + return nameByKey.get(a) === b || nameByKey.get(b) === a; +} + +function registryHas( + declared: string, + keys: ReadonlySet, + nameByKey: ReadonlyMap | null, +): boolean | null { + if (keys.has(declared)) return true; + if (!nameByKey) return null; + for (const key of keys) { + if (nameByKey.get(key) === declared) return true; + } + return false; +} + +/** Declared, loaded, and whether the running process is behind the files. */ +export interface HermesPluginState { + readonly declared: readonly string[]; + /** Null when this box could not be asked — never "nothing loaded". */ + readonly loaded: readonly string[] | null; + /** + * The process serving chat is BEHIND THE FILES: something the box declares as + * enabled is not enabled in the running registry. Null when it could not be + * established. + * + * DERIVED FROM `loaded`, NEVER FROM AN MTIME. It used to compare + * `max(mtime(ledger), mtime(config.yaml))` against the dashboard's start — + * and `config.yaml` is rewritten by every Settings save on this box and by + * the dashboard's own `ExecStartPre`, which is the exact false positive the + * block extraction exists to prevent. An owner changing the assistant's model + * at 14:00 made this `true`, which the MCP tool words as "the agent is still + * behind the files — the plugin is NOT loaded yet". The registry answers the + * real question, so it is what answers. + * + * ONE-DIRECTIONAL on purpose: a name the box declares that the process does + * not have. A plugin the process carries that the declaration no longer names + * (a bundled one, one the owner just disabled) does NOT make this true — + * otherwise a box whose registry and config can never agree would report work + * outstanding for ever, and the watcher's first look would bounce the chat at + * every web-server boot. A disable still reaches the watcher, through the + * declaration's signature. + * + * THE ACCEPTED RESIDUAL, named rather than left as a clean sheet. A `true` + * here is honest about the files and the registry, and there are two real + * boxes on which it can never be made false by restarting: + * + * - a plugin REMOVED from the build but still listed in `plugins.enabled`. + * Hermes warns about exactly this itself ("Removed Hermes plugin %s is + * still listed in plugins.enabled", `hermes_cli/plugins.py`), so a box + * upgraded past a removal carries it; + * - a plugin whose own gate fails, which lands in `_plugins` with + * `enabled=False` and is therefore filtered out of the running set here. + * + * Paired with {@link changedAfterStart}, which any Settings save re-arms, the + * watcher's first look bounces the owner's chat once for a restart that + * cannot change the answer. It is BOUNDED and cannot loop — after the bounce + * the dashboard's start is newer than those files — so it is at most one + * outage per (Settings save → web-server restart) pair, which is the price of + * not reading "the registry disagrees" as "nothing is owed". Tightening it + * would mean requiring the name to be one `plugins.manage` actually + * discovered before calling it missing, and that trades this residual for a + * silent `null` on a genuinely absent plugin. + */ + readonly stale: boolean | null; + readonly dashboardStartedAt: number | null; + /** + * The declaration's inputs were touched AFTER the dashboard started, so the + * running process cannot have read what is on disk now. Null when either half + * is unknown. + * + * A WEAKER FACT THAN {@link stale} AND A DIFFERENT ONE. It is an mtime, so + * every Settings save makes it true — config.yaml is rewritten by each of + * them — which is exactly why it may not stand alone. What it has that + * `stale` has not is that it is SELF-LIMITING: once the dashboard has been + * restarted, its start is newer than any of those files, so a box whose + * declaration and registry can never agree cannot be bounced twice for the + * same reason. The watcher's first look requires both. + */ + readonly changedAfterStart: boolean | null; + /** The declaration's content hash, so a caller can record what it acted on. */ + readonly signature: string; +} + +/** Both questions, asked once. Never throws. */ +export async function readHermesPluginState(): Promise { + const [declaration, loaded, run] = await Promise.all([ + readHermesPluginDeclaration(), + readLoadedHermesPlugins().catch(() => null), + dashboardRun().catch(() => ({ invocationId: null, startedAtMs: null })), + ]); + const running = loaded === null ? null : new Set(loaded); + // THE NAME↔KEY MAP IS ASKED FOR ONLY WHEN IT IS NEEDED. A declared name that + // IS a registry key is settled by the set alone; anything left over — and any + // deny-list entry that has to be paired with an allow-list one — is what + // `plugins.manage` exists to resolve, and on most boxes there is nothing. + const looseNames = running === null ? [] : declaration.enabled.filter((name) => !running.has(name)); + const nameByKey = + looseNames.length > 0 || declaration.disabled.length > 0 ? await pluginNameByKey() : null; + // THE DENY-LIST WINS, as it does in `_plugin_status`. `hermes plugins disable` + // leaves the bare name a person wrote in `plugins.enabled` and adds the + // resolved key to `plugins.disabled`, so a box whose owner switched a plugin + // OFF declares it in both lists — and the running registry is right not to + // have it. Read from `enabled` alone, `stale` was permanently true there: the + // MCP tool warned "the agent is still behind the files" about a working box, + // and the watcher's first look would bounce the owner's chat at the next + // web-server boot over a restart that cannot change the answer. + const unresolved = looseNames.filter( + (name) => !declaration.disabled.some((denied) => samePlugin(name, denied, nameByKey)), + ); + // A DEFINITE MISSING NAME OUTRANKS AN UNRESOLVED ONE, and an unresolved one + // outranks "all present": `true` is what arms a restart of the owner's chat, + // so it is said only about a name the harness itself placed as absent, and + // everything this build could not establish answers null instead of guessing. + let missing = false; + let unknown = false; + for (const name of unresolved) { + const has = registryHas(name, running as ReadonlySet, nameByKey); + if (has === false) missing = true; + else if (has === null) unknown = true; + } + const stale = running === null ? null : missing ? true : unknown ? null : false; + return { + declared: declaration.names, + loaded, + stale, + dashboardStartedAt: run.startedAtMs, + changedAfterStart: + run.startedAtMs === null || declaration.changedAt === null + ? null + : declaration.changedAt > run.startedAtMs, + signature: declaration.signature, + }; +} diff --git a/src/lib/yaml-block-edit.ts b/src/lib/yaml-block-edit.ts index 2dc566cfb..6448e20c1 100644 --- a/src/lib/yaml-block-edit.ts +++ b/src/lib/yaml-block-edit.ts @@ -325,7 +325,7 @@ function unescapeDoubleQuoted(body: string): string | null { * Single quotes have exactly ONE escape, `''`; a backslash is ordinary data * there, and decoding it would invent a value PyYAML never produced. */ -function parseYamlScalar(raw: string): string | null { +export function parseYamlScalar(raw: string): string | null { const value = raw.trim(); if (value.length >= 2 && value.startsWith('"') && value.endsWith('"')) { return unescapeDoubleQuoted(value.slice(1, -1)); @@ -585,6 +585,109 @@ const ANY_KEY_RE = /^( *)(?:"[^"]*"|'[^']*'|[^\s#"'][^:#]*?) *:(?: (.*))?$/; /** A `#` opens a comment only when a token has just ENDED in front of it. */ const COMMENT_OPENERS = new Set([" ", '"', "'", "]", "}"]); +/** + * A line's VALUE text separated from its trailing `# comment`, for a reader + * that holds a whole value rather than one scalar. + * + * {@link splitTrailingComment} answers the same question for the editor and + * deliberately gives up (`closed: false`) on anything whose first character is + * not a quote and whose remainder it cannot place — it is about to REWRITE the + * value, so a shape it cannot name is one it must not touch. A reader has the + * opposite duty: `enabled: [a, b] # owner note` and `- superpowers # installed + * 2026-09-18` are both ordinary YAML that a person writes by hand, and a reader + * that swallowed the comment into the value invented a plugin name no registry + * can ever match — a `stale` that is true for ever and a chat restart that + * cannot fix it. + * + * QUOTE-AWARE, because a `#` inside a quoted scalar is data: `- "weird # name"` + * is one name with a hash in it. Scanning for the quote state is the only way + * to tell the two apart, which is why this is not a regex — a non-greedy + * `/(.*?)\s+#/` cuts `["a # b", c]` at the first hash inside the quotes. + */ +export function splitYamlComment(inline: string): { value: string; comment: string } { + let quote: '"' | "'" | null = null; + for (let i = 0; i < inline.length; i += 1) { + const ch = inline[i]; + if (quote) { + // A backslash escapes the next character in a double-quoted scalar only; + // in a single-quoted one it is ordinary data and `''` is the escape. + if (ch === "\\" && quote === '"') { + i += 1; + continue; + } + if (ch === quote) { + if (quote === "'" && inline[i + 1] === "'") { + i += 1; + continue; + } + quote = null; + } + continue; + } + if (ch === '"' || ch === "'") { + quote = ch; + continue; + } + if (ch === "#" && (i === 0 || COMMENT_OPENERS.has(inline[i - 1]) || inline[i - 1] === "\t")) { + return { value: inline.slice(0, i).trimEnd(), comment: inline.slice(i) }; + } + } + // An unterminated quote leaves the whole line as the value: this reader has + // not established where a comment starts, so it invents neither. + return { value: inline.trimEnd(), comment: "" }; +} + +/** + * The members of a YAML FLOW sequence — `[a, b]`, `["a, b", c]`, `[]` — or null + * when `value` is not one. + * + * `hermes config set plugins.enabled '["a","b"]'` writes this shape, so it is + * as real on a box as the block sequence beside it. Members are split on commas + * that are not inside a quoted scalar or a nested collection; the members + * themselves come back with their quotes still on, for {@link parseYamlScalar} + * to resolve. A run this reader cannot close — an unterminated quote, an + * unbalanced bracket — is `null` rather than a guess at what it meant. + */ +export function parseYamlFlowSequence(value: string): string[] | null { + const text = value.trim(); + if (!text.startsWith("[") || !text.endsWith("]")) return null; + const body = text.slice(1, -1); + const members: string[] = []; + let quote: '"' | "'" | null = null; + let depth = 0; + let start = 0; + for (let i = 0; i < body.length; i += 1) { + const ch = body[i]; + if (quote) { + if (ch === "\\" && quote === '"') { + i += 1; + continue; + } + if (ch === quote) { + if (quote === "'" && body[i + 1] === "'") { + i += 1; + continue; + } + quote = null; + } + continue; + } + if (ch === '"' || ch === "'") { + quote = ch; + continue; + } + if (ch === "[" || ch === "{") depth += 1; + else if (ch === "]" || ch === "}") depth -= 1; + else if (ch === "," && depth === 0) { + members.push(body.slice(start, i)); + start = i + 1; + } + } + if (quote || depth !== 0) return null; + members.push(body.slice(start)); + return members.map((member) => member.trim()).filter((member) => member.length > 0); +} + /** * Does this STRUCTURE line carry a TAB PyYAML would refuse? * diff --git a/src/tests/routes/clawkeep-restore-restart.test.ts b/src/tests/routes/clawkeep-restore-restart.test.ts index a8c40ddf9..9c5381063 100644 --- a/src/tests/routes/clawkeep-restore-restart.test.ts +++ b/src/tests/routes/clawkeep-restore-restart.test.ts @@ -69,8 +69,11 @@ vi.mock("@/lib/port-probe", async (orig) => ({ waitForPortOpen: async () => h.gatewayUp, })); -vi.mock("@/lib/hermes-dashboard-control", () => ({ - bounceHermesDashboard: async () => { +// The restore bounces through the SHARED CLAIM now, so the plugin watcher +// cannot open a window over it and its baseline moves with the ~/.hermes this +// restore has just replaced. Mocked at that seam, which is what the route calls. +vi.mock("@/lib/hermes-bounce-claim", () => ({ + bounceHermesDashboardShared: async () => { h.bounceCalls += 1; return h.bounce; }, diff --git a/src/tests/routes/hermes-plugins-reload.test.ts b/src/tests/routes/hermes-plugins-reload.test.ts new file mode 100644 index 000000000..44c7e9a50 --- /dev/null +++ b/src/tests/routes/hermes-plugins-reload.test.ts @@ -0,0 +1,205 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { installSessionFixture, type SessionFixture } from "@/tests/helpers/session"; +import { saveEnv } from "@/tests/helpers/env"; + +/** + * POST /setup-api/hermes/plugins/reload — the deliberate half of the same + * mechanism the watcher drives on its own. + * + * WHY THE AGENT IS AN INTENDED CALLER, unlike almost every other route that + * changes device state. The owner's transcript is the case: the assistant + * installed a plugin, verified it in a fresh `hermes chat -q`, then tried + * `sudo systemctl restart clawbox-hermes-dashboard` and was refused — correctly, + * because agent shells run with `no_new_privs` and no such grant exists or + * should. The thing it actually needed is not privileged at all, and this route + * is how it asks for it. Refusing the MCP bearer here would leave the assistant + * exactly where it was: able to install a plugin and unable to make it work. + * + * What it can do is bounded to that: restart a unit the clawbox user already + * owns, which comes straight back under `Restart=always`. It cannot start one + * that is stopped, cannot reach any other unit, and takes no argument that + * names one. + */ + +const bounceMock = vi.hoisted(() => vi.fn()); +const notifyMock = vi.hoisted(() => vi.fn()); +const stateMock = vi.hoisted(() => vi.fn()); + +vi.mock("@/lib/hermes-dashboard-control", () => ({ + bounceHermesDashboard: bounceMock, + hermesDashboardMainPid: vi.fn(async () => ({ read: true, pid: 4242 })), + hermesDashboardServing: vi.fn(async () => true), + hermesDashboardUnitState: vi.fn(async () => "running"), +})); +vi.mock("@/lib/email-notify", () => ({ notifyOwner: notifyMock })); +vi.mock("@/lib/hermes-plugin-set", async (importOriginal) => ({ + ...(await importOriginal()), + readHermesPluginState: stateMock, + readHermesPluginDeclaration: vi.fn(async () => ({ + names: ["superpowers"], + enabled: ["superpowers"], + signature: "sig", + changedAt: 1, + })), +})); + +describe("POST /setup-api/hermes/plugins/reload", () => { + let session: SessionFixture; + let restoreEnv: () => void; + const token = "b".repeat(64); + let post: typeof import("@/app/setup-api/hermes/plugins/reload/route").POST; + + const req = (auth: "owner" | "agent" | "none") => + new Request("http://localhost/setup-api/hermes/plugins/reload", { + method: "POST", + headers: + auth === "owner" + ? { Cookie: session.cookie } + : auth === "agent" + ? { Authorization: `Bearer ${token}` } + : {}, + }); + + beforeEach(async () => { + vi.resetModules(); + session = installSessionFixture(); + fs.writeFileSync(path.join(session.root, "data/.mcp-token"), token); + restoreEnv = saveEnv("CLAWBOX_EDITION"); + process.env.CLAWBOX_EDITION = "hermes"; + bounceMock.mockReset(); + notifyMock.mockReset(); + stateMock.mockReset(); + bounceMock.mockResolvedValue("restarted"); + notifyMock.mockResolvedValue(undefined); + stateMock.mockResolvedValue({ + declared: ["superpowers"], + loaded: ["superpowers"], + stale: false, + dashboardStartedAt: 1, + signature: "sig", + }); + post = (await import("@/app/setup-api/hermes/plugins/reload/route")).POST; + }); + + afterEach(() => { + restoreEnv(); + session.cleanup(); + }); + + it("admits the owner's session", async () => { + const response = await post(req("owner")); + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.restarted).toBe(true); + expect(body.ready).toBe(true); + expect(body.plugins).toContain("superpowers"); + }); + + it("admits the MCP bearer — the agent is who needs this", async () => { + const response = await post(req("agent")); + expect(response.status).toBe(200); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("refuses an unauthenticated caller", async () => { + expect((await post(req("none"))).status).toBe(401); + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("refuses a POST fired from ANOTHER SITE'S page, session cookie and all", async () => { + // The owner's browser attaches the session cookie to a POST any page on the + // web fires at the box, and this one drops their chat window — a site they + // merely visited could restart the assistant as often as it liked. + const cross = new Request("http://localhost/setup-api/hermes/plugins/reload", { + method: "POST", + headers: { Cookie: session.cookie, Origin: "https://evil.example" }, + }); + const response = await post(cross); + expect(response.status).toBe(403); + expect((await response.json()).code).toBe("cross_origin"); + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("still admits the header-less MCP bearer, which is the intended caller", async () => { + // The MCP server sends neither `Origin` nor `Sec-Fetch-Site`; its credential + // is what the gate above decides on. A same-origin check that refused a + // header-less caller would have taken the tool away from the agent. + const response = await post(req("agent")); + expect(response.status).toBe(200); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("admits the owner's own page, which sends an Origin of its own", async () => { + const own = new Request("http://localhost/setup-api/hermes/plugins/reload", { + method: "POST", + headers: { Cookie: session.cookie, Origin: "http://localhost", Host: "localhost" }, + }); + expect((await post(own)).status).toBe(200); + }); + + it("refuses on an edition with no Hermes dashboard, and restarts nothing", async () => { + // An OpenClaw box has no dashboard by design. Bouncing "the chat backend" + // there would be this route reaching for a unit the foreign-edition teardown + // deliberately stopped. + process.env.CLAWBOX_EDITION = "openclaw"; + vi.resetModules(); + const openclawPost = (await import("@/app/setup-api/hermes/plugins/reload/route")).POST; + const response = await openclawPost(req("owner")); + expect(response.status).toBe(404); + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("answers 502 — not 200 — when the dashboard did not come back", async () => { + // THE FALSE SUCCESS THIS ROUTE EXISTS TO AVOID. Reporting 200 over a + // dashboard that is down tells the assistant its plugin is live and sends it + // on to use a tool that is not there. + bounceMock.mockResolvedValue("failed"); + const response = await post(req("agent")); + expect(response.status).toBe(502); + const body = await response.json(); + expect(body.restarted).toBe(false); + expect(body.ready).toBe(false); + }); + + it("answers 200 with ready:false when the restart took but has not finished", async () => { + // `pending` means systemd owns it and it is on its way. Acting makes it + // worse, so this is not an error — but it is not "ready" either, and the + // caller must be able to tell. + bounceMock.mockResolvedValue("pending"); + const response = await post(req("agent")); + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.restarted).toBe(true); + expect(body.ready).toBe(false); + }); + + it("tells the owner their chat window dropped", async () => { + await post(req("agent")); + expect(notifyMock).toHaveBeenCalledTimes(1); + expect(String(notifyMock.mock.calls[0][0]).toLowerCase()).toContain("new chat"); + }); + + it("uses no sudo and names no unit — the mechanism needs neither", async () => { + // The whole mechanism, asserted from the outside: the restart this route + // performs is `bounceHermesDashboard()`, which is unprivileged. If this route + // ever grew a `sudo systemctl restart clawbox-hermes-dashboard`, it would + // need a grant that `install-sudoers-migration.test.ts` refuses to give it — + // and that grant would let an OpenClaw box START the dashboard its + // foreign-edition teardown had just stopped and disabled. + // + // COMMENTS ARE STRIPPED FIRST, deliberately. The file's own doc block has to + // be able to say the words "sudo systemctl restart" in order to explain why + // it does not do it, and a check that forbade the explanation would push the + // reasoning out of the file it belongs in. + const source = fs.readFileSync( + path.join(process.cwd(), "src/app/setup-api/hermes/plugins/reload/route.ts"), + "utf-8", + ); + const code = source.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/gm, ""); + expect(code).not.toMatch(/sudo/); + expect(code).not.toMatch(/systemctl/); + expect(code).not.toMatch(/execFile|spawn/); + }); +}); diff --git a/src/tests/unit/hermes-dashboard-control.test.ts b/src/tests/unit/hermes-dashboard-control.test.ts index 36d0aa312..11eb4ffdf 100644 --- a/src/tests/unit/hermes-dashboard-control.test.ts +++ b/src/tests/unit/hermes-dashboard-control.test.ts @@ -125,14 +125,112 @@ describe("bounceHermesDashboard", () => { expect(cliMock).not.toHaveBeenCalled(); }); + it("refuses — WITHOUT stopping anything — when the outgoing PID cannot be read", async () => { + // The baseline this whole bounce is measured against. `MainPID=0` is a + // FACT ("no running main process"); a `systemctl show` that produced + // nothing is not an answer at all, and folding the two together made the + // second read as a valid "nothing was running" — so the very next pid seen, + // which is the process this bounce was supposed to stop, satisfied the + // replacement wait and the bounce reported `restarted` over a dashboard it + // had not replaced. + // + // Refusing before the stop leaves a working dashboard working, and "failed" + // is the honest answer: nothing was done. + execFileMock.mockImplementation((_bin: string, args: string[], _opts: unknown, cb: unknown) => { + const done = cb as (e: Error | null, out: { stdout: string; stderr: string }) => void; + const property = (args as string[]).find((a) => a.startsWith("--property=")) ?? ""; + // The restart policy reads fine; only the pid query is unanswerable. + if (property === "--property=MainPID") { + done(null, { stdout: "", stderr: "" }); + return; + } + done(null, { stdout: "Restart=always\n", stderr: "" }); + }); + + await expect(bounceHermesDashboard()).resolves.toBe("failed"); + expect(cliMock).not.toHaveBeenCalled(); + }); + + it("retries the baseline read once, because one hiccup must not cost a restart", async () => { + let asked = 0; + execFileMock.mockImplementation((_bin: string, args: string[], _opts: unknown, cb: unknown) => { + const done = cb as (e: Error | null, out: { stdout: string; stderr: string }) => void; + const property = (args as string[]).find((a) => a.startsWith("--property=")) ?? ""; + if (property === "--property=MainPID") { + asked += 1; + // The first read fails; from the second on it answers, and the + // replacement is a different process. + done(null, { stdout: asked === 1 ? "" : `MainPID=${asked === 2 ? 4242 : 5353}\n`, stderr: "" }); + return; + } + done(null, { stdout: "Restart=always\n", stderr: "" }); + }); + + await expect(bounceHermesDashboard()).resolves.toBe("restarted"); + expect(cliMock).toHaveBeenCalledWith(["dashboard", "--stop"], expect.anything()); + }); + + it("still treats a valid MainPID=0 as a baseline, because that is an answer", async () => { + // A unit that is genuinely stopped. Systemd answering `0` is a fact, and + // the replacement's pid is different from "none" — which is what the + // existing "counts a respawn even when the unit had no running process" + // case is about. The refusal above must not have swallowed it. + systemd({ pids: ["0", "5353"] }); + await expect(bounceHermesDashboard()).resolves.toBe("restarted"); + }); + + it("counts a stop that exited NON-ZERO but did stop the dashboard as a restart", async () => { + // MEASURED on the owner's Hermes box (2026-09-18): `hermes dashboard --stop` + // exits 143 — SIGTERM, 128+15 — after 764 ms, printing "Terminated", while + // stopping the dashboard perfectly well; MainPID went 17768 → 17877 across + // that call. The CLI signals the process group it is itself in, so it kills + // its own process on the way out. + // + // Reading that code as the outcome was a false failure on EVERY bounce this + // device performs: a ClawKeep restore told the owner its restored state.db + // was not being served, the image refresh told them the box could not draw, + // and the plugin watcher sent no "open a new chat" notice for a chat window + // it had just dropped. The outcome is systemd's new MainPID and a socket + // that answers — neither of which an exit code can fake. + cliMock.mockResolvedValue({ code: 143, stdout: "Terminated", stderr: "" }); + await expect(bounceHermesDashboard()).resolves.toBe("restarted"); + }); + it("reports a failure when the stop itself did not take", async () => { + // The branch the exit-code check used to cover, asked of the thing that + // knows instead: the SAME process is still the unit's main one and systemd + // calls the unit running, so nothing was stopped and nothing is coming. + // `pending` here would be the worst of the three answers — it means "leave + // it alone" over a dashboard that will stay stale until somebody acts. cliMock.mockResolvedValue({ code: 1, stdout: "", stderr: "unkillable" }); - await expect(bounceHermesDashboard()).resolves.toBe("failed"); + systemd({ + pids: ["4242", "4242"], + unit: { LoadState: "loaded", ActiveState: "active", SubState: "running" }, + }); + process.env.HERMES_DASHBOARD_WAIT_MS = "40"; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + await expect(bounceHermesDashboard()).resolves.toBe("failed"); + } finally { + errorSpy.mockRestore(); + delete process.env.HERMES_DASHBOARD_WAIT_MS; + } }); it("does not throw when the CLI is missing entirely", async () => { cliMock.mockRejectedValue(new Error("ENOENT")); - await expect(bounceHermesDashboard()).resolves.toBe("failed"); + systemd({ + pids: ["4242", "4242"], + unit: { LoadState: "loaded", ActiveState: "active", SubState: "running" }, + }); + process.env.HERMES_DASHBOARD_WAIT_MS = "40"; + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + await expect(bounceHermesDashboard()).resolves.toBe("failed"); + } finally { + errorSpy.mockRestore(); + delete process.env.HERMES_DASHBOARD_WAIT_MS; + } }); /** diff --git a/src/tests/unit/hermes-image-refresh.test.ts b/src/tests/unit/hermes-image-refresh.test.ts index 2e0bb9850..7251289da 100644 --- a/src/tests/unit/hermes-image-refresh.test.ts +++ b/src/tests/unit/hermes-image-refresh.test.ts @@ -16,7 +16,7 @@ const bounceMock = vi.hoisted(() => vi.fn()); const reloadMcpMock = vi.hoisted(() => vi.fn()); vi.mock("@/lib/hermes-dashboard-rpc", () => ({ dashboardRpc: rpcMock })); -vi.mock("@/lib/hermes-dashboard-control", () => ({ bounceHermesDashboard: bounceMock })); +vi.mock("@/lib/hermes-bounce-claim", () => ({ bounceHermesDashboardShared: bounceMock })); // Only the ASK is faked. `reportMcpReloadRefused` is the real one, because the // rule it owns — an OpenClaw box has no dashboard and needs no repair, a Hermes // box that refused does — is part of what this file pins. diff --git a/src/tests/unit/hermes-plugin-reload.test.ts b/src/tests/unit/hermes-plugin-reload.test.ts new file mode 100644 index 000000000..27064a899 --- /dev/null +++ b/src/tests/unit/hermes-plugin-reload.test.ts @@ -0,0 +1,881 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { saveEnv } from "@/tests/helpers/env"; + +/** + * A plugin the owner installs after boot has to reach the chat by itself. + * + * WHAT THE MECHANISM IS, because it is the thing most easily got wrong here. + * Hermes has no runtime plugin reload — its own `plugins install` ends with + * "Restart the gateway for the plugin to take effect" — so the only way a + * running process sees a new plugin is a restart of that process. On this SKU + * the process serving chat is `clawbox-hermes-dashboard.service`, and ClawBox + * ALREADY has the restart for it: `bounceHermesDashboard()`, which stops the + * dashboard as the clawbox user that owns it and lets the unit's + * `Restart=always` bring it back, then waits for a NEW main PID and for :9119 to + * answer before it says "restarted". + * + * It needs no root and it must not get any. A `systemctl restart` grant over + * this unit would also START it, which is exactly how an OpenClaw box could + * resurrect the dashboard its foreign-edition teardown had just stopped and + * disabled — the invariant `sudoers-coverage.test.ts` and + * `install-foreign-edition-teardown.test.ts` both own. + * + * So what is new here is not a way to restart. It is knowing WHEN to, doing it + * once per burst, and telling the owner — whose open chat window the restart + * drops, deliberately. + */ + +const bounceMock = vi.hoisted(() => vi.fn()); +const notifyMock = vi.hoisted(() => vi.fn()); +const declarationMock = vi.hoisted(() => vi.fn()); +const stateMock = vi.hoisted(() => vi.fn()); +const servingMock = vi.hoisted(() => vi.fn()); +const unitStateMock = vi.hoisted(() => vi.fn()); +const mainPidMock = vi.hoisted(() => vi.fn()); + +vi.mock("@/lib/hermes-dashboard-control", () => ({ + bounceHermesDashboard: bounceMock, + hermesDashboardServing: servingMock, + hermesDashboardUnitState: unitStateMock, + hermesDashboardMainPid: mainPidMock, +})); +vi.mock("@/lib/email-notify", () => ({ notifyOwner: notifyMock })); +vi.mock("@/lib/hermes-plugin-set", async (importOriginal) => ({ + ...(await importOriginal()), + readHermesPluginDeclaration: declarationMock, + readHermesPluginState: stateMock, +})); + +import { + HERMES_PLUGIN_DEBOUNCE_MS, + _resetHermesPluginReloadStateForTests, + bounceHermesDashboardShared, + createHermesPluginWatcher, + reloadHermesPlugins, +} from "@/lib/hermes-plugin-reload"; + +/** A declaration, as the watcher reads it. */ +function declared(names: string[]) { + return { names, enabled: names, signature: `sig:${names.join(",")}`, changedAt: 1_000 }; +} + +let restoreEnv: () => void; + +beforeEach(() => { + bounceMock.mockReset(); + notifyMock.mockReset(); + declarationMock.mockReset(); + stateMock.mockReset(); + servingMock.mockReset(); + unitStateMock.mockReset(); + mainPidMock.mockReset(); + // A replacement by default: a different pid every read, so a test that says + // nothing about pids gets the ordinary "systemd started a new process". + let nextPid = 1000; + mainPidMock.mockImplementation(async () => ({ read: true, pid: (nextPid += 1) })); + bounceMock.mockResolvedValue("restarted"); + notifyMock.mockResolvedValue(undefined); + servingMock.mockResolvedValue(true); + unitStateMock.mockResolvedValue("running"); + declarationMock.mockResolvedValue(declared(["superpowers"])); + stateMock.mockResolvedValue({ + declared: ["superpowers"], + loaded: ["superpowers"], + stale: false, + changedAfterStart: false, + dashboardStartedAt: 1, + signature: "sig:superpowers", + }); + // The baseline lives in `process-store.ts` now — shared with the route and + // every other caller that bounces the dashboard, and therefore shared between + // tests in one process until it is put back. + _resetHermesPluginReloadStateForTests(); + restoreEnv = saveEnv("CLAWBOX_EDITION"); + process.env.CLAWBOX_EDITION = "hermes"; +}); + +afterEach(() => restoreEnv()); + +describe("the plugin watcher", () => { + /** A watcher whose clock the test moves by hand. */ + function watcherAt(clock: { ms: number }) { + return createHermesPluginWatcher({ now: () => clock.ms }); + } + + it("takes the plugin set it finds at boot as the baseline, and restarts nothing", async () => { + // THE CASE THAT WOULD HAVE MADE THIS A BOOT LOOP. The web server starts, + // reads a box that has had `superpowers` installed for a week, and must not + // conclude that anything changed. Every restart it orders drops the owner's + // chat, and this one would order one at every boot. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + expect(await watcher.tick()).toBe("baseline"); + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("does nothing while the plugin set is unchanged", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + for (let i = 0; i < 5; i++) { + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 2; + expect(await watcher.tick()).toBe("unchanged"); + } + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("restarts the dashboard once when a plugin is installed", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + // The change is SEEN, but not acted on yet: `hermes plugins install` writes + // the ledger and the config a moment apart, and acting on the first write + // would restart into a half-declared set and then again on the second. + expect(await watcher.tick()).toBe("waiting"); + expect(bounceMock).not.toHaveBeenCalled(); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("collapses a burst of writes into ONE restart", async () => { + // `hermes plugins install` touches the ledger, the config and the plugin + // directory; an enable touches the config again. Each of those is a change, + // and a restart per change is four outages for one action. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + + for (const set of [["a"], ["a", "b"], ["a", "b", "c"]]) { + declarationMock.mockResolvedValue(declared(set)); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS / 2; + expect(await watcher.tick()).toBe("waiting"); + } + expect(bounceMock).not.toHaveBeenCalled(); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("does not restart again for the set it has already restarted for", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + await watcher.tick(); + + for (let i = 0; i < 5; i++) { + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 2; + expect(await watcher.tick()).toBe("unchanged"); + } + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("ignores a rewrite whose CONTENT is the same", async () => { + // The dashboard's own `ExecStartPre` re-provisions auth and rewrites + // config.yaml on every start. Keyed on mtime, this watcher would restart the + // dashboard because the dashboard restarted — for ever. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + // A new object every call, same signature: the reader hashes content. + declarationMock.mockImplementation(async () => declared(["superpowers"])); + for (let i = 0; i < 4; i++) { + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 2; + expect(await watcher.tick()).toBe("unchanged"); + } + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("forgets a change that was undone before the window closed", async () => { + // Install then immediately remove: nothing is different from what the + // dashboard is already running, so there is nothing to restart FOR. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + declarationMock.mockResolvedValue(declared(["superpowers", "oops"])); + expect(await watcher.tick()).toBe("waiting"); + declarationMock.mockResolvedValue(declared(["superpowers"])); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("unchanged"); + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("tells the owner the chat restarted, naming the plugin", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + await watcher.tick(); + + expect(notifyMock).toHaveBeenCalledTimes(1); + const message = String(notifyMock.mock.calls[0][0]); + expect(message).toContain("weather"); + // The owner's open chat window went down with the restart. The notice is + // the only thing that tells them why, and what to do about it. + expect(message.toLowerCase()).toContain("new chat"); + }); + + it("does not take the baseline forward when the restart failed — it tries again", async () => { + // FALSE SUCCESS, the exact shape. A bounce that answered "failed" left the + // dashboard running the OLD plugin set — so recording the new signature as + // "done" would mean nothing ever tried again, and the plugin would stay + // invisible until the next reboot with the box reporting no work outstanding. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + bounceMock.mockResolvedValue("failed"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("failed"); + + bounceMock.mockResolvedValue("restarted"); + // The retry waits a WIDER window than the first attempt — see below — so the + // clock has to carry past it, and the change is still outstanding when it does. + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(2); + }); + + it("backs off between failed restarts instead of bouncing a broken box for ever", async () => { + // A dashboard systemd has GIVEN UP on — crash-looped past its start limit, + // which this module is unprivileged to clear — answers "failed" after + // spending the bounce's whole budget. Retrying it on every poll would be a + // SIGTERM every few seconds against a process already failing to start, for + // the life of the web server. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + bounceMock.mockResolvedValue("failed"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("failed"); + expect(bounceMock).toHaveBeenCalledTimes(1); + + // One more debounce window is no longer enough. + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("waiting"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("clears the backoff once a restart succeeds", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + bounceMock.mockResolvedValue("failed"); + declarationMock.mockResolvedValue(declared(["a"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + await watcher.tick(); + + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("restarted"); + + // A LATER, UNRELATED change must not inherit the widened window: the box is + // healthy again, and the owner installing a second plugin should wait the + // ordinary debounce, not the backoff the first failure earned. + declarationMock.mockResolvedValue(declared(["a", "b"])); + expect(await watcher.tick()).toBe("waiting"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("restarted"); + }); + + it("does not seed a baseline over a dashboard that is BEHIND the files", async () => { + // The baseline is supposed to be "the set the RUNNING dashboard loaded", + // and the dashboard OUTLIVES the web server: an update or a + // `clawbox-setup` restart re-seeds it from a file the chat backend has + // never read. `scripts/register-mcp.sh` makes that concrete — it runs at + // EVERY web-server boot and can append the EMAIL-directive hook plugin to + // `plugins.enabled`, and whether the plugin stayed invisible for the rest + // of that dashboard's life was a race between its write and this tick five + // seconds later. + stateMock.mockResolvedValue({ + declared: ["superpowers"], loaded: [], stale: true, changedAfterStart: true, + dashboardStartedAt: 1, signature: "sig:superpowers", + }); + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + expect(await watcher.tick()).toBe("waiting"); + expect(bounceMock).not.toHaveBeenCalled(); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("still seeds when the running dashboard cannot be asked at all", async () => { + // `null` is "could not be established", and a watcher that bounced the + // owner's chat on every boot it could prove nothing about would be worse + // than one that waits for the next real change. + stateMock.mockResolvedValue({ + declared: ["superpowers"], loaded: null, stale: null, changedAfterStart: null, + dashboardStartedAt: null, signature: "sig:superpowers", + }); + const clock = { ms: 0 }; + expect(await watcherAt(clock).tick()).toBe("baseline"); + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("seeds — and bounces NOTHING — on a box that is behind but whose files predate the dashboard", async () => { + // THE DEFECT HARDWARE FOUND, on the owner's own box (2026-09-18). A + // declaration and a registry that can never agree leave `stale` true for + // good, and a seed rule that read it alone bounced the owner's chat at + // EVERY web-server boot — one outage per update, per `clawbox-setup` + // restart, for ever. The second fact is what bounds it: once the dashboard + // has restarted, its start is newer than the files, so there is nothing new + // to restart FOR. + stateMock.mockResolvedValue({ + declared: ["superpowers"], loaded: ["other"], stale: true, changedAfterStart: false, + dashboardStartedAt: 5_000, signature: "sig:superpowers", + }); + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + expect(await watcher.tick()).toBe("baseline"); + for (let i = 0; i < 4; i++) { + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 2; + expect(await watcher.tick()).toBe("unchanged"); + } + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("opens no window while ANOTHER caller's bounce is in flight", async () => { + // The route and the MCP tool bounce through the same function. A watcher + // that opened a window over one of those SIGTERMs a dashboard that is in + // the middle of coming back — one action, two outages, two "open a new + // chat" notices fifteen seconds apart. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + + let release: () => void = () => {}; + let entered: () => void = () => {}; + const inBounce = new Promise((resolve) => { entered = resolve; }); + bounceMock.mockImplementation(() => { + entered(); + return new Promise((resolve) => { release = () => resolve("restarted"); }); + }); + const inFlight = reloadHermesPlugins("the assistant asked"); + // THE BOUNCE HAVING STARTED IS THE BARRIER, not a count of microtask turns: + // `await Promise.resolve()` proved nothing about where the other call had + // got to, so this test could pass on scheduling rather than on the flag. + await inBounce; + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("waiting"); + release(); + await inFlight; + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("takes the baseline from a reload the ROUTE performed, so one change is one restart", async () => { + // THE TWO-MODULE-COPIES TRAP. `instrumentation.ts` reaches this file with + // `require(...)` while the route `import`s it, so a baseline kept in a + // module-level `let` is not the same object in the two — and the assistant + // doing exactly what `hermes_plugins_reload` tells it to do (install, then + // call the tool) dropped the owner's chat twice. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await reloadHermesPlugins("the assistant asked"); + expect(bounceMock).toHaveBeenCalledTimes(1); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("unchanged"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("does not take the baseline forward over a bounce that is only PENDING", async () => { + // `pending` means the stop took and systemd owns the unit — not that the + // replacement is up. Recording it as loaded would leave a plugin invisible + // with the box reporting no work outstanding, which is the same false + // success as recording a failed one. + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + bounceMock.mockResolvedValue("pending"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("pending"); + + // The next window RECONCILES rather than stopping anything again: the + // replacement, whenever it arrives, reads the files as they are now. + servingMock.mockResolvedValue(true); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("reconciled"); + expect(bounceMock).toHaveBeenCalledTimes(1); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("unchanged"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("waits, without stopping anything, while the replacement is still coming up", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + bounceMock.mockResolvedValue("pending"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + await watcher.tick(); + + servingMock.mockResolvedValue(false); + unitStateMock.mockResolvedValue("restarting"); + for (let i = 0; i < 3; i++) { + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("waiting"); + } + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("bounces again only once recovery is established as FAILED", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + bounceMock.mockResolvedValue("pending"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + await watcher.tick(); + + // Not serving, and systemd says nothing is coming on its own. + servingMock.mockResolvedValue(false); + unitStateMock.mockResolvedValue("down"); + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(2); + }); + + it("survives a read that throws, and keeps polling", async () => { + const clock = { ms: 0 }; + const watcher = watcherAt(clock); + await watcher.tick(); + declarationMock.mockRejectedValueOnce(new Error("ENOENT")); + expect(await watcher.tick()).toBe("unreadable"); + declarationMock.mockResolvedValue(declared(["superpowers"])); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 2; + expect(await watcher.tick()).toBe("unchanged"); + }); +}); + +describe("reloadHermesPlugins", () => { + it("restarts the dashboard and reports what it now loads", async () => { + stateMock.mockResolvedValue({ + declared: ["superpowers"], + loaded: ["superpowers"], + stale: false, + dashboardStartedAt: 2, + }); + const result = await reloadHermesPlugins("the assistant asked"); + expect(result.restarted).toBe(true); + expect(result.ready).toBe(true); + expect(result.plugins).toContain("superpowers"); + }); + + it("reports `pending` as not-ready rather than as a failure", async () => { + // `bounceHermesDashboard` tells "the stop took and systemd owns it now" apart + // from "nothing is coming back". They are opposite instructions to the + // caller, and collapsing them is what once had an owner run `systemctl + // restart` by hand over a dashboard that was already restarting. + bounceMock.mockResolvedValue("pending"); + const result = await reloadHermesPlugins("the assistant asked"); + expect(result.restarted).toBe(true); + expect(result.ready).toBe(false); + }); + + it("says so, and restarts nothing, when the bounce failed", async () => { + bounceMock.mockResolvedValue("failed"); + const result = await reloadHermesPlugins("the assistant asked"); + expect(result.restarted).toBe(false); + expect(result.ready).toBe(false); + }); + + it("never claims a plugin loaded when the running dashboard cannot be asked", async () => { + // `loaded: null` is "this box cannot be asked", which is not "it loaded + // nothing". The answer carries the declared set and says the loaded one is + // unknown rather than inventing an empty one. + stateMock.mockResolvedValue({ + declared: ["superpowers"], + loaded: null, + stale: null, + dashboardStartedAt: null, + }); + const result = await reloadHermesPlugins("the assistant asked"); + expect(result.plugins).toEqual(["superpowers"]); + expect(result.loaded).toBeNull(); + }); +}); + +describe("the shared bounce claim", () => { + it("adds no second outage when another caller already holds the claim", async () => { + // THE OWNER'S CARD AND THE AGENT'S TOOL, pressed together. The claim used to + // be taken AFTER an await (reading the declaration), so both calls got past + // the check and both stopped the dashboard — and the first `finally` then + // cleared a flag the second bounce was still relying on, which is the window + // the watcher SIGTERMs a restarting dashboard through. + let release: () => void = () => {}; + let entered: () => void = () => {}; + const inBounce = new Promise((resolve) => { entered = resolve; }); + bounceMock.mockImplementation(() => { + entered(); + return new Promise((resolve) => { release = () => resolve("restarted"); }); + }); + + const first = reloadHermesPlugins("the owner pressed reload"); + await inBounce; + const second = await reloadHermesPlugins("the assistant asked"); + + expect(second.restarted).toBe(false); + expect(second.inFlight).toBe(true); + expect(bounceMock).toHaveBeenCalledTimes(1); + release(); + expect((await first).restarted).toBe(true); + }); + + it("moves the baseline for a DIRECT bounce, so the watcher does not repeat it", async () => { + // `POST /setup-api/clawkeep/restore` rewrites the whole of ~/.hermes and + // bounces the dashboard itself; `hermes-image-refresh` bounces it to pick up + // the image backend. Neither went through this module, so neither raised the + // gate nor moved the baseline: one restore was two chat outages and two + // "open a new chat" notices about a plugin set nobody had touched. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + expect(await bounceHermesDashboardShared("a ClawKeep restore replaced ~/.hermes")).toBe("restarted"); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("unchanged"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("does not reconcile over a dashboard that is still the SAME process", async () => { + // `bounceHermesDashboard` answers `pending` for any unit state that is not + // `running`/`down`, which includes both systemd reads having failed — and + // there the old process never stopped. The port then answers (it is the OLD + // process answering), and the watcher recorded the change as loaded and gave + // up. The pid is the one fact that tells the two apart. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + mainPidMock.mockResolvedValue({ read: true, pid: 4242 }); + bounceMock.mockResolvedValue("pending"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("pending"); + + // The port answers — but it is the same process answering, so this is not a + // replacement and the bounce has to be taken again rather than recorded. + servingMock.mockResolvedValue(true); + bounceMock.mockResolvedValue("restarted"); + mainPidMock.mockResolvedValue({ read: true, pid: 4242 }); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(2); + }); + + it("reconciles when the pid really did move", async () => { + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + mainPidMock.mockResolvedValue({ read: true, pid: 4242 }); + bounceMock.mockResolvedValue("pending"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("pending"); + + servingMock.mockResolvedValue(true); + mainPidMock.mockResolvedValue({ read: true, pid: 5151 }); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("reconciled"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + /** + * A pid that cannot settle the question at all — `awaitingPid` is null when + * the pre-bounce read failed, and when systemd reported the perfectly valid + * `MainPID=0`. The port answering is not the missing proof: a `pending` from + * "systemd could not be asked, twice" leaves the OLD process serving :9119 + * exactly as well as a replacement would. + */ + function bounceWithNoPidBaseline() { + mainPidMock.mockResolvedValue({ read: false, pid: null }); + bounceMock.mockResolvedValue("pending"); + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + } + + it("does not reconcile on the port alone when no pid can prove the replacement", async () => { + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + bounceWithNoPidBaseline(); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("pending"); + + // The port answers, and the running registry STILL does not have what the + // box declares — so whatever is serving has not read the change. + servingMock.mockResolvedValue(true); + stateMock.mockResolvedValue({ + declared: ["superpowers", "weather"], + loaded: ["superpowers"], + stale: true, + changedAfterStart: true, + dashboardStartedAt: 1, + signature: "sig:superpowers,weather", + }); + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(2); + }); + + it("reconciles on the registry's own answer when no pid can prove it", async () => { + // The other half, so the rule above cannot be tightened into a watcher that + // never finishes: `stale: false` is the running process saying it HAS the + // set the box declares, which is exactly what the baseline records. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + bounceWithNoPidBaseline(); + await watcher.tick(); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS + 1; + expect(await watcher.tick()).toBe("pending"); + + servingMock.mockResolvedValue(true); + stateMock.mockResolvedValue({ + declared: ["superpowers", "weather"], + loaded: ["superpowers", "weather"], + stale: false, + changedAfterStart: false, + dashboardStartedAt: 9_000, + signature: "sig:superpowers,weather", + }); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("reconciled"); + expect(bounceMock).toHaveBeenCalledTimes(1); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("unchanged"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("takes one more bounce for a change that arrived while the claim was held", async () => { + // `in_flight` says another restart owns the dashboard. It does NOT say the + // replacement will have read what THIS caller just wrote: a ClawKeep + // restore rewrites the whole of ~/.hermes, and a restore that lands after + // the running bounce has read its files is served by a process that never + // saw it. Nothing then reloaded it — the declaration need not have moved, + // so the watcher's own signature comparison cannot catch it either. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + let release: () => void = () => {}; + let entered: () => void = () => {}; + const inBounce = new Promise((resolve) => { entered = resolve; }); + bounceMock.mockImplementation(() => { + entered(); + return new Promise((resolve) => { release = () => resolve("restarted"); }); + }); + + const first = bounceHermesDashboardShared("the assistant asked"); + await inBounce; + expect(await bounceHermesDashboardShared("a ClawKeep restore replaced ~/.hermes")).toBe("in_flight"); + release(); + await first; + + bounceMock.mockReset(); + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("waiting"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("keeps that debt when the WATCHER is the one holding the claim", async () => { + // The sibling write. `reloadHermesPlugins` leaves the baseline to the claim, + // but the watcher records its own `shared.baseline = acted` after a restart + // that worked — which would put the signature straight back over the mark + // the claim had just made, and the restore that arrived mid-bounce would be + // forgotten again. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + let release: () => void = () => {}; + let entered: () => void = () => {}; + const inBounce = new Promise((resolve) => { entered = resolve; }); + bounceMock.mockImplementation(() => { + entered(); + return new Promise((resolve) => { release = () => resolve("restarted"); }); + }); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("waiting"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + const bouncing = watcher.tick(); + await inBounce; + expect(await bounceHermesDashboardShared("a ClawKeep restore replaced ~/.hermes")).toBe("in_flight"); + release(); + expect(await bouncing).toBe("restarted"); + + bounceMock.mockReset(); + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("waiting"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("adds no extra bounce when nobody arrived during the one in flight", async () => { + // The debt above is owed only to a caller that actually arrived: an + // ordinary bounce must still be ONE restart, which is the whole point of + // the claim recording a baseline. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + expect(await bounceHermesDashboardShared("the owner pressed reload")).toBe("restarted"); + + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("unchanged"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + /** + * A HELD BOUNCE the test can let go of, so a second caller is provably + * inside the claim's window when it arrives. + */ + function heldBounce() { + let release: () => void = () => {}; + let entered: () => void = () => {}; + const inBounce = new Promise((resolve) => { entered = resolve; }); + bounceMock.mockImplementation(() => { + entered(); + return new Promise((resolve) => { release = () => resolve("restarted"); }); + }); + return { inBounce, release: () => release() }; + } + + it("owes NOTHING to a second plugin reload for the same declaration", async () => { + // THE DEBT IS FOR A MUTATION THE RUNNING BOUNCE CANNOT HAVE READ, and a + // plugin reload for the signature that bounce is already acting on is not + // one: the replacement coming up reads those very files. Owed anyway — as + // it was for EVERY in-flight caller — it put `BASELINE_BEHIND` over the + // baseline, and the watcher's next window dropped the owner's chat a + // second time for a plugin set the new process had already loaded. The + // owner's card and the agent's `hermes_plugins_reload` pressed together is + // exactly that pair. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + const held = heldBounce(); + const first = reloadHermesPlugins("the assistant asked", { signature: "sig:superpowers,weather" }); + await held.inBounce; + const second = await reloadHermesPlugins("the owner pressed reload", { + signature: "sig:superpowers,weather", + }); + expect(second.inFlight).toBe(true); + held.release(); + await first; + + bounceMock.mockReset(); + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 8; + expect(await watcher.tick()).toBe("unchanged"); + expect(bounceMock).not.toHaveBeenCalled(); + }); + + it("keeps the debt when the second reload's declaration is a DIFFERENT one", async () => { + // `hermes plugins install` landing while the first bounce is in flight: + // the replacement read the set as it was, and what this caller wrote is + // still outstanding. The signature is what says so. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + const held = heldBounce(); + const first = reloadHermesPlugins("the assistant asked", { signature: "sig:superpowers,weather" }); + await held.inBounce; + declarationMock.mockResolvedValue(declared(["superpowers", "weather", "images"])); + expect( + (await reloadHermesPlugins("the owner pressed reload", { + signature: "sig:superpowers,weather,images", + })).inFlight, + ).toBe(true); + held.release(); + await first; + + bounceMock.mockReset(); + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("waiting"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); + + it("keeps the debt for a RESTORE that arrived during a plugin reload", async () => { + // The signature does not represent what a ClawKeep restore changed: it puts + // back the whole of ~/.hermes and its plugin set can be byte-identical + // beside a completely different `state.db`. A caller that is not a plugin + // reload therefore owes the bounce whatever the signatures say — the same + // for the image refresh, which installs a backend the declaration never + // mentions. + const clock = { ms: 0 }; + const watcher = watcherAtTop(clock); + await watcher.tick(); + + declarationMock.mockResolvedValue(declared(["superpowers", "weather"])); + const held = heldBounce(); + const first = reloadHermesPlugins("the assistant asked", { signature: "sig:superpowers,weather" }); + await held.inBounce; + expect(await bounceHermesDashboardShared("a ClawKeep restore replaced ~/.hermes")).toBe("in_flight"); + held.release(); + await first; + + bounceMock.mockReset(); + bounceMock.mockResolvedValue("restarted"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("waiting"); + clock.ms += HERMES_PLUGIN_DEBOUNCE_MS * 4; + expect(await watcher.tick()).toBe("restarted"); + expect(bounceMock).toHaveBeenCalledTimes(1); + }); +}); + +/** The same clock-driven watcher the suite above builds; hoisted for reuse. */ +function watcherAtTop(clock: { ms: number }) { + return createHermesPluginWatcher({ now: () => clock.ms }); +} diff --git a/src/tests/unit/hermes-plugin-set.test.ts b/src/tests/unit/hermes-plugin-set.test.ts new file mode 100644 index 000000000..70c666836 --- /dev/null +++ b/src/tests/unit/hermes-plugin-set.test.ts @@ -0,0 +1,786 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { saveEnv } from "@/tests/helpers/env"; + +/** + * "Installed" is not "loaded", and this file is where the two are told apart. + * + * The defect: the owner's assistant installed the `superpowers` Hermes plugin + * and PROVED it works — in a fresh `hermes chat -q` process. The chat the owner + * was actually looking at is served by `clawbox-hermes-dashboard.service`, a + * process that had been up since before the install, and Hermes scans for + * plugins exactly once per process (`discover_plugins(force=True)` at start, + * `_ensure_plugins_discovered()` returning early ever after). So every surface + * that reads `~/.hermes` said the plugin was there, and the one process whose + * answer mattered had never seen it. + * + * A reader that says "installed" over that box is the false-success shape: it + * reports the outcome from the fact that a file exists, not from the process + * that has to have read it. + */ + +const execFileMock = vi.hoisted(() => vi.fn()); +vi.mock("child_process", async (importOriginal) => ({ + ...(await importOriginal()), + execFile: execFileMock, +})); + +/** + * HERMES' OWN ANSWER to "what did the running process load", on the socket + * ClawBox already dials. `plugins.list` is a method on the pinned Hermes + * (`tui_gateway/methods_tools.py`), built from `get_plugin_manager()._plugins` + * — the process's own registry rather than a log of what it once printed. + * `null` is a box whose dashboard could not be reached. + */ +const rpcMock = vi.hoisted(() => vi.fn()); +vi.mock("@/lib/hermes-dashboard-rpc", () => ({ dashboardRpc: rpcMock })); + +import { + _resetHermesPluginNameMemoForTests, + hermesPluginsBlock, + readHermesPluginDeclaration, + readHermesPluginState, +} from "@/lib/hermes-plugin-set"; + +let home: string; +let restoreEnv: () => void; + +/** `~/.hermes` as a box carries it: the install ledger plus config.yaml. */ +function writeHermesHome(opts: { + installed?: Record; + configYaml?: string; +}): void { + const plugins = path.join(home, ".hermes", "plugins"); + fs.mkdirSync(plugins, { recursive: true }); + if (opts.installed !== undefined) { + fs.writeFileSync( + path.join(plugins, ".install-metadata.json"), + JSON.stringify(opts.installed, null, 2), + ); + } + if (opts.configYaml !== undefined) { + fs.writeFileSync(path.join(home, ".hermes", "config.yaml"), opts.configYaml); + } +} + +/** The shape `hermes plugins install` leaves behind, narrowed to what we read. */ +function installRecord(name: string) { + return { [name]: { source: "git", installed_at: "2026-09-18T12:59:00Z" } }; +} + +const CONFIG_WITH = (names: string[], rest = "") => `providers: + clawai: + api_key: redacted +plugins: + enabled: +${names.map((n) => ` - ${n}`).join("\n")} +${rest}agents: + defaults: + model: claude-opus-5 +`; + +/** + * THE OTHER WRITER ON THE BOX, and the layout that made the reader answer + * nothing at all. + * + * Hermes' own dumper forces `indentless=False`, so its list items sit two + * columns further in than `enabled:` — that is {@link CONFIG_WITH}, and it is + * what every fixture used to assume. `scripts/register-mcp.sh` re-serialises + * the WHOLE file with plain `yaml.safe_dump`, and PyYAML's default in a mapping + * context is an INDENTLESS block sequence: `enabled:` at column 2 and `- clawai` + * at column 2 as well. Reproduced with the shipped flags (PyYAML 6.0.1): + * + * plugins: + * enabled: + * - clawai + * + * Every box that script has ever had something to change carries this layout — + * which is every box that got the EMAIL-directive hook. + */ +const CONFIG_INDENTLESS = (names: string[], rest = "") => `providers: + clawai: + api_key: redacted +plugins: + enabled: +${names.map((n) => ` - ${n}`).join("\n")} +${rest}agents: + defaults: + model: claude-opus-5 +`; + +/** + * The two questions ClawBox puts to the running dashboard, answered apart. + * + * `plugins.list` is the RUNNING registry — what this process actually holds, + * keyed by registry key. `plugins.manage {action:"list"}` is Hermes' own plugin + * DISCOVERY, and the only place the name↔key map exists: `manifest_key()` is + * `manifest.key or manifest.name`, so a nested plugin's key says nothing about + * the name a person enables it under. `manifests: null` is a box whose Hermes + * has no such method — which must read as "could not establish", never as "no". + */ +function registryAnswers(opts: { + running: { name: string; enabled?: boolean }[] | null; + manifests?: { key: string; name: string }[] | null; +}): void { + rpcMock.mockImplementation(async (method: string) => { + if (method === "plugins.list") return opts.running ? { plugins: opts.running } : null; + if (method === "plugins.manage") return opts.manifests ? { plugins: opts.manifests } : null; + return null; + }); +} + +/** + * What `systemctl show` and `journalctl` answer. Both are read through + * `execFile`, so one mock serves both; anything unasked-for answers empty, + * which is the "cannot be asked" every reader here has to survive. + */ +function systemAnswers(answers: { show?: string; journal?: string }): void { + execFileMock.mockImplementation((bin: string, _args: string[], _opts: unknown, cb: unknown) => { + const done = typeof _opts === "function" ? _opts : cb; + const stdout = bin.includes("systemctl") ? (answers.show ?? "") : (answers.journal ?? ""); + (done as (e: null, r: { stdout: string; stderr: string }) => void)(null, { stdout, stderr: "" }); + return undefined as never; + }); +} + +beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "clawbox-hermes-plugins-")); + restoreEnv = saveEnv("HOME", "HERMES_HOME", "CLAWBOX_EDITION"); + process.env.HOME = home; + delete process.env.HERMES_HOME; + process.env.CLAWBOX_EDITION = "hermes"; + execFileMock.mockReset(); + // The name↔key memo lives in the process store, so it is shared between the + // cases in this file until it is put back. + _resetHermesPluginNameMemoForTests(); + rpcMock.mockReset().mockResolvedValue(null); + systemAnswers({}); +}); + +afterEach(() => { + restoreEnv(); + fs.rmSync(home, { recursive: true, force: true }); +}); + +describe("hermesPluginsBlock", () => { + it("takes the plugins: block and nothing else", () => { + // THE WHOLE POINT of extracting a block rather than hashing config.yaml. + // That file is rewritten by every Settings save on the box — a provider + // key, a model change, a voice toggle — and a watcher keyed on the whole + // file would bounce the box's chat backend on each one. + const block = hermesPluginsBlock(CONFIG_WITH(["superpowers"])); + expect(block).toContain("superpowers"); + expect(block).not.toContain("clawai"); + expect(block).not.toContain("claude-opus-5"); + }); + + it("is stable when an unrelated key changes", () => { + const before = hermesPluginsBlock(CONFIG_WITH(["superpowers"])); + const after = hermesPluginsBlock( + CONFIG_WITH(["superpowers"]).replace("claude-opus-5", "claude-fable-5"), + ); + expect(after).toBe(before); + }); + + it("keeps the disabled deny-list, which decides loading just as much", () => { + // `plugins.disabled` wins over `plugins.enabled` in Hermes' own + // `_plugin_status`, so a box where the owner disabled a plugin has changed + // its plugin set as surely as one that installed another. + const block = hermesPluginsBlock(`plugins: + enabled: + - superpowers + disabled: + - noisy +`); + expect(block).toContain("disabled"); + expect(block).toContain("noisy"); + }); + + it("is not ended by a comment at column zero INSIDE it", () => { + // YAML has no rule that a `#` closes a mapping, and `hermes config` and a + // person editing by hand both leave them. Ending the extraction there cut + // `disabled:` — the deny-list `_plugin_status` gives precedence to — out of + // the signature entirely, so disabling a plugin changed nothing the watcher + // could see and the box went on serving it. + const yaml = [ + "plugins:", + " enabled:", + " - superpowers", + "# the owner's own note about why the next one is off", + " disabled:", + " - noisy-plugin", + "agents:", + " defaults:", + " model: claude-opus-5", + ].join("\n"); + const block = hermesPluginsBlock(yaml); + expect(block).toContain("disabled"); + expect(block).toContain("noisy-plugin"); + expect(block).not.toContain("agents"); + expect(block).not.toContain("claude-opus-5"); + }); + + it("does not take a comment that FOLLOWS the block into it", () => { + // The other direction: a comment sitting between the block and the next + // top-level key belongs to what comes after, and swallowing it would make + // an edit to that comment read as a plugin change. + const withNote = hermesPluginsBlock([ + "plugins:", + " enabled:", + " - superpowers", + "# a note about the agents block below", + "agents:", + " defaults:", + " model: claude-opus-5", + ].join("\n")); + const without = hermesPluginsBlock([ + "plugins:", + " enabled:", + " - superpowers", + "agents:", + " defaults:", + " model: claude-opus-5", + ].join("\n")); + expect(withNote).toBe(without); + }); + + it("answers empty for a config with no plugins block", () => { + expect(hermesPluginsBlock("agents:\n defaults:\n model: x\n")).toBe(""); + }); + + it("does not mistake a nested plugins: key for the top-level one", () => { + // `dashboard.hidden_plugins` and `plugins.entries.` both put the word + // further in. Only column zero is the block this watches. + const block = hermesPluginsBlock(`dashboard: + plugins: + - not-this-one +agents: + defaults: + model: x +`); + expect(block).toBe(""); + }); +}); + +describe("readHermesPluginDeclaration", () => { + it("names the plugins ~/.hermes declares", async () => { + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]), + }); + const declared = await readHermesPluginDeclaration(); + expect(declared.names).toEqual(["superpowers"]); + expect(declared.signature).toMatch(/^[0-9a-f]{16,}$/); + }); + + it("gives an identical signature to a byte-identical rewrite", async () => { + // A HASH, NOT AN MTIME, and this is the case that forces it: the dashboard's + // own ExecStartPre re-provisions auth and rewrites config.yaml on every + // start, so an mtime watcher would see a change the moment it restarted — + // and restart again, for ever. + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]), + }); + const first = await readHermesPluginDeclaration(); + await new Promise((r) => setTimeout(r, 10)); + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]), + }); + expect((await readHermesPluginDeclaration()).signature).toBe(first.signature); + }); + + it("changes the signature when a plugin is enabled", async () => { + writeHermesHome({ installed: {}, configYaml: CONFIG_WITH([]) }); + const before = await readHermesPluginDeclaration(); + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]), + }); + const after = await readHermesPluginDeclaration(); + expect(after.signature).not.toBe(before.signature); + expect(after.names).toContain("superpowers"); + }); + + it("leaves a plugin's OWN `enabled:` list out of the declaration", async () => { + // `plugins.entries.` is a plugin's own settings, and the block + // extraction keeps it deliberately out of the signature: a preference a + // person changes is not a plugin set that changed. The key match ran at ANY + // depth inside the block, though, so a plugin whose settings schema has a + // LIST-valued key literally named `enabled` (or `disabled`) put its members + // into the box's declaration — a ghost name no registry can ever resolve, + // which is a permanent `stale`/`null`, and a signature that moves when a + // preference does. The key belongs to `plugins:` only at the block's own + // first indent level. + writeHermesHome({ + installed: installRecord("clawai"), + configYaml: `plugins: + enabled: + - clawai + entries: + thing: + enabled: + - ghost + disabled: + - phantom +agents: + defaults: + model: claude-opus-5 +`, + }); + const declared = await readHermesPluginDeclaration(); + expect(declared.enabled).toEqual(["clawai"]); + expect(declared.names).toEqual(["clawai"]); + }); + + it("still reads a scalar `enabled:` under a plugin's entry as nothing", async () => { + // The shape the box actually carries (`enabled: true`), which has always + // yielded no names and must go on doing so. + writeHermesHome({ + installed: installRecord("clawai"), + configYaml: `plugins: + enabled: + - clawai + entries: + thing: + enabled: true +agents: + defaults: + model: claude-opus-5 +`, + }); + expect((await readHermesPluginDeclaration()).enabled).toEqual(["clawai"]); + }); + + it("does not change the signature when only an unrelated setting changes", async () => { + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]), + }); + const before = await readHermesPluginDeclaration(); + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]).replace("claude-opus-5", "claude-fable-5"), + }); + expect((await readHermesPluginDeclaration()).signature).toBe(before.signature); + }); + + it("reads the enabled names out of PyYAML's INDENTLESS block sequence", async () => { + // `scripts/register-mcp.sh` re-serialises config.yaml with plain + // `yaml.safe_dump`, whose block sequences sit at the KEY'S indent rather + // than deeper. The old rule accepted an item only when it was indented + // FURTHER (`item[1].length > listIndent`), so on every box that script had + // written, `plugins.enabled` read as EMPTY — and `stale`, which is + // `enabled.some(...)`, was permanently false. The MCP tool then told the + // owner "the agent now serving chat has read the current plugin set" about + // a plugin that had never loaded: the exact false success this module's + // null-handling exists to prevent, arriving through the other reader. + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_INDENTLESS(["clawai", "clawbox_email_directives", "superpowers"]), + }); + const declared = await readHermesPluginDeclaration(); + expect(declared.enabled).toEqual(["clawai", "clawbox_email_directives", "superpowers"]); + }); + + it("gives ONE signature to the two layouts, so a re-serialisation is not a plugin change", async () => { + // THE RESTART THAT NOTHING ASKED FOR. The signature used to hash the raw + // `plugins:` text, and `register-mcp.sh` rewrites the whole file through + // `yaml.safe_dump` whenever anything at all changed — spawned at every + // web-server boot from `production-server.js` and again from the ClawBox-MCP + // toggle in Settings. Its write normalises the block's indentation, so the + // watcher saw "a signature I have not seen", bounced the dashboard, and the + // owner's chat window closed with "The assistant restarted to load the + // plugin …" over a plugin set that had not moved by one name. + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["clawai", "superpowers"]), + }); + const hermesLayout = await readHermesPluginDeclaration(); + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_INDENTLESS(["clawai", "superpowers"]), + }); + expect((await readHermesPluginDeclaration()).signature).toBe(hermesLayout.signature); + }); + + it("still changes the signature when the re-serialised set is DIFFERENT", async () => { + // The other half of the rule above: hashing the parsed set must not make a + // real change invisible, which would leave a plugin unloaded for ever with + // the box reporting no work outstanding. + writeHermesHome({ installed: {}, configYaml: CONFIG_WITH(["clawai"]) }); + const before = await readHermesPluginDeclaration(); + writeHermesHome({ installed: {}, configYaml: CONFIG_INDENTLESS(["clawai", "superpowers"]) }); + expect((await readHermesPluginDeclaration()).signature).not.toBe(before.signature); + }); + + it("changes the signature when an installed plugin is updated in place", async () => { + // `hermes plugins update` pulls the plugin's git checkout and records the + // new HEAD in `.install-metadata.json` — the plugin NAME does not move but + // the code does, and the running process is holding the old one. + const plugins = path.join(home, ".hermes", "plugins"); + fs.mkdirSync(plugins, { recursive: true }); + const write = (revision: string) => + fs.writeFileSync( + path.join(plugins, ".install-metadata.json"), + JSON.stringify({ superpowers: { source: "git", revision } }), + ); + write("a".repeat(40)); + const before = await readHermesPluginDeclaration(); + write("b".repeat(40)); + expect((await readHermesPluginDeclaration()).signature).not.toBe(before.signature); + }); + + it("does not take a trailing YAML comment for part of the plugin name", async () => { + // `- superpowers # installed 2026-09-18` is ordinary YAML that a person + // writes by hand. Read as the NAME "superpowers # installed 2026-09-18" it + // matches no registry key ever, so `stale` is true for good — the MCP tool + // warns "the plugin is NOT loaded yet" on a box that is serving it, and the + // next config write bounces the owner's chat for a restart that cannot + // change the answer. + writeHermesHome({ + installed: {}, + configYaml: `plugins: + enabled: + - superpowers # installed 2026-09-18 + - "weird # name" +`, + }); + expect((await readHermesPluginDeclaration()).enabled).toEqual(["superpowers", "weird # name"]); + }); + + it("reads a FLOW list, with or without a comment after it", async () => { + // `hermes config set plugins.enabled '["a","b"]'` writes this shape. The + // old expression required the `]` to be the last thing on the line, so a + // comment after it dropped the whole list — and a genuinely stale plugin + // went unreported. + writeHermesHome({ + installed: {}, + configYaml: `plugins: + enabled: [superpowers, "clawai"] # owner note +`, + }); + expect((await readHermesPluginDeclaration()).enabled).toEqual(["clawai", "superpowers"]); + }); + + it("reads a box with no plugins at all without throwing", async () => { + const declared = await readHermesPluginDeclaration(); + expect(declared.names).toEqual([]); + expect(typeof declared.signature).toBe("string"); + }); + + it("survives a half-written install ledger", async () => { + // `hermes plugins install` writes this file; a watcher polling every few + // seconds WILL catch it mid-write, and an exception there would take the + // poll loop down for the life of the web server. + const plugins = path.join(home, ".hermes", "plugins"); + fs.mkdirSync(plugins, { recursive: true }); + fs.writeFileSync(path.join(plugins, ".install-metadata.json"), '{"superpowers":'); + const declared = await readHermesPluginDeclaration(); + expect(Array.isArray(declared.names)).toBe(true); + }); +}); + +describe("readHermesPluginState", () => { + it("separates what is declared from what the running dashboard loaded", async () => { + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]), + }); + systemAnswers({ + show: "InvocationID=abc123def456\nExecMainStartTimestampMonotonic=1000000\n", + // The journal of a dashboard that started BEFORE the install: it + // registered the bundled auth plugin and nothing else. + journal: "Plugin 'basic' registered dashboard-auth provider: basic (password)\n", + }); + const state = await readHermesPluginState(); + expect(state.declared).toContain("superpowers"); + expect(state.loaded).not.toContain("superpowers"); + }); + + it("reports a plugin the running dashboard did register", async () => { + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["superpowers"]), + }); + systemAnswers({ + show: "InvocationID=abc123def456\nExecMainStartTimestampMonotonic=1000000\n", + journal: "Plugin 'superpowers' registered tool: brainstorm\n", + }); + expect((await readHermesPluginState()).loaded).toContain("superpowers"); + }); + + it("answers null — not [] — for a dashboard whose journal carries no registration lines", async () => { + // MEASURED ON THE OWNER'S BOX (2026-09-18). Its dashboard's whole journal + // for the current invocation is 187 lines of `sessions.changed` events and + // the readiness banner, and not ONE `Plugin … registered` line — Hermes logs + // those on its Python logger and this process does not route them out. + // + // A successful read that matched nothing is therefore NOT "no plugin + // loaded": Hermes always registers the bundled `basic` dashboard-auth plugin + // on a gated bind, so zero registration lines can only mean this box does not + // publish them. Answering [] would have the route tell the owner that every + // plugin on their working device is missing — the false-failure shape, on the + // exact box this feature was built for. + systemAnswers({ + show: "InvocationID=abc123def456\nExecMainStartTimestampMonotonic=1000000\n", + journal: "HERMES_DASHBOARD_READY port=9119\n{\"jsonrpc\": \"2.0\", \"method\": \"event\"}\n", + }); + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + const state = await readHermesPluginState(); + expect(state.loaded).toBeNull(); + // …and the DECLARED half still answers, so the caller is not left with nothing. + expect(state.declared).toContain("superpowers"); + }); + + it("asks the HARNESS first, and never opens the journal when it answers", async () => { + // Leverage the harness first. The journal scrape was a re-implementation of + // a question Hermes answers natively, and the re-implementation did not + // work on the box it was built for — the dashboard publishes no + // registration lines at all there, so `loaded` was permanently null. + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + systemAnswers({ show: "InvocationID=abc123def456\nExecMainStartTimestampMonotonic=1000000\n", journal: "" }); + rpcMock.mockResolvedValue({ + plugins: [ + { name: "superpowers", version: "1.2.0", enabled: true }, + { name: "basic", version: "?", enabled: true }, + ], + }); + + const state = await readHermesPluginState(); + expect(rpcMock).toHaveBeenCalledWith("plugins.list", {}, expect.anything()); + expect(state.loaded).toEqual(["basic", "superpowers"]); + // …and it is the registry, so `stale` can finally be a fact about the + // RUNNING process rather than a guess from a file's mtime. + expect(state.stale).toBe(false); + expect(execFileMock.mock.calls.some((call) => String(call[0]).includes("journalctl"))).toBe(false); + }); + + it("does not count a plugin the running process has DISABLED as loaded", async () => { + // `_plugins` holds every manifest discovery found, bundled ones included, + // each with the verdict the config gave it. "Loaded" to the person asking + // means the tools are in their chat. + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + registryAnswers({ + running: [{ name: "superpowers", enabled: false }], + manifests: [{ key: "superpowers", name: "superpowers" }], + }); + const state = await readHermesPluginState(); + // An answer with rows and none of them on is the registry saying "no", not + // a box that could not be asked — so it is `[]` and the staleness that + // follows is a fact. + expect(state.loaded).toEqual([]); + expect(state.stale).toBe(true); + }); + + it("falls back to the journal when the dashboard cannot be asked", async () => { + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + rpcMock.mockResolvedValue(null); + systemAnswers({ + show: "InvocationID=abc123def456\nExecMainStartTimestampMonotonic=1000000\n", + journal: "Plugin 'superpowers' registered tool: brainstorm\n", + }); + expect((await readHermesPluginState()).loaded).toEqual(["superpowers"]); + }); + + it("says the process is BEHIND THE FILES when what the box declares is not in the registry", async () => { + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + registryAnswers({ + running: [{ name: "basic", enabled: true }], + manifests: [{ key: "basic", name: "basic" }], + }); + const state = await readHermesPluginState(); + expect(state.stale).toBe(true); + }); + + it("matches a declared name the way HERMES does — the whole key or the manifest name", async () => { + // MEASURED ON THE OWNER'S BOX (2026-09-18). `plugins.list` answers the + // registry key — `image_gen/clawai`, `superpowers/.hermes-plugin`, + // `dashboard_auth/basic` — while `plugins.enabled` holds the name a person + // wrote. Hermes' own matcher is `names = {manifest name, registry key}` + // (`hermes_cli/plugins_cmd.py:_plugin_status`, and `manifest_key()` is + // `manifest.key or manifest.name`), so the name↔key map is the missing + // half — it comes from `plugins.manage {action:"list"}`, which carries both. + writeHermesHome({ + installed: installRecord("superpowers"), + configYaml: CONFIG_WITH(["clawai", "clawbox_email_directives", "superpowers"]), + }); + registryAnswers({ + running: [ + { name: "image_gen/clawai", enabled: true }, + { name: "superpowers/.hermes-plugin", enabled: true }, + { name: "clawbox_email_directives", enabled: true }, + { name: "dashboard_auth/basic", enabled: true }, + ], + manifests: [ + { key: "image_gen/clawai", name: "clawai" }, + { key: "superpowers/.hermes-plugin", name: "superpowers" }, + { key: "clawbox_email_directives", name: "clawbox_email_directives" }, + { key: "dashboard_auth/basic", name: "basic" }, + ], + }); + expect((await readHermesPluginState()).stale).toBe(false); + }); + + it("matches a manifest name a PATH SEGMENT never could", async () => { + // The case the segment rule could not reach and reported as behind for + // ever: a plugin installed into a directory whose name differs from its + // manifest `name`, enabled under the manifest name. The key carries + // neither, so the segment rule answered "missing" on a box that was serving + // it — the MCP tool's "the plugin is NOT loaded yet" over a working device, + // and a chat restart at the next web-server boot that could not fix it. + writeHermesHome({ installed: {}, configYaml: CONFIG_WITH(["weather"]) }); + registryAnswers({ + running: [{ name: "community/wx-tools", enabled: true }], + manifests: [{ key: "community/wx-tools", name: "weather" }], + }); + expect((await readHermesPluginState()).stale).toBe(false); + }); + + it("does not call a plugin loaded because its name is a PATH SEGMENT of some other key", async () => { + // Looser than Hermes in the other direction: `clawai` is not loaded just + // because some unrelated `image_gen/clawai` key exists whose manifest is + // named something else. Reporting that as up to date is the false success + // on the reader the MCP tool quotes to the owner. + writeHermesHome({ installed: {}, configYaml: CONFIG_WITH(["clawai"]) }); + registryAnswers({ + running: [{ name: "image_gen/clawai", enabled: true }], + manifests: [{ key: "image_gen/clawai", name: "openai_images" }], + }); + expect((await readHermesPluginState()).stale).toBe(true); + }); + + it("asks for the name↔key map once and reuses it, but never remembers a box it could not ask", async () => { + // `plugins.manage {action:"list"}` re-discovers every manifest from disk + // and consults the catalogue, which is why this file budgets it at 15 s; + // the deny-list branch made it ordinary rather than rare, so it is memoed. + // Only a SUCCESSFUL map: remembering "could not be asked" would turn a + // transient failure into a stale `null` verdict for the whole window. + writeHermesHome({ installed: {}, configYaml: CONFIG_WITH(["weather"]) }); + registryAnswers({ running: [{ name: "community/wx-tools", enabled: true }], manifests: null }); + expect((await readHermesPluginState()).stale).toBeNull(); + const askedWhileUnanswered = rpcMock.mock.calls.filter((c) => c[0] === "plugins.manage").length; + expect((await readHermesPluginState()).stale).toBeNull(); + expect(rpcMock.mock.calls.filter((c) => c[0] === "plugins.manage").length) + .toBeGreaterThan(askedWhileUnanswered); + + registryAnswers({ + running: [{ name: "community/wx-tools", enabled: true }], + manifests: [{ key: "community/wx-tools", name: "weather" }], + }); + expect((await readHermesPluginState()).stale).toBe(false); + const askedOnce = rpcMock.mock.calls.filter((c) => c[0] === "plugins.manage").length; + expect((await readHermesPluginState()).stale).toBe(false); + expect(rpcMock.mock.calls.filter((c) => c[0] === "plugins.manage").length).toBe(askedOnce); + }); + + it("answers NULL, never `true`, when the name↔key map could not be read", async () => { + // A declared name that is not itself a registry key can only be resolved + // through the manifest, and an older Hermes has no `plugins.manage`. "We + // could not establish it" is the honest answer; `true` there would be a + // permanent "behind the files" that re-arms a chat restart at every + // web-server boot over a box nobody can prove anything about. + writeHermesHome({ installed: {}, configYaml: CONFIG_WITH(["superpowers"]) }); + registryAnswers({ running: [{ name: "superpowers/.hermes-plugin", enabled: true }], manifests: null }); + expect((await readHermesPluginState()).stale).toBeNull(); + }); + + it("does not call a plugin the box's own deny-list has switched off BEHIND the files", async () => { + // MEASURED ON THE OWNER'S BOX (2026-09-18) and confirmed against Hermes' + // `cmd_disable`: `hermes plugins disable superpowers` discards the KEY and + // its leaf from `plugins.enabled` and adds the key to `plugins.disabled` — + // the bare-name entry a person wrote stays behind. `_plugin_status` gives + // the deny-list precedence, so the plugin is OFF and the running registry is + // right not to have it. Reading `enabled` alone made `stale` true for good + // on a box whose owner had deliberately switched a plugin off: the MCP tool + // warned "the agent is still behind the files", and with `changedAfterStart` + // re-armed by any Settings save the watcher's first look would bounce the + // owner's chat at the next web-server boot, for a restart that cannot change + // the answer. + writeHermesHome({ + installed: {}, + configYaml: `plugins: + enabled: + - clawai + - superpowers + disabled: + - superpowers/.hermes-plugin +`, + }); + registryAnswers({ + running: [{ name: "image_gen/clawai", enabled: true }], + manifests: [ + { key: "image_gen/clawai", name: "clawai" }, + { key: "superpowers/.hermes-plugin", name: "superpowers" }, + ], + }); + expect((await readHermesPluginState()).stale).toBe(false); + }); + + it("never asks for the name↔key map when every declared name IS a registry key", async () => { + // `plugins.manage list` re-discovers from disk and consults the live plugin + // catalogue over the network; it is not a price the common box should pay. + writeHermesHome({ installed: {}, configYaml: CONFIG_WITH(["superpowers"]) }); + registryAnswers({ running: [{ name: "superpowers", enabled: true }], manifests: [] }); + expect((await readHermesPluginState()).stale).toBe(false); + expect(rpcMock.mock.calls.some((call) => call[0] === "plugins.manage")).toBe(false); + }); + + it("reports whether the files were touched after the dashboard started, apart from staleness", async () => { + // The self-limiting half of the watcher's seed rule, and the reason it is a + // separate field: it is an mtime, so a Settings save makes it true — but + // once the dashboard has restarted, its start is newer than those files, so + // a box whose declaration and registry can never agree cannot be bounced + // twice for the same reason. + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + rpcMock.mockResolvedValue({ plugins: [{ name: "superpowers", enabled: true }] }); + // A dashboard that started long before this boot. + systemAnswers({ show: "InvocationID=abc123def456\nExecMainStartTimestampMonotonic=1000\n" }); + expect((await readHermesPluginState()).changedAfterStart).toBe(true); + }); + + it("does not call a box stale because an unrelated Settings save rewrote config.yaml", async () => { + // THE FALSE POSITIVE `stale` USED TO CARRY. It was + // `max(mtime(ledger), mtime(config.yaml)) > dashboardStartedAt`, and + // config.yaml is rewritten by every Settings save on this box and by the + // dashboard's own ExecStartPre. An owner changing the assistant's model at + // 14:00 made the MCP tool warn "the agent is still behind the files — the + // plugin is NOT loaded yet" about a plugin that was loaded at 10:52. + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + rpcMock.mockResolvedValue({ plugins: [{ name: "superpowers", enabled: true }] }); + // A save well after the dashboard started, touching nothing about plugins. + writeHermesHome({ configYaml: CONFIG_WITH(["superpowers"], "") }); + fs.utimesSync(path.join(home, ".hermes", "config.yaml"), new Date(), new Date()); + systemAnswers({ show: "InvocationID=abc123def456\nExecMainStartTimestampMonotonic=1000\n" }); + expect((await readHermesPluginState()).stale).toBe(false); + }); + + it("answers null for `stale` when neither the registry nor the journal can be read", async () => { + // Never a guess in either direction: "could not be established" is a real + // answer and the route and the MCP tool both word it as one. + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + rpcMock.mockResolvedValue(null); + systemAnswers({ show: "", journal: "" }); + const state = await readHermesPluginState(); + expect(state.loaded).toBeNull(); + expect(state.stale).toBeNull(); + }); + + it("answers null for `loaded` when the journal cannot be read", async () => { + // A BOX THAT CANNOT BE ASKED IS NOT A BOX WITH NO PLUGINS. An empty array + // here would let the route report "superpowers is not loaded" over a + // dashboard that had loaded it perfectly well and merely logs at a level + // this cannot see — the false-failure shape, on the one reader whose job is + // to be believed. + execFileMock.mockImplementation((_b: string, _a: string[], _o: unknown, cb: unknown) => { + const done = typeof _o === "function" ? _o : cb; + (done as (e: Error) => void)(new Error("no journalctl")); + return undefined as never; + }); + writeHermesHome({ installed: installRecord("superpowers"), configYaml: CONFIG_WITH(["superpowers"]) }); + expect((await readHermesPluginState()).loaded).toBeNull(); + }); +});