From aba949367b5978fffa9f4ab080cf3b5f8650d1ef Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 13:43:51 -0600 Subject: [PATCH 1/8] Replace local SLM embeddings with scope.models.embed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the bundled bge-small-en-v1.5 model and the harper-fabric-embeddings + node-llama-cpp dependency chain. Embeddings now run through Harper's new scope.models API (harper#510), which dispatches to whatever embedding backend the host has configured (Ollama on GPU hosts, Anthropic via OpenAI gateway, etc.) — zero model lifecycle to manage in the app. - lib/modelCapture.js: tiny Plugin API hook that stashes Scope on globalThis so Resource classes can reach scope.models. Wired up via `extensionModule:` in config.yaml. - lib/embeddings.js: now a one-line wrapper around scope.models.embed(). - package.json: bumps engines.harperdb to ^5.1, drops the SLM-only deps. - scripts/download-model.js and models/ removed. Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 1 - config.yaml | 4 + lib/embeddings.js | 22 ++-- lib/modelCapture.js | 11 ++ package-lock.json | 204 +------------------------------------- package.json | 11 +- resources/Chat.js | 4 +- scripts/download-model.js | 45 --------- 8 files changed, 30 insertions(+), 272 deletions(-) create mode 100644 lib/modelCapture.js delete mode 100644 scripts/download-model.js diff --git a/.gitignore b/.gitignore index ca8a006..cb565e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ node_modules/ -models/ .env CONFIG.env *.env diff --git a/config.yaml b/config.yaml index d6be5d1..861fdcc 100644 --- a/config.yaml +++ b/config.yaml @@ -6,6 +6,10 @@ loadEnv: rest: true +# Captures the Scope object so resources can call scope.models.embed(). +# See lib/modelCapture.js. +extensionModule: 'lib/modelCapture.js' + graphqlSchema: files: 'schemas/*.graphql' diff --git a/lib/embeddings.js b/lib/embeddings.js index 0b3102e..bddf169 100644 --- a/lib/embeddings.js +++ b/lib/embeddings.js @@ -1,14 +1,14 @@ -import { init, embed as llamaEmbed } from 'harper-fabric-embeddings' -import { resolve } from 'path' -import { fileURLToPath } from 'url' - -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelPath = resolve(__dirname, '../models/bge-small-en-v1.5-q4_k_m.gguf') - -// Model is pre-downloaded by the predev/prestart npm hook (scripts/download-model.js) -const initPromise = init({ modelPath }) +// Thin wrapper around `scope.models.embed()` (harper#510). The host's +// configured backend (Ollama on Fabric GPU hosts, or any backend configured +// via the `models:` block in harperdb-config.yaml / env vars) handles the +// actual inference. Returns a plain Array for compatibility with +// Harper's HNSW vector index storage. export async function embed(text) { - await initPromise - return llamaEmbed(text) + const scope = globalThis.harperScope; + if (!scope) { + throw new Error('Harper scope not yet captured — modelCapture plugin must run before first embed call'); + } + const [vector] = await scope.models.embed(text); + return Array.from(vector); } diff --git a/lib/modelCapture.js b/lib/modelCapture.js new file mode 100644 index 0000000..018348e --- /dev/null +++ b/lib/modelCapture.js @@ -0,0 +1,11 @@ +// Plugin entry — captures the Scope object so `resources/*.js` can call +// `scope.models.embed()` against the host's configured embedding backend. +// +// Harper's `scope` is passed to plugins via `handleApplication(scope)` but +// isn't exposed as a global to Resource classes. This tiny plugin stashes the +// Scope on `globalThis.harperScope` at app boot, then `lib/embeddings.js` +// reads it from there. + +export function handleApplication(scope) { + globalThis.harperScope = scope; +} diff --git a/package-lock.json b/package-lock.json index 1142a3b..48a64ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,7 @@ "@anthropic-ai/sdk": "^0.39.0", "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", @@ -20,11 +19,6 @@ }, "engines": { "harper": "^5.0" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" } }, "node_modules/@agoric/babel-generator": { @@ -1987,131 +1981,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/@node-llama-cpp/linux-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-arm64/-/linux-arm64-3.18.1.tgz", - "integrity": "sha512-rXMgZxUay78FOJV/fJ67apYP9eElH5jd4df5YRKPlLhLHHchuOSyDn+qtyW/L/EnPzpogoLkmULqCkdXU39XsQ==", - "cpu": [ - "arm64", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-armv7l": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-armv7l/-/linux-armv7l-3.18.1.tgz", - "integrity": "sha512-BrJL2cGo0pN5xd5nw+CzTn2rFMpz9MJyZZPUY81ptGkF2uIuXT2hdCVh56i9ImQrTwBfq1YcZL/l/Qe/1+HR/Q==", - "cpu": [ - "arm", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.17.1.tgz", - "integrity": "sha512-/o/UoqAdslg4ExdKYyYPqbw+21Dr4cQ2JgouXg8Ji3opRKoTMrlUNfrMwIsYZfbDDJ8l7xFnwfGIwdlQ5RPwJg==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.17.1.tgz", - "integrity": "sha512-oRq6/7qCMsazO2Cw0oCyiILZmMvejKJgLAIG60E00WOZWhpJGjh71JGnOybRycKA015mFPNDHzT3SDdUZtZBew==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.17.1.tgz", - "integrity": "sha512-3L0nFVi70j+Qk7Xb8p/RQVMU0E28G0xXX0YL6Vzkirq3DazPYhWOLWUUs9MtGW2FrBg/6PLqyddmxwBfCpjm3w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-arm64/-/win-arm64-3.18.1.tgz", - "integrity": "sha512-S05YUzBMVSRS5KNbOS26cDYugeQHqogI3uewtTUBVC0tPbTHRSKjsdicmgWru1eNAry399LWWhzOf/3St/qsAw==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-x64/-/win-x64-3.18.1.tgz", - "integrity": "sha512-QLDVphPl+YDI+x/VYYgIV1N9g0GMXk3PqcoopOUG3cBRUtce7FO+YX903YdRJezs4oKbIp8YaO+xYBgeUSqhpA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -5558,77 +5427,6 @@ } } }, - "node_modules/harper-fabric-embeddings": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/harper-fabric-embeddings/-/harper-fabric-embeddings-0.2.3.tgz", - "integrity": "sha512-25F1xzRTJ+19NlDiMI0RLF47u4Fwd5Ve/V03q06kJjCioqvEc2yrAASQ3NY4O+LJDU9rNq9WQivFeU+9JKt4IA==", - "hasInstallScript": true, - "license": "MIT", - "engines": { - "node": ">=22" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-arm64": "3.18.1", - "@node-llama-cpp/linux-armv7l": "3.18.1", - "@node-llama-cpp/linux-x64": "3.18.1", - "@node-llama-cpp/mac-arm64-metal": "3.18.1", - "@node-llama-cpp/mac-x64": "3.18.1", - "@node-llama-cpp/win-arm64": "3.18.1", - "@node-llama-cpp/win-x64": "3.18.1" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/linux-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.18.1.tgz", - "integrity": "sha512-tRmWcsyvAcqJHQHXHsaOkx6muGbcirA9nRdNgH6n7bjGUw4VuoBD3dChyNF3/Ktt7ohB9kz+XhhyZjbDHpXyMA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.18.1.tgz", - "integrity": "sha512-cyZTdsUMlvuRlGmkkoBbN3v/DT6NuruEqoQYd9CqIrPyLa1xLNBTSKIZ9SgRnw23iCOj4URfITvRP+2pu63LuQ==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.18.1.tgz", - "integrity": "sha512-GfCPgdltaIpBhEnQ7WfsrRXrZO9r9pBtDUAQMXRuJwOPP5q7xKrQZUXI6J6mpc8tAG0//CTIuGn4hTKoD/8V8w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/harper/node_modules/uuid": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", diff --git a/package.json b/package.json index 2ea6531..102229b 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,6 @@ "harper": "^5.0" }, "scripts": { - "setup": "node scripts/download-model.js", - "predev": "node scripts/download-model.js", - "prestart": "node scripts/download-model.js", "start": "npx -y dotenv-cli -e .env -o -- harper run .", "dev": "npx -y dotenv-cli -e .env -o -- harper dev .", "login": "node login.js", @@ -20,16 +17,10 @@ "@anthropic-ai/sdk": "^0.39.0", "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", "@types/node": "^22.19.19" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" } } diff --git a/resources/Chat.js b/resources/Chat.js index 41ce403..f5f08eb 100644 --- a/resources/Chat.js +++ b/resources/Chat.js @@ -350,8 +350,8 @@ const HTML = /* html */ ` - Local SLM · bge-small-en-v1.5 - embeddings run in Harper · no API cost + Shared model · nomic-embed-text + scope.models.embed() · GPU on Fabric · no API cost diff --git a/scripts/download-model.js b/scripts/download-model.js deleted file mode 100644 index a8aeb63..0000000 --- a/scripts/download-model.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node -// Downloads the bge-small-en-v1.5 embedding model if not already present. -// Run automatically via the predev / prestart npm hooks. - -import { createWriteStream, existsSync, mkdirSync } from 'fs' -import { pipeline } from 'stream/promises' -import { resolve } from 'path' -import { fileURLToPath } from 'url' - -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelsDir = resolve(__dirname, '../models') -const modelPath = resolve(modelsDir, 'bge-small-en-v1.5-q4_k_m.gguf') -const MODEL_URL = - 'https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-q4_k_m.gguf' - -if (existsSync(modelPath)) { - console.log('✓ Embedding model already downloaded.') - process.exit(0) -} - -console.log('Downloading bge-small-en-v1.5 embedding model (~24 MB)...') -mkdirSync(modelsDir, { recursive: true }) - -const response = await fetch(MODEL_URL) -if (!response.ok) { - console.error(`Download failed: ${response.status} ${response.statusText}`) - process.exit(1) -} - -const total = Number(response.headers.get('content-length') || 0) -let downloaded = 0 - -const progress = new TransformStream({ - transform(chunk, controller) { - downloaded += chunk.byteLength - if (total) { - const pct = Math.round((downloaded / total) * 100) - process.stdout.write(`\r ${pct}% (${(downloaded / 1024 / 1024).toFixed(1)} MB)`) - } - controller.enqueue(chunk) - }, -}) - -await pipeline(response.body.pipeThrough(progress), createWriteStream(modelPath)) -console.log('\n✓ Model ready.') From a3f944ee1886ff5bc10f9edd9c3b70bc9a9c189b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 20:07:08 -0600 Subject: [PATCH 2/8] Switch Agent from Anthropic SDK to scope.models.generate() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the @anthropic-ai/sdk + @anthropic-ai/vertex-sdk direct calls with a single scope.models.generate() call. Routes to whatever backend the host has configured for models.generative.default — vLLM on Fabric GPU hosts, Anthropic/OpenAI/Ollama elsewhere. Removes: - @anthropic-ai/sdk + @anthropic-ai/vertex-sdk deps - lib/config.js (LLM_PROVIDER / ANTHROPIC_API_KEY / VERTEX_*) - Anthropic web search tool + pause_turn handling - Per-token cost calculation (local model = no $) - Multi-block response stitching (scope.models returns plain content) Embeddings already used scope.models.embed() (prior commit on this branch). The Stats table now tracks cacheHits only; totalSaved is held at zero. Co-Authored-By: Claude Sonnet 4.6 --- lib/config.js | 21 --- package-lock.json | 376 +-------------------------------------------- package.json | 4 +- resources/Agent.js | 121 ++++----------- 4 files changed, 31 insertions(+), 491 deletions(-) delete mode 100644 lib/config.js diff --git a/lib/config.js b/lib/config.js deleted file mode 100644 index d1e1885..0000000 --- a/lib/config.js +++ /dev/null @@ -1,21 +0,0 @@ -const required = (name) => { - const value = process.env[name] - if (!value) throw new Error(`Missing required env var: ${name}`) - return value -} - -const optional = (name, fallback) => process.env[name] ?? fallback - -export const config = { - // "anthropic" (direct API) or "vertex" (Google Cloud Vertex AI) - provider: () => optional('LLM_PROVIDER', 'anthropic'), - anthropic: { - apiKey: () => required('ANTHROPIC_API_KEY'), - model: () => optional('CLAUDE_MODEL', 'claude-sonnet-4-5-20250929'), - }, - vertex: { - projectId: () => required('VERTEX_PROJECT_ID'), - region: () => optional('VERTEX_REGION', 'global'), - model: () => optional('VERTEX_MODEL', 'claude-sonnet-4-6'), - }, -} diff --git a/package-lock.json b/package-lock.json index 48a64ee..6deb47b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,8 +8,6 @@ "name": "agent-example-harper", "version": "1.0.0", "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", "harper": "^5.2.1" }, @@ -35,67 +33,6 @@ "node": ">=6.9.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.39.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.39.0.tgz", - "integrity": "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==", - "license": "MIT", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/@anthropic-ai/vertex-sdk": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/vertex-sdk/-/vertex-sdk-0.15.0.tgz", - "integrity": "sha512-i2LDdu6VB8Lqqip+kbNSXRxQgFsCg6GPBO/X2zRJwLl99dNzf28nb6Rdi0EodONXsyJfY2TKdGR+y5l1/AKFEg==", - "license": "MIT", - "dependencies": { - "@anthropic-ai/sdk": ">=0.50.3 <1", - "google-auth-library": "^9.4.2" - } - }, - "node_modules/@anthropic-ai/vertex-sdk/node_modules/@anthropic-ai/sdk": { - "version": "0.116.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.116.0.tgz", - "integrity": "sha512-4UEapYQ+epLEMsAuLZDvW8ExVSOtHD8a7zTyLzhw0H9RXJ1eilPgmqhjwgcdg22diwx13spw6fJ4rONZ+bS7Ww==", - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, "node_modules/@aws-sdk/checksums": { "version": "3.1000.26", "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.26.tgz", @@ -2425,12 +2362,6 @@ "node": ">=18.0.0" } }, - "node_modules/@stablelib/base64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", - "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", - "license": "MIT" - }, "node_modules/@turf/area": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@turf/area/-/area-6.5.0.tgz", @@ -2791,16 +2722,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, "node_modules/@types/readable-stream": { "version": "4.0.24", "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.24.tgz", @@ -2907,22 +2828,12 @@ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -3117,12 +3028,6 @@ "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==", "license": "MIT" }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -3944,18 +3849,6 @@ "node": ">=0.1.90" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/commander": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", @@ -4210,15 +4103,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4429,21 +4313,6 @@ "node": ">= 0.4" } }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -4533,12 +4402,6 @@ "optional": true, "peer": true }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, "node_modules/eyes": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", @@ -4631,12 +4494,6 @@ "node": ">=6" } }, - "node_modules/fast-sha256": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", - "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", - "license": "Unlicense" - }, "node_modules/fast-uri": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.2.tgz", @@ -4921,41 +4778,6 @@ "optional": true, "peer": true }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/fraction.js": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.4.tgz", @@ -5033,36 +4855,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", - "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^6.1.1", - "google-logging-utils": "^0.0.2", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -5194,32 +4986,6 @@ "node": ">= 6" } }, - "node_modules/google-auth-library": { - "version": "9.15.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", - "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/google-logging-utils": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", - "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -5262,19 +5028,6 @@ "graphql": ">=0.11 <=17" } }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "license": "MIT", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/gunzip-maybe": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", @@ -5552,6 +5305,8 @@ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "4" @@ -5569,15 +5324,6 @@ "knuth-shuffle": "^1.0.0" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -5925,18 +5671,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-unicode-supported": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", @@ -6273,15 +6007,6 @@ "node": ">=4" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, "node_modules/json-bigint-fixes": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/json-bigint-fixes/-/json-bigint-fixes-1.1.0.tgz", @@ -6310,19 +6035,6 @@ "dequal": "^2.0.3" } }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -7246,27 +6958,6 @@ "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -7466,26 +7157,6 @@ "node": "^18 || ^20 || >= 21" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -9454,16 +9125,6 @@ "node": ">=8" } }, - "node_modules/standardwebhooks": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", - "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", - "license": "MIT", - "dependencies": { - "@stablelib/base64": "^1.0.0", - "fast-sha256": "^1.3.0" - } - }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -9945,12 +9606,6 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -10092,20 +9747,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/validate.js": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/validate.js/-/validate.js-0.13.1.tgz", @@ -10146,15 +9787,6 @@ "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==", "license": "MIT" }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index 102229b..15c4535 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "agent-example-harper", "version": "1.0.0", - "description": "A conversational AI agent with persistent memory, built on Harper and Claude", + "description": "A conversational AI agent with persistent memory, built on Harper with scope.models", "type": "module", "engines": { "harper": "^5.0" @@ -14,8 +14,6 @@ "test:integration": "harper-integration-test-run 'integrationTests/**/*.test.ts'" }, "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", "harper": "^5.2.1" }, diff --git a/resources/Agent.js b/resources/Agent.js index 02e9310..3851fa7 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -1,40 +1,10 @@ import { Resource, tables } from 'harper' -import Anthropic from '@anthropic-ai/sdk' -import { AnthropicVertex } from '@anthropic-ai/vertex-sdk' -import { config } from '../lib/config.js' import { embed } from '../lib/embeddings.js' -let _client -const getClient = () => { - if (_client) return _client - if (config.provider() === 'vertex') { - _client = new AnthropicVertex({ - projectId: config.vertex.projectId(), - region: config.vertex.region(), - }) - } else { - _client = new Anthropic({ apiKey: config.anthropic.apiKey() }) - } - return _client -} - -const getModel = () => - config.provider() === 'vertex' ? config.vertex.model() : config.anthropic.model() - const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the user's current question. \ Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` -// Approximate pricing for Claude Sonnet 4.5 (per token) -const COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens -const COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens -const COST_PER_WEB_SEARCH = 10 / 1_000 // $10 / 1K searches - -// Anthropic web search tool — executed server-side, no external API key needed. -// Not available on Vertex AI without an org policy change. -const WEB_SEARCH_TOOL = { type: 'web_search_20250305', name: 'web_search', max_uses: 5 } -const isVertex = () => config.provider() === 'vertex' - // Normalize text for embedding cache key — lowercase, strip punctuation, collapse whitespace const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() @@ -44,8 +14,7 @@ const normalize = (s) => // equivalent to cosine similarity >= 0.88 (distance = 1 - similarity = 0.12). const CACHE_DISTANCE_THRESHOLD = 0.12 -// Get or compute an embedding, using Harper as a cache to skip the SLM on repeated text. -// On Fabric, the SLM takes ~2.3s per embedding — this cache makes repeat queries instant. +// Get or compute an embedding, using Harper as a cache to skip the model on repeated text. async function cachedEmbed(text) { const key = normalize(text) const cached = await tables.EmbeddingCache.get(key) @@ -133,23 +102,17 @@ export class Agent extends Resource { const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } console.log('[Agent] timing:', JSON.stringify(timing)) - // Return the cached answer — zero LLM cost + // Return the cached answer — zero LLM call if (cachedReply) { - const t5 = Date.now() - let savedCost = 0 try { - const origMsg = await tables.Message.get(cachedReply.id) - savedCost = origMsg?.cost ?? 0 const stats = await tables.Stats.get('global') await tables.Stats.put({ id: 'global', - totalSaved: ((stats?.totalSaved) ?? 0) + savedCost, - cacheHits: ((stats?.cacheHits) ?? 0) + 1, - updatedAt: new Date().toISOString(), + totalSaved: stats?.totalSaved ?? 0, + cacheHits: ((stats?.cacheHits) ?? 0) + 1, + updatedAt: new Date().toISOString(), }) } catch {} - const tStats = Date.now() - t5 - console.log('[Agent] cache hit stats update:', tStats + 'ms') return { conversationId, message: { role: 'assistant', content: cachedReply.content }, @@ -157,66 +120,40 @@ export class Agent extends Resource { latencyMs: Date.now() - startTime, timing, tokens: { input: 0, output: 0, total: 0 }, - cost: { input: 0, output: 0, total: 0, saved: savedCost }, vectorContext: { hit: true, count: 1, cached: true }, }, } } - // 5. Call Claude with web search enabled — standalone question, no conversation history. - // Anthropic executes searches server-side, no external search API or key required. - const messages = [{ role: 'user', content: message }] - - const tools = isVertex() ? [] : [WEB_SEARCH_TOOL] - - let apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), - system: SYSTEM_PROMPT, - messages, - }) + // 5. Generate via scope.models.generate() — routes to whatever backend the host + // has configured for `models.generative.default` (vLLM on Fabric GPU hosts, + // Ollama / OpenAI / Anthropic on other deployments). + const scope = globalThis.harperScope + if (!scope) { + throw new Error('Harper scope not yet captured — modelCapture plugin must run before first generate call') + } - // Handle pause_turn — server hit the max_uses limit mid-response; continue once - if (apiResponse.stop_reason === 'pause_turn') { - apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), + const result = await scope.models.generate( + { + messages: [{ role: 'user', content: message }], system: SYSTEM_PROMPT, - messages: [...messages, { role: 'assistant', content: apiResponse.content }], - }) - } + }, + { maxTokens: 1024 }, + ) const latencyMs = Date.now() - startTime - - // The API can split the answer across multiple text blocks (sentence fragments joined - // without separators) and may emit a text block BEFORE the web search tool call. - // Strategy: find the last non-text block (tool use / search result) and take only the - // text blocks that follow it — these form the actual answer. Join with '' since the - // fragments are already continuous prose. Falls back to all text blocks if no tools used. - const lastToolIdx = apiResponse.content.reduce((acc, b, i) => b.type !== 'text' ? i : acc, -1) - const assistantContent = apiResponse.content - .slice(lastToolIdx + 1) - .filter((b) => b.type === 'text') - .map((b) => b.text) - .join('') - .trim() - - const { input_tokens, output_tokens } = apiResponse.usage - const webSearches = apiResponse.usage?.server_tool_use?.web_search_requests ?? 0 + const assistantContent = result.content?.trim() ?? '' + const promptTokens = result.usage?.promptTokens ?? 0 + const completionTokens = result.usage?.completionTokens ?? 0 // 9. Store the assistant's response with its embedding const assistantMsgId = crypto.randomUUID() const assistantEmbedding = await cachedEmbed(assistantContent) - const searchCost = webSearches * COST_PER_WEB_SEARCH - const totalCost = (input_tokens * COST_INPUT_PER_TOKEN) + (output_tokens * COST_OUTPUT_PER_TOKEN) + searchCost await tables.Message.put({ id: assistantMsgId, conversationId, role: 'assistant', content: assistantContent, - cost: totalCost, embedding: assistantEmbedding, createdAt: new Date().toISOString(), }) @@ -232,18 +169,12 @@ export class Agent extends Resource { message: { role: 'assistant', content: assistantContent }, meta: { latencyMs, + timing, tokens: { - input: input_tokens, - output: output_tokens, - total: input_tokens + output_tokens, - }, - cost: { - input: +(input_tokens * COST_INPUT_PER_TOKEN).toFixed(6), - output: +(output_tokens * COST_OUTPUT_PER_TOKEN).toFixed(6), - search: +searchCost.toFixed(6), - total: +totalCost.toFixed(6), + input: promptTokens, + output: completionTokens, + total: promptTokens + completionTokens, }, - webSearches, vectorContext: { hit: false, count: 0, cached: false }, }, } @@ -253,6 +184,6 @@ export class Agent extends Resource { export class PublicStats extends Resource { static async get(target) { target.checkPermission = false - return await tables.Stats.get('global') ?? { id: 'global', totalSaved: 0, cacheHits: 0 } + return await tables.Stats.get('global') ?? { id: 'global', cacheHits: 0 } } } From 99c8f69b9ff7a39bc150121493ed885fe0e46251 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 20:13:35 -0600 Subject: [PATCH 3/8] Restore estimated-cost reporting (Claude pricing as comparator) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UI was breaking on `meta.cost.saved` undefined. Bring back the cost block but interpret it as a hypothetical — what each generation WOULD have cost on Claude Sonnet 4.5 ($3/1M input, $15/1M output). Local GPU compute is effectively $0; the dashboard tracks what we're saving by self-hosting + the semantic cache. - Estimated cost computed from token counts vLLM returns - Stored on each assistant Message (same Float field as before) - Cache hits credit the original message's cost to Stats.totalSaved - meta.cost { input, output, total, saved } shape matches what Chat.js's buildMeta() reads, so no UI changes needed Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 3851fa7..5633792 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -5,6 +5,16 @@ const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the use Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` +// Hypothetical Claude Sonnet 4.5 pricing used to estimate what each generation +// WOULD have cost if we'd called Anthropic instead of the local GPU. Real local +// compute cost is roughly $0 (sunk-cost GPU); the dashboard shows what we're +// saving by self-hosting + caching. +const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens +const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens + +const estimateClaudeCost = (promptTokens, completionTokens) => + promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN + // Normalize text for embedding cache key — lowercase, strip punctuation, collapse whitespace const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() @@ -102,13 +112,18 @@ export class Agent extends Resource { const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } console.log('[Agent] timing:', JSON.stringify(timing)) - // Return the cached answer — zero LLM call + // Return the cached answer — zero LLM call. We credit the original message's + // estimated cost to `totalSaved` so the dashboard shows the running benefit + // of the semantic cache (and self-hosting more broadly). if (cachedReply) { + let savedCost = 0 try { + const origMsg = await tables.Message.get(cachedReply.id) + savedCost = origMsg?.cost ?? 0 const stats = await tables.Stats.get('global') await tables.Stats.put({ id: 'global', - totalSaved: stats?.totalSaved ?? 0, + totalSaved: (stats?.totalSaved ?? 0) + savedCost, cacheHits: ((stats?.cacheHits) ?? 0) + 1, updatedAt: new Date().toISOString(), }) @@ -120,6 +135,7 @@ export class Agent extends Resource { latencyMs: Date.now() - startTime, timing, tokens: { input: 0, output: 0, total: 0 }, + cost: { input: 0, output: 0, total: 0, saved: savedCost }, vectorContext: { hit: true, count: 1, cached: true }, }, } @@ -145,8 +161,11 @@ export class Agent extends Resource { const assistantContent = result.content?.trim() ?? '' const promptTokens = result.usage?.promptTokens ?? 0 const completionTokens = result.usage?.completionTokens ?? 0 + const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) - // 9. Store the assistant's response with its embedding + // 9. Store the assistant's response with its embedding. We persist the + // *hypothetical* Claude cost so a future cache-hit on this same message + // can credit that amount to `totalSaved`. const assistantMsgId = crypto.randomUUID() const assistantEmbedding = await cachedEmbed(assistantContent) await tables.Message.put({ @@ -154,6 +173,7 @@ export class Agent extends Resource { conversationId, role: 'assistant', content: assistantContent, + cost: estimatedCost, embedding: assistantEmbedding, createdAt: new Date().toISOString(), }) @@ -175,6 +195,13 @@ export class Agent extends Resource { output: completionTokens, total: promptTokens + completionTokens, }, + cost: { + input: +(promptTokens * CLAUDE_COST_INPUT_PER_TOKEN).toFixed(6), + output: +(completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN).toFixed(6), + total: +estimatedCost.toFixed(6), + // `saved` is what cache hits credit; on a real generation it stays 0. + saved: 0, + }, vectorContext: { hit: false, count: 0, cached: false }, }, } @@ -184,6 +211,6 @@ export class Agent extends Resource { export class PublicStats extends Resource { static async get(target) { target.checkPermission = false - return await tables.Stats.get('global') ?? { id: 'global', cacheHits: 0 } + return await tables.Stats.get('global') ?? { id: 'global', totalSaved: 0, cacheHits: 0 } } } From 7d0521171bb27642926c2a26298c7a0d92036413 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 20:14:49 -0600 Subject: [PATCH 4/8] Estimate token counts from text length (~4 chars/token heuristic) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scope.models.generate() only returns { content, finishReason } today — the backend's usage info isn't surfaced to the caller. Approximate input/output tokens from character counts so the estimated-Claude-cost comparator on the Chat dashboard shows non-zero values. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 5633792..844b23d 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -12,6 +12,11 @@ as background knowledge only if it is directly relevant. Never recite or recap p const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens +// scope.models.generate() returns only { content, finishReason } today — the +// backend's token usage isn't surfaced to callers. Approximate with the +// ~4-chars-per-token rule of thumb for English; close enough for a comparator. +const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) + const estimateClaudeCost = (promptTokens, completionTokens) => promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN @@ -159,8 +164,8 @@ export class Agent extends Resource { const latencyMs = Date.now() - startTime const assistantContent = result.content?.trim() ?? '' - const promptTokens = result.usage?.promptTokens ?? 0 - const completionTokens = result.usage?.completionTokens ?? 0 + const promptTokens = estimateTokens(SYSTEM_PROMPT + message) + const completionTokens = estimateTokens(assistantContent) const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) // 9. Store the assistant's response with its embedding. We persist the From 61a9b3802b5a5185131ab93c409ce81d24bedf67 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:14:56 -0600 Subject: [PATCH 5/8] Fix cache-match selection: explicit distance sort + tighter threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs were stacking up: 1. The HNSW search returned matches that satisfy the threshold but didn't guarantee distance-ascending iteration order, so the loop's first match wasn't necessarily the closest. Collect candidates, compute cosine distance explicitly, sort, then pick the closest valid one. 2. The threshold (0.12 cosine distance ≈ 0.88 similarity) was loose enough that distinct queries about the same topic would collide ("describe the moon landing" vs "tell me about apollo 11"). Tighten to 0.05 (≈ 0.95 similarity), which still catches near-paraphrases and re-wordings while filtering out merely-related questions. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 844b23d..5c96654 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -25,9 +25,23 @@ const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() // Cosine distance threshold for Harper's native HNSW vector search. -// Harper uses cosine *distance* (0 = identical, 2 = opposite), so this is -// equivalent to cosine similarity >= 0.88 (distance = 1 - similarity = 0.12). -const CACHE_DISTANCE_THRESHOLD = 0.12 +// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.05 ≈ cosine +// similarity 0.95 — strict enough that semantically different queries +// ("describe the moon landing" vs "tell me about apollo 11") don't collide. +const CACHE_DISTANCE_THRESHOLD = 0.05 + +// HNSW search returns matches that satisfy the threshold but doesn't guarantee +// distance-ordered iteration. We compute distance ourselves and pick the closest. +function cosineDistance(a, b) { + let dot = 0, na = 0, nb = 0 + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i] + na += a[i] * a[i] + nb += b[i] * b[i] + } + const denom = Math.sqrt(na) * Math.sqrt(nb) + return denom === 0 ? 1 : 1 - dot / denom +} // Get or compute an embedding, using Harper as a cache to skip the model on repeated text. async function cachedEmbed(text) { @@ -84,6 +98,9 @@ export class Agent extends Resource { const tStore = Date.now() - t3 // 4. Semantic cache — Harper-native HNSW vector search with distance threshold. + // HNSW search returns matches under the threshold but iteration order isn't + // guaranteed to be distance-ascending, so we collect candidates, compute + // cosine distance ourselves, and pick the closest valid one. const t4 = Date.now() let cachedReply = null const nearbyMsgs = tables.Message.search({ @@ -93,11 +110,17 @@ export class Agent extends Resource { value: CACHE_DISTANCE_THRESHOLD, target: userEmbedding, }, - limit: 10, + limit: 20, }) + const candidates = [] for await (const match of nearbyMsgs) { - if (match.id === userMsgId || match.role !== 'user') continue + if (match.id === userMsgId || match.role !== 'user' || !match.embedding) continue + candidates.push({ match, distance: cosineDistance(userEmbedding, match.embedding) }) + } + candidates.sort((a, b) => a.distance - b.distance) + + for (const { match } of candidates) { const matchConvMsgs = [] const matchHistory = tables.Message.search({ conditions: [{ attribute: 'conversationId', value: match.conversationId }], From 6e8910075c6e99fd4c2e5451354dd342428f18fc Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:19:20 -0600 Subject: [PATCH 6/8] Debug: log cache candidates with computed distance + hard threshold filter --- resources/Agent.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/resources/Agent.js b/resources/Agent.js index 5c96654..6b2185f 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -120,7 +120,20 @@ export class Agent extends Resource { } candidates.sort((a, b) => a.distance - b.distance) - for (const { match } of candidates) { + // Debug: print what the search returned along with the actual computed distance. + // Harper's HNSW `lt` filter doesn't always cull things outside the threshold, + // so we apply a hard check using the distance we computed ourselves. + if (candidates.length > 0) { + console.log('[Agent] cache candidates:', candidates.slice(0, 5).map((c) => ({ + id: c.match.id, + role: c.match.role, + dist: +c.distance.toFixed(4), + content: c.match.content?.slice(0, 60), + }))) + } + const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) + + for (const { match } of filtered) { const matchConvMsgs = [] const matchHistory = tables.Message.search({ conditions: [{ attribute: 'conversationId', value: match.conversationId }], From 14b07d569d4ba9e6608e593b05aa44d60606dcf8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:23:46 -0600 Subject: [PATCH 7/8] Fix cache: assistant reply must be the IMMEDIATE next message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous .find(role==='assistant') walked past any subsequent user messages and grabbed the next assistant reply even if it was many turns later in the conversation. With a long conversation like: user: Is soccer fun? assistant: ...soccer is fun... user: Is soccer fun (cache hit, no new assistant stored) user: Is soccer fun?? (cache hit, no new assistant stored) user: what is 2 plus 3 assistant: 2 plus 3 is 5 matching the second "Is soccer fun" returned "2 plus 3 is 5" — the next assistant message in chronological order, but completely unrelated. Only accept the IMMEDIATELY-following message as the reply. If the next message is another user message, skip this candidate and try the next. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 6b2185f..5b25d25 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -120,17 +120,8 @@ export class Agent extends Resource { } candidates.sort((a, b) => a.distance - b.distance) - // Debug: print what the search returned along with the actual computed distance. - // Harper's HNSW `lt` filter doesn't always cull things outside the threshold, + // Harper's HNSW `lt` filter doesn't always cull matches outside the threshold, // so we apply a hard check using the distance we computed ourselves. - if (candidates.length > 0) { - console.log('[Agent] cache candidates:', candidates.slice(0, 5).map((c) => ({ - id: c.match.id, - role: c.match.role, - dist: +c.distance.toFixed(4), - content: c.match.content?.slice(0, 60), - }))) - } const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) for (const { match } of filtered) { @@ -142,9 +133,14 @@ export class Agent extends Resource { for await (const m of matchHistory) matchConvMsgs.push(m) matchConvMsgs.sort((a, b) => a.createdAt.localeCompare(b.createdAt)) const midx = matchConvMsgs.findIndex((m) => m.id === match.id) - const reply = matchConvMsgs.slice(midx + 1).find((m) => m.role === 'assistant') - if (reply) { - cachedReply = reply + // The matched message's reply must be the IMMEDIATELY following message. + // `.find()` would walk past any subsequent user-msgs (cache hits that didn't + // generate a reply) and pull an unrelated answer from much later in the + // conversation — e.g. matching "is soccer fun" but returning the assistant + // reply to a later "what is 2 plus 3" question in the same conversation. + const next = matchConvMsgs[midx + 1] + if (next?.role === 'assistant') { + cachedReply = next break } } From e2447e773f2e7ea6502e1d6c1d77fe7b97bee9f3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:28:47 -0600 Subject: [PATCH 8/8] Loosen cache threshold to 0.15 (~0.85 similarity) Now that the immediate-next-message fix is in, the threshold can be relaxed to catch more rewordings without serving wrong answers. The 0.05 setting was paranoia about false positives that turned out to come from the slice/find bug. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 5b25d25..db61e01 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -25,10 +25,11 @@ const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() // Cosine distance threshold for Harper's native HNSW vector search. -// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.05 ≈ cosine -// similarity 0.95 — strict enough that semantically different queries -// ("describe the moon landing" vs "tell me about apollo 11") don't collide. -const CACHE_DISTANCE_THRESHOLD = 0.05 +// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.15 ≈ cosine +// similarity 0.85 — loose enough to catch rewordings and related phrasings +// ("describe the moon landing" / "tell me about apollo 11"), tight enough +// that the matched reply is reasonably on-topic. +const CACHE_DISTANCE_THRESHOLD = 0.15 // HNSW search returns matches that satisfy the threshold but doesn't guarantee // distance-ordered iteration. We compute distance ourselves and pick the closest.