diff --git a/.gitignore b/.gitignore index ca8a006..cb565e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ node_modules/ -models/ .env CONFIG.env *.env diff --git a/config.yaml b/config.yaml index d6be5d1..861fdcc 100644 --- a/config.yaml +++ b/config.yaml @@ -6,6 +6,10 @@ loadEnv: rest: true +# Captures the Scope object so resources can call scope.models.embed(). +# See lib/modelCapture.js. +extensionModule: 'lib/modelCapture.js' + graphqlSchema: files: 'schemas/*.graphql' diff --git a/lib/config.js b/lib/config.js deleted file mode 100644 index d1e1885..0000000 --- a/lib/config.js +++ /dev/null @@ -1,21 +0,0 @@ -const required = (name) => { - const value = process.env[name] - if (!value) throw new Error(`Missing required env var: ${name}`) - return value -} - -const optional = (name, fallback) => process.env[name] ?? fallback - -export const config = { - // "anthropic" (direct API) or "vertex" (Google Cloud Vertex AI) - provider: () => optional('LLM_PROVIDER', 'anthropic'), - anthropic: { - apiKey: () => required('ANTHROPIC_API_KEY'), - model: () => optional('CLAUDE_MODEL', 'claude-sonnet-4-5-20250929'), - }, - vertex: { - projectId: () => required('VERTEX_PROJECT_ID'), - region: () => optional('VERTEX_REGION', 'global'), - model: () => optional('VERTEX_MODEL', 'claude-sonnet-4-6'), - }, -} diff --git a/lib/embeddings.js b/lib/embeddings.js index 0b3102e..bddf169 100644 --- a/lib/embeddings.js +++ b/lib/embeddings.js @@ -1,14 +1,14 @@ -import { init, embed as llamaEmbed } from 'harper-fabric-embeddings' -import { resolve } from 'path' -import { fileURLToPath } from 'url' - -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelPath = resolve(__dirname, '../models/bge-small-en-v1.5-q4_k_m.gguf') - -// Model is pre-downloaded by the predev/prestart npm hook (scripts/download-model.js) -const initPromise = init({ modelPath }) +// Thin wrapper around `scope.models.embed()` (harper#510). The host's +// configured backend (Ollama on Fabric GPU hosts, or any backend configured +// via the `models:` block in harperdb-config.yaml / env vars) handles the +// actual inference. Returns a plain Array for compatibility with +// Harper's HNSW vector index storage. export async function embed(text) { - await initPromise - return llamaEmbed(text) + const scope = globalThis.harperScope; + if (!scope) { + throw new Error('Harper scope not yet captured — modelCapture plugin must run before first embed call'); + } + const [vector] = await scope.models.embed(text); + return Array.from(vector); } diff --git a/lib/modelCapture.js b/lib/modelCapture.js new file mode 100644 index 0000000..018348e --- /dev/null +++ b/lib/modelCapture.js @@ -0,0 +1,11 @@ +// Plugin entry — captures the Scope object so `resources/*.js` can call +// `scope.models.embed()` against the host's configured embedding backend. +// +// Harper's `scope` is passed to plugins via `handleApplication(scope)` but +// isn't exposed as a global to Resource classes. This tiny plugin stashes the +// Scope on `globalThis.harperScope` at app boot, then `lib/embeddings.js` +// reads it from there. + +export function handleApplication(scope) { + globalThis.harperScope = scope; +} diff --git a/package-lock.json b/package-lock.json index 1142a3b..6deb47b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,11 +8,8 @@ "name": "agent-example-harper", "version": "1.0.0", "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", @@ -20,11 +17,6 @@ }, "engines": { "harper": "^5.0" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" } }, "node_modules/@agoric/babel-generator": { @@ -41,67 +33,6 @@ "node": ">=6.9.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.39.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.39.0.tgz", - "integrity": "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==", - "license": "MIT", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/@anthropic-ai/vertex-sdk": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/vertex-sdk/-/vertex-sdk-0.15.0.tgz", - "integrity": "sha512-i2LDdu6VB8Lqqip+kbNSXRxQgFsCg6GPBO/X2zRJwLl99dNzf28nb6Rdi0EodONXsyJfY2TKdGR+y5l1/AKFEg==", - "license": "MIT", - "dependencies": { - "@anthropic-ai/sdk": ">=0.50.3 <1", - "google-auth-library": "^9.4.2" - } - }, - "node_modules/@anthropic-ai/vertex-sdk/node_modules/@anthropic-ai/sdk": { - "version": "0.116.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.116.0.tgz", - "integrity": "sha512-4UEapYQ+epLEMsAuLZDvW8ExVSOtHD8a7zTyLzhw0H9RXJ1eilPgmqhjwgcdg22diwx13spw6fJ4rONZ+bS7Ww==", - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, "node_modules/@aws-sdk/checksums": { "version": "3.1000.26", "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.26.tgz", @@ -1987,131 +1918,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/@node-llama-cpp/linux-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-arm64/-/linux-arm64-3.18.1.tgz", - "integrity": "sha512-rXMgZxUay78FOJV/fJ67apYP9eElH5jd4df5YRKPlLhLHHchuOSyDn+qtyW/L/EnPzpogoLkmULqCkdXU39XsQ==", - "cpu": [ - "arm64", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-armv7l": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-armv7l/-/linux-armv7l-3.18.1.tgz", - "integrity": "sha512-BrJL2cGo0pN5xd5nw+CzTn2rFMpz9MJyZZPUY81ptGkF2uIuXT2hdCVh56i9ImQrTwBfq1YcZL/l/Qe/1+HR/Q==", - "cpu": [ - "arm", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.17.1.tgz", - "integrity": "sha512-/o/UoqAdslg4ExdKYyYPqbw+21Dr4cQ2JgouXg8Ji3opRKoTMrlUNfrMwIsYZfbDDJ8l7xFnwfGIwdlQ5RPwJg==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.17.1.tgz", - "integrity": "sha512-oRq6/7qCMsazO2Cw0oCyiILZmMvejKJgLAIG60E00WOZWhpJGjh71JGnOybRycKA015mFPNDHzT3SDdUZtZBew==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.17.1.tgz", - "integrity": "sha512-3L0nFVi70j+Qk7Xb8p/RQVMU0E28G0xXX0YL6Vzkirq3DazPYhWOLWUUs9MtGW2FrBg/6PLqyddmxwBfCpjm3w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-arm64/-/win-arm64-3.18.1.tgz", - "integrity": "sha512-S05YUzBMVSRS5KNbOS26cDYugeQHqogI3uewtTUBVC0tPbTHRSKjsdicmgWru1eNAry399LWWhzOf/3St/qsAw==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-x64/-/win-x64-3.18.1.tgz", - "integrity": "sha512-QLDVphPl+YDI+x/VYYgIV1N9g0GMXk3PqcoopOUG3cBRUtce7FO+YX903YdRJezs4oKbIp8YaO+xYBgeUSqhpA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2556,12 +2362,6 @@ "node": ">=18.0.0" } }, - "node_modules/@stablelib/base64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", - "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", - "license": "MIT" - }, "node_modules/@turf/area": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@turf/area/-/area-6.5.0.tgz", @@ -2922,16 +2722,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, "node_modules/@types/readable-stream": { "version": "4.0.24", "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.24.tgz", @@ -3038,22 +2828,12 @@ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -3248,12 +3028,6 @@ "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==", "license": "MIT" }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -4075,18 +3849,6 @@ "node": ">=0.1.90" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/commander": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", @@ -4341,15 +4103,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4560,21 +4313,6 @@ "node": ">= 0.4" } }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -4664,12 +4402,6 @@ "optional": true, "peer": true }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, "node_modules/eyes": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", @@ -4762,12 +4494,6 @@ "node": ">=6" } }, - "node_modules/fast-sha256": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", - "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", - "license": "Unlicense" - }, "node_modules/fast-uri": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.2.tgz", @@ -5052,41 +4778,6 @@ "optional": true, "peer": true }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/fraction.js": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.4.tgz", @@ -5164,36 +4855,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", - "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^6.1.1", - "google-logging-utils": "^0.0.2", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -5325,32 +4986,6 @@ "node": ">= 6" } }, - "node_modules/google-auth-library": { - "version": "9.15.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", - "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/google-logging-utils": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", - "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -5393,19 +5028,6 @@ "graphql": ">=0.11 <=17" } }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "license": "MIT", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/gunzip-maybe": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", @@ -5558,77 +5180,6 @@ } } }, - "node_modules/harper-fabric-embeddings": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/harper-fabric-embeddings/-/harper-fabric-embeddings-0.2.3.tgz", - "integrity": "sha512-25F1xzRTJ+19NlDiMI0RLF47u4Fwd5Ve/V03q06kJjCioqvEc2yrAASQ3NY4O+LJDU9rNq9WQivFeU+9JKt4IA==", - "hasInstallScript": true, - "license": "MIT", - "engines": { - "node": ">=22" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-arm64": "3.18.1", - "@node-llama-cpp/linux-armv7l": "3.18.1", - "@node-llama-cpp/linux-x64": "3.18.1", - "@node-llama-cpp/mac-arm64-metal": "3.18.1", - "@node-llama-cpp/mac-x64": "3.18.1", - "@node-llama-cpp/win-arm64": "3.18.1", - "@node-llama-cpp/win-x64": "3.18.1" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/linux-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.18.1.tgz", - "integrity": "sha512-tRmWcsyvAcqJHQHXHsaOkx6muGbcirA9nRdNgH6n7bjGUw4VuoBD3dChyNF3/Ktt7ohB9kz+XhhyZjbDHpXyMA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.18.1.tgz", - "integrity": "sha512-cyZTdsUMlvuRlGmkkoBbN3v/DT6NuruEqoQYd9CqIrPyLa1xLNBTSKIZ9SgRnw23iCOj4URfITvRP+2pu63LuQ==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.18.1.tgz", - "integrity": "sha512-GfCPgdltaIpBhEnQ7WfsrRXrZO9r9pBtDUAQMXRuJwOPP5q7xKrQZUXI6J6mpc8tAG0//CTIuGn4hTKoD/8V8w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/harper/node_modules/uuid": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", @@ -5754,6 +5305,8 @@ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "4" @@ -5771,15 +5324,6 @@ "knuth-shuffle": "^1.0.0" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -6127,18 +5671,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-unicode-supported": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", @@ -6475,15 +6007,6 @@ "node": ">=4" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, "node_modules/json-bigint-fixes": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/json-bigint-fixes/-/json-bigint-fixes-1.1.0.tgz", @@ -6512,19 +6035,6 @@ "dequal": "^2.0.3" } }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -7448,27 +6958,6 @@ "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -7668,26 +7157,6 @@ "node": "^18 || ^20 || >= 21" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -9656,16 +9125,6 @@ "node": ">=8" } }, - "node_modules/standardwebhooks": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", - "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", - "license": "MIT", - "dependencies": { - "@stablelib/base64": "^1.0.0", - "fast-sha256": "^1.3.0" - } - }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -10147,12 +9606,6 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -10294,20 +9747,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/validate.js": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/validate.js/-/validate.js-0.13.1.tgz", @@ -10348,15 +9787,6 @@ "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==", "license": "MIT" }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index 2ea6531..15c4535 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,12 @@ { "name": "agent-example-harper", "version": "1.0.0", - "description": "A conversational AI agent with persistent memory, built on Harper and Claude", + "description": "A conversational AI agent with persistent memory, built on Harper with scope.models", "type": "module", "engines": { "harper": "^5.0" }, "scripts": { - "setup": "node scripts/download-model.js", - "predev": "node scripts/download-model.js", - "prestart": "node scripts/download-model.js", "start": "npx -y dotenv-cli -e .env -o -- harper run .", "dev": "npx -y dotenv-cli -e .env -o -- harper dev .", "login": "node login.js", @@ -17,19 +14,11 @@ "test:integration": "harper-integration-test-run 'integrationTests/**/*.test.ts'" }, "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", "@types/node": "^22.19.19" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" } } diff --git a/resources/Agent.js b/resources/Agent.js index 02e9310..db61e01 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -1,51 +1,50 @@ import { Resource, tables } from 'harper' -import Anthropic from '@anthropic-ai/sdk' -import { AnthropicVertex } from '@anthropic-ai/vertex-sdk' -import { config } from '../lib/config.js' import { embed } from '../lib/embeddings.js' -let _client -const getClient = () => { - if (_client) return _client - if (config.provider() === 'vertex') { - _client = new AnthropicVertex({ - projectId: config.vertex.projectId(), - region: config.vertex.region(), - }) - } else { - _client = new Anthropic({ apiKey: config.anthropic.apiKey() }) - } - return _client -} - -const getModel = () => - config.provider() === 'vertex' ? config.vertex.model() : config.anthropic.model() - const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the user's current question. \ Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` -// Approximate pricing for Claude Sonnet 4.5 (per token) -const COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens -const COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens -const COST_PER_WEB_SEARCH = 10 / 1_000 // $10 / 1K searches +// Hypothetical Claude Sonnet 4.5 pricing used to estimate what each generation +// WOULD have cost if we'd called Anthropic instead of the local GPU. Real local +// compute cost is roughly $0 (sunk-cost GPU); the dashboard shows what we're +// saving by self-hosting + caching. +const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens +const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens + +// scope.models.generate() returns only { content, finishReason } today — the +// backend's token usage isn't surfaced to callers. Approximate with the +// ~4-chars-per-token rule of thumb for English; close enough for a comparator. +const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) -// Anthropic web search tool — executed server-side, no external API key needed. -// Not available on Vertex AI without an org policy change. -const WEB_SEARCH_TOOL = { type: 'web_search_20250305', name: 'web_search', max_uses: 5 } -const isVertex = () => config.provider() === 'vertex' +const estimateClaudeCost = (promptTokens, completionTokens) => + promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN // Normalize text for embedding cache key — lowercase, strip punctuation, collapse whitespace const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() // Cosine distance threshold for Harper's native HNSW vector search. -// Harper uses cosine *distance* (0 = identical, 2 = opposite), so this is -// equivalent to cosine similarity >= 0.88 (distance = 1 - similarity = 0.12). -const CACHE_DISTANCE_THRESHOLD = 0.12 +// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.15 ≈ cosine +// similarity 0.85 — loose enough to catch rewordings and related phrasings +// ("describe the moon landing" / "tell me about apollo 11"), tight enough +// that the matched reply is reasonably on-topic. +const CACHE_DISTANCE_THRESHOLD = 0.15 + +// HNSW search returns matches that satisfy the threshold but doesn't guarantee +// distance-ordered iteration. We compute distance ourselves and pick the closest. +function cosineDistance(a, b) { + let dot = 0, na = 0, nb = 0 + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i] + na += a[i] * a[i] + nb += b[i] * b[i] + } + const denom = Math.sqrt(na) * Math.sqrt(nb) + return denom === 0 ? 1 : 1 - dot / denom +} -// Get or compute an embedding, using Harper as a cache to skip the SLM on repeated text. -// On Fabric, the SLM takes ~2.3s per embedding — this cache makes repeat queries instant. +// Get or compute an embedding, using Harper as a cache to skip the model on repeated text. async function cachedEmbed(text) { const key = normalize(text) const cached = await tables.EmbeddingCache.get(key) @@ -100,6 +99,9 @@ export class Agent extends Resource { const tStore = Date.now() - t3 // 4. Semantic cache — Harper-native HNSW vector search with distance threshold. + // HNSW search returns matches under the threshold but iteration order isn't + // guaranteed to be distance-ascending, so we collect candidates, compute + // cosine distance ourselves, and pick the closest valid one. const t4 = Date.now() let cachedReply = null const nearbyMsgs = tables.Message.search({ @@ -109,11 +111,21 @@ export class Agent extends Resource { value: CACHE_DISTANCE_THRESHOLD, target: userEmbedding, }, - limit: 10, + limit: 20, }) + const candidates = [] for await (const match of nearbyMsgs) { - if (match.id === userMsgId || match.role !== 'user') continue + if (match.id === userMsgId || match.role !== 'user' || !match.embedding) continue + candidates.push({ match, distance: cosineDistance(userEmbedding, match.embedding) }) + } + candidates.sort((a, b) => a.distance - b.distance) + + // Harper's HNSW `lt` filter doesn't always cull matches outside the threshold, + // so we apply a hard check using the distance we computed ourselves. + const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) + + for (const { match } of filtered) { const matchConvMsgs = [] const matchHistory = tables.Message.search({ conditions: [{ attribute: 'conversationId', value: match.conversationId }], @@ -122,9 +134,14 @@ export class Agent extends Resource { for await (const m of matchHistory) matchConvMsgs.push(m) matchConvMsgs.sort((a, b) => a.createdAt.localeCompare(b.createdAt)) const midx = matchConvMsgs.findIndex((m) => m.id === match.id) - const reply = matchConvMsgs.slice(midx + 1).find((m) => m.role === 'assistant') - if (reply) { - cachedReply = reply + // The matched message's reply must be the IMMEDIATELY following message. + // `.find()` would walk past any subsequent user-msgs (cache hits that didn't + // generate a reply) and pull an unrelated answer from much later in the + // conversation — e.g. matching "is soccer fun" but returning the assistant + // reply to a later "what is 2 plus 3" question in the same conversation. + const next = matchConvMsgs[midx + 1] + if (next?.role === 'assistant') { + cachedReply = next break } } @@ -133,9 +150,10 @@ export class Agent extends Resource { const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } console.log('[Agent] timing:', JSON.stringify(timing)) - // Return the cached answer — zero LLM cost + // Return the cached answer — zero LLM call. We credit the original message's + // estimated cost to `totalSaved` so the dashboard shows the running benefit + // of the semantic cache (and self-hosting more broadly). if (cachedReply) { - const t5 = Date.now() let savedCost = 0 try { const origMsg = await tables.Message.get(cachedReply.id) @@ -143,13 +161,11 @@ export class Agent extends Resource { const stats = await tables.Stats.get('global') await tables.Stats.put({ id: 'global', - totalSaved: ((stats?.totalSaved) ?? 0) + savedCost, - cacheHits: ((stats?.cacheHits) ?? 0) + 1, - updatedAt: new Date().toISOString(), + totalSaved: (stats?.totalSaved ?? 0) + savedCost, + cacheHits: ((stats?.cacheHits) ?? 0) + 1, + updatedAt: new Date().toISOString(), }) } catch {} - const tStats = Date.now() - t5 - console.log('[Agent] cache hit stats update:', tStats + 'ms') return { conversationId, message: { role: 'assistant', content: cachedReply.content }, @@ -157,66 +173,45 @@ export class Agent extends Resource { latencyMs: Date.now() - startTime, timing, tokens: { input: 0, output: 0, total: 0 }, - cost: { input: 0, output: 0, total: 0, saved: savedCost }, + cost: { input: 0, output: 0, total: 0, saved: savedCost }, vectorContext: { hit: true, count: 1, cached: true }, }, } } - // 5. Call Claude with web search enabled — standalone question, no conversation history. - // Anthropic executes searches server-side, no external search API or key required. - const messages = [{ role: 'user', content: message }] - - const tools = isVertex() ? [] : [WEB_SEARCH_TOOL] - - let apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), - system: SYSTEM_PROMPT, - messages, - }) + // 5. Generate via scope.models.generate() — routes to whatever backend the host + // has configured for `models.generative.default` (vLLM on Fabric GPU hosts, + // Ollama / OpenAI / Anthropic on other deployments). + const scope = globalThis.harperScope + if (!scope) { + throw new Error('Harper scope not yet captured — modelCapture plugin must run before first generate call') + } - // Handle pause_turn — server hit the max_uses limit mid-response; continue once - if (apiResponse.stop_reason === 'pause_turn') { - apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), + const result = await scope.models.generate( + { + messages: [{ role: 'user', content: message }], system: SYSTEM_PROMPT, - messages: [...messages, { role: 'assistant', content: apiResponse.content }], - }) - } + }, + { maxTokens: 1024 }, + ) const latencyMs = Date.now() - startTime - - // The API can split the answer across multiple text blocks (sentence fragments joined - // without separators) and may emit a text block BEFORE the web search tool call. - // Strategy: find the last non-text block (tool use / search result) and take only the - // text blocks that follow it — these form the actual answer. Join with '' since the - // fragments are already continuous prose. Falls back to all text blocks if no tools used. - const lastToolIdx = apiResponse.content.reduce((acc, b, i) => b.type !== 'text' ? i : acc, -1) - const assistantContent = apiResponse.content - .slice(lastToolIdx + 1) - .filter((b) => b.type === 'text') - .map((b) => b.text) - .join('') - .trim() - - const { input_tokens, output_tokens } = apiResponse.usage - const webSearches = apiResponse.usage?.server_tool_use?.web_search_requests ?? 0 - - // 9. Store the assistant's response with its embedding + const assistantContent = result.content?.trim() ?? '' + const promptTokens = estimateTokens(SYSTEM_PROMPT + message) + const completionTokens = estimateTokens(assistantContent) + const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) + + // 9. Store the assistant's response with its embedding. We persist the + // *hypothetical* Claude cost so a future cache-hit on this same message + // can credit that amount to `totalSaved`. const assistantMsgId = crypto.randomUUID() const assistantEmbedding = await cachedEmbed(assistantContent) - const searchCost = webSearches * COST_PER_WEB_SEARCH - const totalCost = (input_tokens * COST_INPUT_PER_TOKEN) + (output_tokens * COST_OUTPUT_PER_TOKEN) + searchCost await tables.Message.put({ id: assistantMsgId, conversationId, role: 'assistant', content: assistantContent, - cost: totalCost, + cost: estimatedCost, embedding: assistantEmbedding, createdAt: new Date().toISOString(), }) @@ -232,18 +227,19 @@ export class Agent extends Resource { message: { role: 'assistant', content: assistantContent }, meta: { latencyMs, + timing, tokens: { - input: input_tokens, - output: output_tokens, - total: input_tokens + output_tokens, + input: promptTokens, + output: completionTokens, + total: promptTokens + completionTokens, }, cost: { - input: +(input_tokens * COST_INPUT_PER_TOKEN).toFixed(6), - output: +(output_tokens * COST_OUTPUT_PER_TOKEN).toFixed(6), - search: +searchCost.toFixed(6), - total: +totalCost.toFixed(6), + input: +(promptTokens * CLAUDE_COST_INPUT_PER_TOKEN).toFixed(6), + output: +(completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN).toFixed(6), + total: +estimatedCost.toFixed(6), + // `saved` is what cache hits credit; on a real generation it stays 0. + saved: 0, }, - webSearches, vectorContext: { hit: false, count: 0, cached: false }, }, } diff --git a/resources/Chat.js b/resources/Chat.js index 41ce403..f5f08eb 100644 --- a/resources/Chat.js +++ b/resources/Chat.js @@ -350,8 +350,8 @@ const HTML = /* html */ ` - Local SLM · bge-small-en-v1.5 - embeddings run in Harper · no API cost + Shared model · nomic-embed-text + scope.models.embed() · GPU on Fabric · no API cost diff --git a/scripts/download-model.js b/scripts/download-model.js deleted file mode 100644 index a8aeb63..0000000 --- a/scripts/download-model.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node -// Downloads the bge-small-en-v1.5 embedding model if not already present. -// Run automatically via the predev / prestart npm hooks. - -import { createWriteStream, existsSync, mkdirSync } from 'fs' -import { pipeline } from 'stream/promises' -import { resolve } from 'path' -import { fileURLToPath } from 'url' - -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelsDir = resolve(__dirname, '../models') -const modelPath = resolve(modelsDir, 'bge-small-en-v1.5-q4_k_m.gguf') -const MODEL_URL = - 'https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-q4_k_m.gguf' - -if (existsSync(modelPath)) { - console.log('✓ Embedding model already downloaded.') - process.exit(0) -} - -console.log('Downloading bge-small-en-v1.5 embedding model (~24 MB)...') -mkdirSync(modelsDir, { recursive: true }) - -const response = await fetch(MODEL_URL) -if (!response.ok) { - console.error(`Download failed: ${response.status} ${response.statusText}`) - process.exit(1) -} - -const total = Number(response.headers.get('content-length') || 0) -let downloaded = 0 - -const progress = new TransformStream({ - transform(chunk, controller) { - downloaded += chunk.byteLength - if (total) { - const pct = Math.round((downloaded / total) * 100) - process.stdout.write(`\r ${pct}% (${(downloaded / 1024 / 1024).toFixed(1)} MB)`) - } - controller.enqueue(chunk) - }, -}) - -await pipeline(response.body.pipeThrough(progress), createWriteStream(modelPath)) -console.log('\n✓ Model ready.')