diff --git a/bun.lock b/bun.lock index 8e6605ef8..a4edc2a28 100644 --- a/bun.lock +++ b/bun.lock @@ -802,6 +802,7 @@ "name": "@corbits/inference-catalog", "version": "0.0.1", "dependencies": { + "@corbits/ollama-adapter": "workspace:*", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", "@intx/inference-catalog": "0.3.0", diff --git a/packages/hub-client/src/seed.ts b/packages/hub-client/src/seed.ts index 9b2d334d0..fc7995816 100644 --- a/packages/hub-client/src/seed.ts +++ b/packages/hub-client/src/seed.ts @@ -49,6 +49,7 @@ import { } from "@corbits/recurring-task-workflow"; import { WORKFLOW_CATALOG } from "@corbits/workflow-catalog"; import { capabilitiesForDeployment } from "@corbits/inference-catalog/offering-capabilities"; +import { quirksForDeployment } from "@corbits/inference-catalog/ollama-context-defaults"; import { publishCorbitsToolsRegistry, type PublishCorbitsToolsRegistryArgs, @@ -1264,18 +1265,21 @@ async function ensureCatalogOffering( providerId: string; priority: number; capabilities: readonly Capability[]; + quirks?: Record; }, log: (line: string) => void, ): Promise { + const body: Record = { + modelId: args.modelId, + providerId: args.providerId, + priority: args.priority, + capabilities: args.capabilities, + }; + if (args.quirks !== undefined) body["quirks"] = args.quirks; const created = await api( "POST", `/api/tenants/${args.tenantId}/catalog/offerings`, - { - modelId: args.modelId, - providerId: args.providerId, - priority: args.priority, - capabilities: args.capabilities, - }, + body, cookies, ); if (created.status === 201) { @@ -1553,6 +1557,18 @@ export async function seedCatalog( canonicalName: model.canonicalName, capabilities, }); + // Ollama's own openai-compatible endpoint otherwise falls back to a + // small built-in context window and `@intx/inference`'s built-in + // adapter falls back to 4096 output tokens -- both silent, both + // truncating a real conversation. `quirksForDeployment` resolves this + // model's real ceiling (or `undefined` for a provider outside this + // mechanism's scope, or a model this catalog has not vetted a ceiling + // for), landing on the offering's `quirks` column exactly the way + // `capabilitiesForDeployment` lands on its `capabilities` column. + const quirks = quirksForDeployment({ + providerName: seed.provider.name, + canonicalName: model.canonicalName, + }); await ensureCatalogOffering( api, cookies, @@ -1562,6 +1578,7 @@ export async function seedCatalog( providerId: catalogProviderId, priority: offeringPriority, capabilities, + ...(quirks !== undefined ? { quirks } : {}), }, log, ); diff --git a/packages/hub-client/test/seed.test.ts b/packages/hub-client/test/seed.test.ts index a137a3317..dcf26057f 100644 --- a/packages/hub-client/test/seed.test.ts +++ b/packages/hub-client/test/seed.test.ts @@ -12,6 +12,7 @@ import { } from "../src/seed"; import { DEFAULT_SKILLS } from "../src/default-skills"; import { CATALOG_SEEDS } from "../src/catalog-seed-data"; +import { OLLAMA_MODEL_DEFAULTS } from "@corbits/inference-catalog/ollama-context-defaults"; import { assetRow, collector, @@ -1279,6 +1280,92 @@ describe("seedCatalog", () => { expect(output).toContain("catalog ready: anthropic/claude-sonnet-5"); }); + test("an Ollama offering's quirks carry that model's real context-window ceiling, not the built-in 4096 default", async () => { + const { log } = collector(); + const offeringBodies: Record[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "ollama") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "ollama-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const modelBody = body as { canonicalName: string }; + return { + status: 201, + data: catalogModelRow( + `mdl_${modelBody.canonicalName}`, + modelBody.canonicalName, + ), + }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow( + "cpv_1", + "ollama", + "cre_1", + "openai-compatible", + "http://127.0.0.1:1/v1", + ), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + offeringBodies.push(body as Record); + return { + status: 201, + data: catalogOfferingRow("off_1", "mdl_1", "cpv_1"), + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + provider: "ollama", + // Port 1 on loopback refuses instantly (nothing ever listens there), + // so this test's own `fetchOllamaModelCatalog` probe fails fast and + // falls back to the curated static seed, instead of depending on + // whatever Ollama instance (if any) happens to be reachable from + // wherever this test runs. + baseURLOverride: "http://127.0.0.1:1/v1", + apiKey: "unused", + log, + }); + + const gptOssDefault = OLLAMA_MODEL_DEFAULTS["gpt-oss:20b"]; + const qwenDefault = OLLAMA_MODEL_DEFAULTS["qwen3.8:27b"]; + if (gptOssDefault === undefined || qwenDefault === undefined) { + throw new Error("missing fixture entry"); + } + const gptOss = offeringBodies.find( + (entry) => entry["modelId"] === "mdl_gpt-oss:20b", + ); + expect(gptOss?.["quirks"]).toEqual({ default: gptOssDefault }); + const qwen = offeringBodies.find( + (entry) => entry["modelId"] === "mdl_qwen3.8:27b", + ); + expect(qwen?.["quirks"]).toEqual({ default: qwenDefault }); + expect( + (qwen?.["quirks"] as { default: { numCtx: number } })?.default?.numCtx, + ).toBeLessThan( + (gptOss?.["quirks"] as { default: { numCtx: number } })?.default?.numCtx, + ); + }); + test("an oauth_token credential with metadata posts both through to the credential row", async () => { const { log } = collector(); let credentialBody: unknown; diff --git a/packages/inference-catalog/package.json b/packages/inference-catalog/package.json index 7764d34a8..2bf187615 100644 --- a/packages/inference-catalog/package.json +++ b/packages/inference-catalog/package.json @@ -10,6 +10,7 @@ "./concepts": "./src/concepts.ts", "./resolve-chain": "./src/resolve-chain.ts", "./offering-capabilities": "./src/offering-capabilities.ts", + "./ollama-context-defaults": "./src/ollama-context-defaults.ts", "./migrations": "./src/migrations.ts" }, "scripts": { @@ -17,6 +18,7 @@ "test": "bun test" }, "dependencies": { + "@corbits/ollama-adapter": "workspace:*", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", "@intx/inference-catalog": "0.3.0", diff --git a/packages/inference-catalog/src/ollama-context-defaults.test.ts b/packages/inference-catalog/src/ollama-context-defaults.test.ts new file mode 100644 index 000000000..e002c4b9c --- /dev/null +++ b/packages/inference-catalog/src/ollama-context-defaults.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, test } from "bun:test"; +import { createOllamaAdapter } from "@corbits/ollama-adapter"; +import type { LastCycleSource } from "@intx/types/runtime"; +import type { ConversationTurn, InferenceOptions } from "@intx/types/runtime"; + +import { + OLLAMA_MODEL_DEFAULTS, + quirksForDeployment, +} from "./ollama-context-defaults"; + +describe("quirksForDeployment", () => { + test("a curated Ollama model resolves its advertised native context window", () => { + const quirks = quirksForDeployment({ + providerName: "ollama", + canonicalName: "gpt-oss:20b", + }); + const gptOssDefault = OLLAMA_MODEL_DEFAULTS["gpt-oss:20b"]; + if (gptOssDefault === undefined) throw new Error("missing fixture entry"); + expect(quirks).toEqual({ default: gptOssDefault }); + expect(quirks?.default?.numCtx).toBeGreaterThanOrEqual(131_072); + }); + + test("a model with a smaller real ceiling gets its own limit, not the larger model's", () => { + const bigModel = quirksForDeployment({ + providerName: "ollama", + canonicalName: "llama3.1:8b", + }); + const smallerModel = quirksForDeployment({ + providerName: "ollama", + canonicalName: "qwen3.8:27b", + }); + expect(bigModel?.default?.numCtx).toBe(131_072); + expect(smallerModel?.default?.numCtx).toBe(32_768); + expect(smallerModel?.default?.numCtx).toBeLessThan( + bigModel?.default?.numCtx ?? 0, + ); + }); + + test("a caller-supplied override wins over the built-in table", () => { + const quirks = quirksForDeployment( + { providerName: "ollama", canonicalName: "gpt-oss:20b" }, + { "gpt-oss:20b": { numCtx: 8192 } }, + ); + expect(quirks).toEqual({ default: { numCtx: 8192 } }); + }); + + test("a model absent from the table gets no override, never a guessed number", () => { + const quirks = quirksForDeployment({ + providerName: "ollama", + canonicalName: "some-future-model:1b", + }); + expect(quirks).toBeUndefined(); + }); + + test("a non-Ollama provider is left untouched even on the same openai-compatible wire", () => { + const quirks = quirksForDeployment({ + providerName: "groq", + canonicalName: "gpt-oss:20b", + }); + expect(quirks).toBeUndefined(); + }); +}); + +describe("the resolved quirks reach the built Ollama request body", () => { + const source: LastCycleSource = { + sourceId: "ollama-test", + provider: "ollama", + model: "gpt-oss:20b", + }; + const messages: ConversationTurn[] = [ + { role: "user", content: [{ type: "text", text: "hello" }], timestamp: 0 }, + ]; + const options: InferenceOptions = {}; + + test("gpt-oss:20b's request carries its 128K context window, not the built-in 4096 default", () => { + const quirks = quirksForDeployment({ + providerName: "ollama", + canonicalName: "gpt-oss:20b", + }); + const adapter = createOllamaAdapter(source, quirks); + const built = adapter.buildRequest(messages, "gpt-oss:20b", options); + const body = JSON.parse(built.body) as Record; + expect(body["options"]).toEqual({ num_ctx: 131_072 }); + expect(body["max_tokens"]).toBe(32_768); + expect(body["max_tokens"]).not.toBe(4096); + }); + + test("qwen3.8:27b's request carries its own smaller real ceiling", () => { + const quirks = quirksForDeployment({ + providerName: "ollama", + canonicalName: "qwen3.8:27b", + }); + const adapter = createOllamaAdapter(source, quirks); + const built = adapter.buildRequest(messages, "qwen3.8:27b", options); + const body = JSON.parse(built.body) as Record; + expect(body["options"]).toEqual({ num_ctx: 32_768 }); + }); +}); diff --git a/packages/inference-catalog/src/ollama-context-defaults.ts b/packages/inference-catalog/src/ollama-context-defaults.ts new file mode 100644 index 000000000..4109eece9 --- /dev/null +++ b/packages/inference-catalog/src/ollama-context-defaults.ts @@ -0,0 +1,81 @@ +// Per-model context-window and output-token defaults for locally served +// Ollama models, resolved the same way `capabilitiesForDeployment` resolves +// capabilities: from the deployment's identity, at the moment an offering +// is created. +// +// `@intx/inference`'s built-in OpenAI-shaped adapter defaults output tokens +// to `options.maxTokens ?? 4096` when a caller sets none, and Ollama's own +// openai-compatible endpoint silently defaults `options.num_ctx` (its real +// context window) to a small built-in size when nothing sets it either -- +// truncating real conversations for every locally served chat agent, with +// no error to say so. `@corbits/ollama-adapter`'s `OllamaAdapterConfig` is +// where that override actually lands on the built request body (see its +// `resolveOverride` and `createOllamaAdapter`); this module supplies the +// real per-model values it should carry, stored as a `model_offering`'s +// `quirks` column so it reaches `InferenceSource.quirks` -- and from there +// the adapter -- through the platform's existing resolution, no parallel +// override path of its own. +// +// Each table entry is the model's advertised *native* context window, not a +// YaRN/rope-extended ceiling: requesting a `num_ctx` past what a model (and +// the host's memory) can actually back can fail allocation or force heavy +// swap on the inference host, a real operational risk rather than a +// convention to enforce. A model absent from this table gets no override at +// all -- Ollama's own model-specific default stands, rather than this +// module guessing at a ceiling it cannot back up. +import type { + OllamaAdapterConfig, + OllamaAdapterOverride, +} from "@corbits/ollama-adapter"; + +export type OllamaModelDefaults = Readonly< + Record +>; + +export const OLLAMA_MODEL_DEFAULTS: OllamaModelDefaults = { + // OpenAI gpt-oss: 128K native context window. + "gpt-oss:20b": { numCtx: 131_072, maxOutputTokens: 32_768 }, + // Qwen3's 27B/30B-class models: 32K native context window. Reaching 128K + // on this family requires YaRN rope scaling, which this table does not + // treat as "genuinely" supported per-model context. + "qwen3.8:27b": { numCtx: 32_768, maxOutputTokens: 8192 }, + "qwen3.5:9b-mlx": { numCtx: 32_768, maxOutputTokens: 8192 }, + // Meta Llama 3.1: 128K native context window. + "llama3.1:8b": { numCtx: 131_072, maxOutputTokens: 8192 }, +}; + +export type OllamaDeploymentIdentity = { + readonly providerName: string; + readonly canonicalName: string; +}; + +/** + * The `quirks` value to store on a newly created Ollama offering. Each + * offering is already scoped to one exact model, so the resolved override + * is wrapped in `default` -- the field `resolveOverride` applies whenever + * no `perModel` entry is present -- rather than `perModel`, which exists + * for a single shared connection serving several models through one quirks + * bag. + * + * `overrides` lets a caller's own probed or operator-configured ceilings + * win over this table (the "maximum flexibility, sane by default" seam); + * unset resolves to this module's built-in table alone. + * + * Returns `undefined` for any provider other than `ollama` (this + * mechanism's whole scope -- other providers' built-in adapters handle + * their own request shape unmodified) and for any model neither `overrides` + * nor the built-in table names, so an unvetted model is left exactly as + * the built-in adapter already handles it rather than getting a guessed + * ceiling. + */ +export function quirksForDeployment( + deployment: OllamaDeploymentIdentity, + overrides: OllamaModelDefaults = {}, +): OllamaAdapterConfig | undefined { + if (deployment.providerName !== "ollama") return undefined; + const resolved = + overrides[deployment.canonicalName] ?? + OLLAMA_MODEL_DEFAULTS[deployment.canonicalName]; + if (resolved === undefined) return undefined; + return { default: resolved }; +}