diff --git a/packages/hub-client/src/catalog-seed-data.ts b/packages/hub-client/src/catalog-seed-data.ts index 0ad37a43a..742bdfd3d 100644 --- a/packages/hub-client/src/catalog-seed-data.ts +++ b/packages/hub-client/src/catalog-seed-data.ts @@ -217,10 +217,13 @@ export const CATALOG_SEEDS: Readonly< baseURL: "http://localhost:11434/v1", }, // Curated against a real instance's `/api/tags` (see this ticket's - // report): whichever of these two the person's own Ollama actually - // has loaded, `qwen3.8:27b` leads since it is the one confirmed to - // serve tool calls and thinking. + // report): whichever of these the person's own Ollama actually has + // loaded. `gpt-oss:20b` leads — CL-6477's overnight run confirmed it + // correct and tool-obedient, and 12-25x faster than `qwen3.8:27b` (8s + // vs 100-216s for the same task) — with `qwen3.8:27b` kept next as + // the prior confirmed-good default and `qwen3.5:9b-mlx` last. models: [ + { canonicalName: "gpt-oss:20b", displayName: "GPT-OSS 20B (Ollama)" }, { canonicalName: "qwen3.8:27b", displayName: "Qwen 3.8 27B (Ollama)" }, { canonicalName: "qwen3.5:9b-mlx", diff --git a/packages/hub-client/src/model-capability.test.ts b/packages/hub-client/src/model-capability.test.ts index 56cfde9ec..e89ccf483 100644 --- a/packages/hub-client/src/model-capability.test.ts +++ b/packages/hub-client/src/model-capability.test.ts @@ -50,6 +50,51 @@ describe("preferCompletionCapable", () => { [], ); }); + + // CL-6477: "embeddinggemma" has no delimiter between "embedding" and + // "gemma", unlike every other embedding model name this filter has seen + // (nomic-embed-text, all-minilm, bge-m3, qwen3-embedding all delimit + // with "-" or "_"). A regex that required a trailing delimiter after + // "embed(ding)?" let it through, where it then won the alphabetical + // default-model tiebreak and answered every chat turn with "does not + // support chat". + test("drops an uncataloged embeddinggemma offering even with no delimiter after 'embedding'", () => { + const offerings = [noData("embeddinggemma:300m"), completion("qwen3:8b")]; + expect( + preferCompletionCapable(offerings, capabilitiesOf, nameOf).map( + (o) => o.name, + ), + ).toEqual(["qwen3:8b"]); + }); + + test("still filters the delimited embedding-model names (no regression)", () => { + const offerings = [ + noData("nomic-embed-text"), + noData("all-minilm"), + noData("bge-m3"), + noData("qwen3-embedding"), + completion("qwen3:8b"), + ]; + expect( + preferCompletionCapable(offerings, capabilitiesOf, nameOf).map( + (o) => o.name, + ), + ).toEqual(["qwen3:8b"]); + }); + + test("prefers real capability data over the name fallback: a probed completion offering named like an embedding model is kept", () => { + const probedCompletion: Offering = { + name: "embeddinggemma:300m", + capabilities: ["plain-text"], + }; + expect( + preferCompletionCapable( + [probedCompletion, noData("all-minilm")], + capabilitiesOf, + nameOf, + ).map((o) => o.name), + ).toEqual(["embeddinggemma:300m"]); + }); }); describe("hasCompletionCapableModel", () => { @@ -72,4 +117,24 @@ describe("hasCompletionCapableModel", () => { ), ).toBe(false); }); + + test("false when the only offering is an uncataloged embeddinggemma pull (CL-6477)", () => { + expect( + hasCompletionCapableModel( + [noData("embeddinggemma:300m")], + capabilitiesOf, + nameOf, + ), + ).toBe(false); + }); + + test("a chat-capable model is selected as the default when both an embeddinggemma pull and a chat model are present", () => { + const offerings = [noData("embeddinggemma:300m"), completion("qwen3:8b")]; + const chatDefault = preferCompletionCapable( + offerings, + capabilitiesOf, + nameOf, + )[0]; + expect(chatDefault?.name).toBe("qwen3:8b"); + }); }); diff --git a/packages/hub-client/src/model-capability.ts b/packages/hub-client/src/model-capability.ts index 2d3b17c0b..5c7a59051 100644 --- a/packages/hub-client/src/model-capability.ts +++ b/packages/hub-client/src/model-capability.ts @@ -23,9 +23,24 @@ // case this falls back to recognizing common embedding-model name // families by convention — the same signal CL-6351's first pass used // before the pinned catalog existed — rather than trusting an empty list -// as "no data" and letting an embedding model win anyway. +// as "no data" and letting an embedding model win anyway. Capability data +// is always tried first; the name check below only ever runs as a +// belt-and-braces fallback for the uncataloged case. +// +// CL-6477: `embed(ding)?` carries no required delimiter after it, unlike +// the other families here. Google's `embeddinggemma:300m` — an +// uncataloged Ollama pull with no capability data, same as any other +// local embedding model — has no delimiter between "embedding" and the +// "gemma" model family name, so a trailing-delimiter requirement lets it +// slip past the filter entirely and then win the alphabetical +// default-model tiebreak with a model that answers every chat turn with +// "does not support chat" (CL-6351 reopened through a name its own fix +// missed). The other short abbreviations here (`minilm`, `bge`, `gte`, +// `e5`, `arctic-embed`) keep a required trailing delimiter: they are +// short enough that matching them as a bare substring risks false +// positives no real model name has exercised yet. const EMBEDDING_MODEL_NAME_PATTERN = - /(^|[-_/])(embed(ding)?|minilm|bge|gte|e5|arctic-embed)(-|_|:|$)/i; + /(^|[-_/])embed(ding)?|(^|[-_/])(minilm|bge|gte|e5|arctic-embed)(-|_|:|$)/i; function isEmbeddingModelName(canonicalName: string): boolean { return EMBEDDING_MODEL_NAME_PATTERN.test(canonicalName); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index e264ab886..cf2acd4dd 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -149,9 +149,8 @@ describe("modelSourceFor", () => { }); // CL-6366 red/green: a fresh instance whose live catalog carries only - // llama3.2 (the curated `qwen3.8:27b` name is absent entirely) still - // resolves to what the instance actually serves, never a pin it can't - // answer for. + // llama3.2 (the curated default name is absent entirely) still resolves + // to what the instance actually serves, never a pin it can't answer for. test("ollama resolves to the instance's own seeded model, never the curated pin it may lack", async () => { const api: ApiCall = async (method, path) => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { @@ -218,7 +217,7 @@ describe("modelSourceFor", () => { capabilities: ["plain-text"], }, { - canonicalName: "qwen3.8:27b", + canonicalName: "gpt-oss:20b", providerName: "ollama", capabilities: ["plain-text"], }, @@ -234,7 +233,7 @@ describe("modelSourceFor", () => { "ollama", "ollama", ); - expect(result.model).toBe("qwen3.8:27b"); + expect(result.model).toBe("gpt-oss:20b"); }); test("ollama's baseURLOverride is normalized to the /v1 form", async () => {