Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions packages/hub-client/src/catalog-seed-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -217,10 +217,13 @@ export const CATALOG_SEEDS: Readonly<
baseURL: "http://localhost:11434/v1",
},
// Curated against a real instance's `/api/tags` (see this ticket's
// report): whichever of these two the person's own Ollama actually
// has loaded, `qwen3.8:27b` leads since it is the one confirmed to
// serve tool calls and thinking.
// report): whichever of these the person's own Ollama actually has
// loaded. `gpt-oss:20b` leads — CL-6477's overnight run confirmed it
// correct and tool-obedient, and 12-25x faster than `qwen3.8:27b` (8s
// vs 100-216s for the same task) — with `qwen3.8:27b` kept next as
// the prior confirmed-good default and `qwen3.5:9b-mlx` last.
models: [
{ canonicalName: "gpt-oss:20b", displayName: "GPT-OSS 20B (Ollama)" },
{ canonicalName: "qwen3.8:27b", displayName: "Qwen 3.8 27B (Ollama)" },
{
canonicalName: "qwen3.5:9b-mlx",
Expand Down
65 changes: 65 additions & 0 deletions packages/hub-client/src/model-capability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,51 @@ describe("preferCompletionCapable", () => {
[],
);
});

// CL-6477: "embeddinggemma" has no delimiter between "embedding" and
// "gemma", unlike every other embedding model name this filter has seen
// (nomic-embed-text, all-minilm, bge-m3, qwen3-embedding all delimit
// with "-" or "_"). A regex that required a trailing delimiter after
// "embed(ding)?" let it through, where it then won the alphabetical
// default-model tiebreak and answered every chat turn with "does not
// support chat".
test("drops an uncataloged embeddinggemma offering even with no delimiter after 'embedding'", () => {
const offerings = [noData("embeddinggemma:300m"), completion("qwen3:8b")];
expect(
preferCompletionCapable(offerings, capabilitiesOf, nameOf).map(
(o) => o.name,
),
).toEqual(["qwen3:8b"]);
});

test("still filters the delimited embedding-model names (no regression)", () => {
const offerings = [
noData("nomic-embed-text"),
noData("all-minilm"),
noData("bge-m3"),
noData("qwen3-embedding"),
completion("qwen3:8b"),
];
expect(
preferCompletionCapable(offerings, capabilitiesOf, nameOf).map(
(o) => o.name,
),
).toEqual(["qwen3:8b"]);
});

test("prefers real capability data over the name fallback: a probed completion offering named like an embedding model is kept", () => {
const probedCompletion: Offering = {
name: "embeddinggemma:300m",
capabilities: ["plain-text"],
};
expect(
preferCompletionCapable(
[probedCompletion, noData("all-minilm")],
capabilitiesOf,
nameOf,
).map((o) => o.name),
).toEqual(["embeddinggemma:300m"]);
});
});

describe("hasCompletionCapableModel", () => {
Expand All @@ -72,4 +117,24 @@ describe("hasCompletionCapableModel", () => {
),
).toBe(false);
});

test("false when the only offering is an uncataloged embeddinggemma pull (CL-6477)", () => {
expect(
hasCompletionCapableModel(
[noData("embeddinggemma:300m")],
capabilitiesOf,
nameOf,
),
).toBe(false);
});

test("a chat-capable model is selected as the default when both an embeddinggemma pull and a chat model are present", () => {
const offerings = [noData("embeddinggemma:300m"), completion("qwen3:8b")];
const chatDefault = preferCompletionCapable(
offerings,
capabilitiesOf,
nameOf,
)[0];
expect(chatDefault?.name).toBe("qwen3:8b");
});
});
19 changes: 17 additions & 2 deletions packages/hub-client/src/model-capability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,24 @@
// case this falls back to recognizing common embedding-model name
// families by convention — the same signal CL-6351's first pass used
// before the pinned catalog existed — rather than trusting an empty list
// as "no data" and letting an embedding model win anyway.
// as "no data" and letting an embedding model win anyway. Capability data
// is always tried first; the name check below only ever runs as a
// belt-and-braces fallback for the uncataloged case.
//
// CL-6477: `embed(ding)?` carries no required delimiter after it, unlike
// the other families here. Google's `embeddinggemma:300m` — an
// uncataloged Ollama pull with no capability data, same as any other
// local embedding model — has no delimiter between "embedding" and the
// "gemma" model family name, so a trailing-delimiter requirement lets it
// slip past the filter entirely and then win the alphabetical
// default-model tiebreak with a model that answers every chat turn with
// "does not support chat" (CL-6351 reopened through a name its own fix
// missed). The other short abbreviations here (`minilm`, `bge`, `gte`,
// `e5`, `arctic-embed`) keep a required trailing delimiter: they are
// short enough that matching them as a bare substring risks false
// positives no real model name has exercised yet.
const EMBEDDING_MODEL_NAME_PATTERN =
/(^|[-_/])(embed(ding)?|minilm|bge|gte|e5|arctic-embed)(-|_|:|$)/i;
/(^|[-_/])embed(ding)?|(^|[-_/])(minilm|bge|gte|e5|arctic-embed)(-|_|:|$)/i;

function isEmbeddingModelName(canonicalName: string): boolean {
return EMBEDDING_MODEL_NAME_PATTERN.test(canonicalName);
Expand Down
9 changes: 4 additions & 5 deletions packages/onboarding/test/complete-credential.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,8 @@ describe("modelSourceFor", () => {
});

// CL-6366 red/green: a fresh instance whose live catalog carries only
// llama3.2 (the curated `qwen3.8:27b` name is absent entirely) still
// resolves to what the instance actually serves, never a pin it can't
// answer for.
// llama3.2 (the curated default name is absent entirely) still resolves
// to what the instance actually serves, never a pin it can't answer for.
test("ollama resolves to the instance's own seeded model, never the curated pin it may lack", async () => {
const api: ApiCall = async (method, path) => {
if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) {
Expand Down Expand Up @@ -218,7 +217,7 @@ describe("modelSourceFor", () => {
capabilities: ["plain-text"],
},
{
canonicalName: "qwen3.8:27b",
canonicalName: "gpt-oss:20b",
providerName: "ollama",
capabilities: ["plain-text"],
},
Expand All @@ -234,7 +233,7 @@ describe("modelSourceFor", () => {
"ollama",
"ollama",
);
expect(result.model).toBe("qwen3.8:27b");
expect(result.model).toBe("gpt-oss:20b");
});

test("ollama's baseURLOverride is normalized to the /v1 form", async () => {
Expand Down
Loading