diff --git a/controlplane/src/db/schema.ts b/controlplane/src/db/schema.ts index e1950074..8697d70b 100644 --- a/controlplane/src/db/schema.ts +++ b/controlplane/src/db/schema.ts @@ -1148,6 +1148,32 @@ CREATE TABLE IF NOT EXISTS link_edge_map ( PRIMARY KEY (link_id, edge_a_id) ); CREATE UNIQUE INDEX IF NOT EXISTS idx_link_edge_b ON link_edge_map(link_id, edge_b_id); + +-- Blind preference ballots for the open-weight TTS benchmark. A ballot is issued +-- server-side (so the client cannot choose its own comparison set), then filled +-- in by ranking the items best to worst. +-- +-- anchor_config is a deliberately weak engine seeded into some ballots: a ballot +-- that ranks it first is noise or gaming, and is stored with status 'rejected' +-- so it is auditable rather than silently dropped. +-- +-- NOTE for future tables: this belongs HERE, in the SCHEMA constant that +-- initializeDatabase executes. blog_subscribers was appended to the bottom of a +-- conditional migration function instead, which returns early on any migrated +-- database, so that table can never be created by /init-db. +CREATE TABLE IF NOT EXISTS tts_ballots ( + id TEXT PRIMARY KEY, + run TEXT NOT NULL, + items_json TEXT NOT NULL, + anchor_config TEXT, + ranking_json TEXT, + status TEXT NOT NULL DEFAULT 'issued' CHECK (status IN ('issued','counted','rejected')), + voter_hash TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + submitted_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_tts_ballots_status ON tts_ballots(status, run); +CREATE INDEX IF NOT EXISTS idx_tts_ballots_voter ON tts_ballots(voter_hash, created_at); `; // Initialize the database diff --git a/controlplane/src/index.ts b/controlplane/src/index.ts index f49c9f4d..7c8c1cef 100644 --- a/controlplane/src/index.ts +++ b/controlplane/src/index.ts @@ -52,6 +52,7 @@ import { createApiToken, listApiTokens, revokeApiToken, revokeSelfApiToken } fro import { checkAndCacheSandbоxHealth, getCachedHealth } from './health/checker'; import { sendEmail, buildInterestThankYouEmail, buildInterestNotificationEmail, buildTemplateReviewEmail } from './email/resend'; import * as blog from './blog/handler'; +import * as tts from './tts/handler'; import * as releases from './releases/handler'; import { sandboxHeaders, sandboxUrl } from './sandbox/fetch'; @@ -994,6 +995,7 @@ async function handleRequest(request: Request, env: EnvWithBindings, ctx: Pick(a: T[]): T[] { + const out = [...a]; + for (let i = out.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [out[i], out[j]] = [out[j], out[i]]; + } + return out; +} + +/** Coarse voter fingerprint for rate limiting and dedup. Hashed, never stored raw. */ +async function voterHash(request: Request): Promise { + const ip = request.headers.get('cf-connecting-ip') ?? request.headers.get('x-forwarded-for') ?? ''; + const ua = request.headers.get('user-agent') ?? ''; + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(`${ip}|${ua}`)); + return [...new Uint8Array(digest)].slice(0, 16).map((b) => b.toString(16).padStart(2, '0')).join(''); +} + +/** What this voter has already done. Both limits below are enforced here rather + * than in the client, which cannot be trusted to count its own votes. */ +async function voterHistory(env: Env, voter: string) { + const { results } = await env.DB.prepare( + `SELECT items_json, status FROM tts_ballots + WHERE voter_hash = ? AND run = ? AND status IN ('counted','rejected')` + ).bind(voter, RUN).all<{ items_json: string; status: string }>(); + + // Only submitted ballots count. An issued-then-abandoned one must not burn an + // attempt or retire four clips, or closing the dialog would quietly cost the + // reader a vote. + const heard = new Set(); + for (const r of results ?? []) { + try { for (const c of JSON.parse(r.items_json)) heard.add(c); } catch { /* skip */ } + } + return { submitted: (results ?? []).length, heard }; +} + +/** GET /tts/ballot - issue a blind comparison set. */ +export async function issueBallot(env: Env, request: Request): Promise { + const voter = await voterHash(request); + const { submitted, heard } = await voterHistory(env, voter); + + if (submitted >= MAX_BALLOTS_PER_VOTER) { + return Response.json({ exhausted: 'limit', submitted, max: MAX_BALLOTS_PER_VOTER }); + } + + // Nothing already ranked comes back. A second opinion on the same clip from + // the same ears is not a second data point, and Bradley-Terry would treat it + // as one. + // Widened to string[]: CONFIGS is a literal tuple, and ANCHORS is not. + const fresh: string[] = CONFIGS.filter((c) => !heard.has(c)); + if (fresh.length < 2) { + return Response.json({ exhausted: 'clips', submitted, max: MAX_BALLOTS_PER_VOTER }); + } + + // The anchor has to be one they have not heard either, or the attention check + // is just a memory test. + const freshAnchors = ANCHORS.filter((a) => fresh.includes(a)); + const anchor = + freshAnchors.length && Math.random() < ANCHOR_RATE + ? freshAnchors[Math.floor(Math.random() * freshAnchors.length)] + : null; + + const pool = fresh.filter((c) => c !== anchor); + const want = Math.min(ITEMS_PER_BALLOT, fresh.length) - (anchor ? 1 : 0); + const picked = shuffle(pool).slice(0, want); + const items = shuffle(anchor ? [...picked, anchor] : picked); + + const id = crypto.randomUUID(); + await env.DB.prepare( + `INSERT INTO tts_ballots (id, run, items_json, anchor_config, voter_hash) VALUES (?, ?, ?, ?, ?)` + ).bind(id, RUN, JSON.stringify(items), anchor, voter).run(); + + // The anchor is deliberately NOT returned: the client must not be able to + // tell which item is the attention check. + return Response.json({ ballotId: id, items, submitted, max: MAX_BALLOTS_PER_VOTER }); +} + +/** POST /tts/ballot - record a ranking, best first. */ +export async function submitBallot( + env: Env, + request: Request, + body: { ballotId?: string; ranking?: string[] } +): Promise { + const { ballotId, ranking } = body; + if (!ballotId || !Array.isArray(ranking)) { + return Response.json({ error: 'E79601: ballotId and ranking are required' }, { status: 400 }); + } + + const row = await env.DB.prepare( + `SELECT items_json, anchor_config, status, voter_hash FROM tts_ballots WHERE id = ?` + ).bind(ballotId).first<{ items_json: string; anchor_config: string | null; status: string; voter_hash: string }>(); + + if (!row) return Response.json({ error: 'E79602: Unknown ballot' }, { status: 404 }); + + // Checked again at submit, not only at issue. Issuing is cheap and + // unauthenticated, so anyone can hold several open ballots and submit them + // all; the cap has to be applied where the vote is actually recorded. + const { submitted } = await voterHistory(env, row.voter_hash); + if (submitted >= MAX_BALLOTS_PER_VOTER) { + return Response.json( + { error: `E79605: This voter has already submitted ${MAX_BALLOTS_PER_VOTER} ballots`, code: 'BALLOT_LIMIT' }, + { status: 409 } + ); + } + if (row.status !== 'issued') { + return Response.json({ error: 'E79603: Ballot already submitted' }, { status: 409 }); + } + + // The ranking must be a permutation of exactly the items issued, so a client + // cannot inject configurations it was not shown. + const items: string[] = JSON.parse(row.items_json); + const same = + ranking.length === items.length && + [...ranking].sort().join('|') === [...items].sort().join('|'); + if (!same) { + return Response.json({ error: 'E79604: Ranking must order exactly the issued items' }, { status: 400 }); + } + + // Attention check: the seeded weak engine placed first means the ballot is + // noise or gaming. Stored, not deleted, so the rejection rate is auditable. + const failed = row.anchor_config !== null && ranking[0] === row.anchor_config; + await env.DB.prepare( + `UPDATE tts_ballots SET ranking_json = ?, status = ?, submitted_at = datetime('now') WHERE id = ?` + ).bind(JSON.stringify(ranking), failed ? 'rejected' : 'counted', ballotId).run(); + + return Response.json({ ok: true, counted: !failed }); +} + +interface Score { + config: string; + rating: number | null; + ballots: number; + wins: number; + comparisons: number; +} + +/** + * GET /tts/scores - Bradley-Terry ratings over the counted ballots. + * + * A 4-way ranking is six pairwise outcomes. Bradley-Terry is used rather than a + * mean rank because engines do not all face the same opponents: mean rank + * rewards whoever happened to draw weak company, while BT solves for the + * strength that best explains who beat whom. + */ +export async function getScores(env: Env): Promise { + const { results } = await env.DB.prepare( + `SELECT ranking_json FROM tts_ballots WHERE status = 'counted' AND run = ?` + ).bind(RUN).all<{ ranking_json: string }>(); + + const wins = new Map>(); + const ballots = new Map(); + const winCount = new Map(); + const comparisons = new Map(); + const bump = (m: Map, k: string, n = 1) => m.set(k, (m.get(k) ?? 0) + n); + + for (const r of results ?? []) { + let order: string[]; + try { order = JSON.parse(r.ranking_json); } catch { continue; } + for (const c of order) bump(ballots, c); + for (let i = 0; i < order.length; i++) { + for (let j = i + 1; j < order.length; j++) { + const better = order[i], worse = order[j]; + if (!wins.has(better)) wins.set(better, new Map()); + bump(wins.get(better)!, worse); + bump(winCount, better); + bump(comparisons, better); + bump(comparisons, worse); + } + } + } + + // Bradley-Terry by MM iteration. Converges quickly at this size; the loop is + // bounded so a pathological matrix cannot hang the request. + const players = [...ballots.keys()]; + const strength = new Map(players.map((p) => [p, 1])); + const beat = (a: string, b: string) => wins.get(a)?.get(b) ?? 0; + for (let iter = 0; iter < 200; iter++) { + let maxDelta = 0; + for (const p of players) { + let num = 0, den = 0; + for (const q of players) { + if (p === q) continue; + const pq = beat(p, q), qp = beat(q, p), n = pq + qp; + if (!n) continue; + num += pq; + den += n / (strength.get(p)! + strength.get(q)!); + } + if (den > 0 && num > 0) { + const next = num / den; + maxDelta = Math.max(maxDelta, Math.abs(next - strength.get(p)!)); + strength.set(p, next); + } + } + if (maxDelta < 1e-9) break; + } + + // Present as 0-100 with 50 at the field's geometric mean, via a logistic of + // the log-strength. A linear map is unbounded: on a well-separated field it + // produced ratings above 130 and below zero, which reads as broken in a table. + // The logistic cannot leave the range however far apart the engines are, and + // is still monotonic in strength, so the ordering is unchanged. + const shown = players.filter((p) => (ballots.get(p) ?? 0) >= MIN_BALLOTS_TO_SHOW); + const logs = shown.map((p) => Math.log(strength.get(p)!)); + const mean = logs.length ? logs.reduce((a, b) => a + b, 0) / logs.length : 0; + const rate = (s: number) => Math.round((100 / (1 + Math.exp(-(Math.log(s) - mean)))) * 10) / 10; + + const scores: Score[] = CONFIGS.map((c) => { + const n = ballots.get(c) ?? 0; + return { + config: c, + rating: n >= MIN_BALLOTS_TO_SHOW ? rate(strength.get(c) ?? 1) : null, + ballots: n, + wins: winCount.get(c) ?? 0, + comparisons: comparisons.get(c) ?? 0, + }; + }); + + const counted = (results ?? []).length; + return Response.json( + { run: RUN, countedBallots: counted, minBallots: MIN_BALLOTS_TO_SHOW, scores }, + { headers: { 'Cache-Control': 'public, max-age=60' } } + ); +} diff --git a/frontend/content/benchmarks/open-weight-tts.md b/frontend/content/benchmarks/open-weight-tts.md new file mode 100644 index 00000000..a95a19a1 --- /dev/null +++ b/frontend/content/benchmarks/open-weight-tts.md @@ -0,0 +1,99 @@ +--- +title: Testing open-weight Text To Speech models +date: 2026-08-17 +description: Every open-weight TTS engine that can run on a consumer laptop, tested for WER using Whisper. +author: Rob Macrae +toc: false +--- + +We tested all the open weight TTS models against the same corpus, using Whisper medium to transcribe back to text. + +Every row can be played, so you can hear what a word error rate actually sounds like. + +```chart +tts-vendors +``` + +## Results + +Sorted by compute per phrase, fastest first. Click any column to re-sort. + +```chart +tts-results +``` + +```chart +tts-error-cost +``` + +```chart +tts-preference +``` + +## How to read each column + +The measurements are not interchangeable, and two of them actively mislead if taken at +face value. + +- **RTF** is compute seconds per second of audio, the conventional measure. It **flatters + any engine that emits excess silence**, because padding inflates the denominator. Prefer + **x̄ synth**, which is compute per phrase: every engine speaks the same corpus, so it + compares directly and cannot be gamed by padding. +- **WER** is the round-trip word error rate, scored by Whisper `base.en`. The stronger + `medium.en` was run too, but it did not finish for twenty-one of the thirty-six + configurations, so `base.en` is the one recogniser that covers the whole table. Being the + weaker model it hallucinates words onto trailing silence, which understates good engines + more than bad ones — so the true spread between engines is **wider** than this column + shows, not narrower. +- **PESQ** is a no-reference perceptual quality estimate from torchaudio's SQUIM, scored on + the sample in the first column. It is a second axis word error cannot see, because word + error saturates once speech is merely intelligible. It measures signal quality, not + naturalness: 2E and Qwen3-TTS sit within 0.01 of each other while sounding clearly + different. +- **Class** describes the architecture. `det-ff` is a single forward pass with a + deterministic duration predictor, so timing is identical every run and it cannot + hallucinate. `st-ff` is the same shape with sampling inside, so output length varies. + `ar-lm` samples audio tokens one at a time: length is emergent, cloning and emotion + become possible, and real-time factor is floored by sequential decoding regardless of + quantization. +- **RSS** is peak resident memory, and is dominated by the runtime rather than the model. Engines served by the C++ + binary carry no interpreter; those running in Python carry torch, transformers and their + dependency trees. + +## Methodology + +**Machine.** Apple M2, macOS. One machine, one English corpus, one recogniser family. + +**Nothing is filtered out.** Every configuration compared is listed, including the twelve +that cannot keep up with their own speech, because "how far off is it" is a real question +and a table that quietly omits the answer cannot be checked. The **Real time only** control +above the table hides anything above 2x for readers who only care about what can be spoken +live, and sorting by RTF or compute per phrase draws a line at the cutoff so the two groups +are visible at once. + +The line falls at 2x exactly. Nothing sits awkwardly against it: the slowest engine that +keeps up is OmniVoice at 1.51x, and the fastest that does not is Chatterbox Q4 at 2.03x. + +**Corpus.** 84 phrases, spoken identically by every configuration, spanning core, edge and +long-form categories. + +**Scoring.** Round-trip word error rate, capped at 1.0 per phrase so a single runaway +cannot swamp the mean. Leading silence is worth knowing about here: Whisper transcribes it +as a word ("You", "Thank you.") rather than returning nothing, so an engine that pads the +start of a clip manufactures insertion errors and scores worse than it sounds. PESQ is +scored on **trimmed** audio: scoring untrimmed rewards models that pad with silence, by up +to 0.78, because silence pulls the estimate toward the ceiling and only mediocre audio has +room to rise. + +**Passes.** Autoregressive engines are averaged over two passes, feed-forward over one. +Differences under roughly two points are not resolvable at this sample size. + +### Limitations + +Word error rate measures intelligibility and nothing else. It is blind to naturalness, +expressiveness and speaker similarity, which is precisely what the LM-backed engines exist +to provide, so this benchmark understates them by construction. Treat the table as a +shortlist for "will this be understood, and what will it cost me", and the play buttons as +the part that answers "does it sound any good". + +One machine also means the device-level findings generalise no further than Apple silicon. diff --git a/frontend/next.config.ts b/frontend/next.config.ts index d044fd2a..10cf835e 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -10,22 +10,38 @@ const nextConfig: NextConfig = { images: { unoptimized: true, }, + // There is no benchmarks index page. /benchmarks lands on the default + // benchmark itself, because an index that lists one thing and then makes you + // click again is a page nobody wants to read. The tabs on each benchmark do + // the navigating. + // + // Keep this in step with the FIRST entry of BENCHMARK_TABS in + // src/components/BenchmarkTabs.tsx — next.config cannot import it, so the + // default slug is duplicated here on purpose. + // // The Labs section became Benchmarks. /labs URLs are live on the internet // (shared links, social cards), so every one of them permanently redirects. // Order matters — Next matches top-down: // 1. the published Labs post -> its benchmark's permanent page - // 2. the section index + // 2. the section root -> the default benchmark, NOT via /benchmarks: that + // would be two hops for a link someone else already published // 3. everything else under /labs, which also covers the moved static assets // (e.g. /labs/og-do-skills.png, referenced by already-cached social cards) async redirects() { + const DEFAULT_BENCHMARK = "/benchmarks/agent-skills"; return [ { source: "/labs/do-skills-improve-coding-agent-accuracy", - destination: "/benchmarks/agent-skills", + destination: DEFAULT_BENCHMARK, permanent: true, }, - { source: "/labs", destination: "/benchmarks", permanent: true }, + { source: "/labs", destination: DEFAULT_BENCHMARK, permanent: true }, { source: "/labs/:path*", destination: "/benchmarks/:path*", permanent: true }, + // Temporary on purpose. /benchmarks is the URL people share and the one + // in the site nav, and which benchmark is the default will change as more + // land. A 308 would be cached by browsers and CDNs and strand readers on + // whichever benchmark happened to be first today. + { source: "/benchmarks", destination: DEFAULT_BENCHMARK, permanent: false }, ]; }, }; diff --git a/frontend/public/benchmarks/tts/bananamind-tts.mp3 b/frontend/public/benchmarks/tts/bananamind-tts.mp3 new file mode 100644 index 00000000..f0a4dbe9 Binary files /dev/null and b/frontend/public/benchmarks/tts/bananamind-tts.mp3 differ diff --git a/frontend/public/benchmarks/tts/bark.mp3 b/frontend/public/benchmarks/tts/bark.mp3 new file mode 100644 index 00000000..66a67e88 Binary files /dev/null and b/frontend/public/benchmarks/tts/bark.mp3 differ diff --git a/frontend/public/benchmarks/tts/chatterbox-q4.mp3 b/frontend/public/benchmarks/tts/chatterbox-q4.mp3 new file mode 100644 index 00000000..174e8b00 Binary files /dev/null and b/frontend/public/benchmarks/tts/chatterbox-q4.mp3 differ diff --git a/frontend/public/benchmarks/tts/chatterbox-q8.mp3 b/frontend/public/benchmarks/tts/chatterbox-q8.mp3 new file mode 100644 index 00000000..d14938de Binary files /dev/null and b/frontend/public/benchmarks/tts/chatterbox-q8.mp3 differ diff --git a/frontend/public/benchmarks/tts/chatterbox-turbo.mp3 b/frontend/public/benchmarks/tts/chatterbox-turbo.mp3 new file mode 100644 index 00000000..6b96b544 Binary files /dev/null and b/frontend/public/benchmarks/tts/chatterbox-turbo.mp3 differ diff --git a/frontend/public/benchmarks/tts/chatterbox.mp3 b/frontend/public/benchmarks/tts/chatterbox.mp3 new file mode 100644 index 00000000..401f37d8 Binary files /dev/null and b/frontend/public/benchmarks/tts/chatterbox.mp3 differ diff --git a/frontend/public/benchmarks/tts/cosyvoice3-rl.mp3 b/frontend/public/benchmarks/tts/cosyvoice3-rl.mp3 new file mode 100644 index 00000000..aac75e74 Binary files /dev/null and b/frontend/public/benchmarks/tts/cosyvoice3-rl.mp3 differ diff --git a/frontend/public/benchmarks/tts/cosyvoice3.mp3 b/frontend/public/benchmarks/tts/cosyvoice3.mp3 new file mode 100644 index 00000000..12fcc3d9 Binary files /dev/null and b/frontend/public/benchmarks/tts/cosyvoice3.mp3 differ diff --git a/frontend/public/benchmarks/tts/csm.mp3 b/frontend/public/benchmarks/tts/csm.mp3 new file mode 100644 index 00000000..514fc635 Binary files /dev/null and b/frontend/public/benchmarks/tts/csm.mp3 differ diff --git a/frontend/public/benchmarks/tts/dots-tts.mp3 b/frontend/public/benchmarks/tts/dots-tts.mp3 new file mode 100644 index 00000000..fa7e7634 Binary files /dev/null and b/frontend/public/benchmarks/tts/dots-tts.mp3 differ diff --git a/frontend/public/benchmarks/tts/f5-tts.mp3 b/frontend/public/benchmarks/tts/f5-tts.mp3 new file mode 100644 index 00000000..51708e38 Binary files /dev/null and b/frontend/public/benchmarks/tts/f5-tts.mp3 differ diff --git a/frontend/public/benchmarks/tts/fastpitch.mp3 b/frontend/public/benchmarks/tts/fastpitch.mp3 new file mode 100644 index 00000000..10d0ab4b Binary files /dev/null and b/frontend/public/benchmarks/tts/fastpitch.mp3 differ diff --git a/frontend/public/benchmarks/tts/kittentts-micro.mp3 b/frontend/public/benchmarks/tts/kittentts-micro.mp3 new file mode 100644 index 00000000..57a90f1d Binary files /dev/null and b/frontend/public/benchmarks/tts/kittentts-micro.mp3 differ diff --git a/frontend/public/benchmarks/tts/kittentts-mini.mp3 b/frontend/public/benchmarks/tts/kittentts-mini.mp3 new file mode 100644 index 00000000..e16b0176 Binary files /dev/null and b/frontend/public/benchmarks/tts/kittentts-mini.mp3 differ diff --git a/frontend/public/benchmarks/tts/kittentts-nano-int8.mp3 b/frontend/public/benchmarks/tts/kittentts-nano-int8.mp3 new file mode 100644 index 00000000..89af1eac Binary files /dev/null and b/frontend/public/benchmarks/tts/kittentts-nano-int8.mp3 differ diff --git a/frontend/public/benchmarks/tts/kittentts-nano.mp3 b/frontend/public/benchmarks/tts/kittentts-nano.mp3 new file mode 100644 index 00000000..5ee5ddda Binary files /dev/null and b/frontend/public/benchmarks/tts/kittentts-nano.mp3 differ diff --git a/frontend/public/benchmarks/tts/kokoro.mp3 b/frontend/public/benchmarks/tts/kokoro.mp3 new file mode 100644 index 00000000..b1eda118 Binary files /dev/null and b/frontend/public/benchmarks/tts/kokoro.mp3 differ diff --git a/frontend/public/benchmarks/tts/melotts.mp3 b/frontend/public/benchmarks/tts/melotts.mp3 new file mode 100644 index 00000000..82a1aa41 Binary files /dev/null and b/frontend/public/benchmarks/tts/melotts.mp3 differ diff --git a/frontend/public/benchmarks/tts/mms-tts.mp3 b/frontend/public/benchmarks/tts/mms-tts.mp3 new file mode 100644 index 00000000..88e97ab5 Binary files /dev/null and b/frontend/public/benchmarks/tts/mms-tts.mp3 differ diff --git a/frontend/public/benchmarks/tts/nt-2e-fp32-mps.mp3 b/frontend/public/benchmarks/tts/nt-2e-fp32-mps.mp3 new file mode 100644 index 00000000..ea62670c Binary files /dev/null and b/frontend/public/benchmarks/tts/nt-2e-fp32-mps.mp3 differ diff --git a/frontend/public/benchmarks/tts/nt-2e-q4-metal.mp3 b/frontend/public/benchmarks/tts/nt-2e-q4-metal.mp3 new file mode 100644 index 00000000..c457f86c Binary files /dev/null and b/frontend/public/benchmarks/tts/nt-2e-q4-metal.mp3 differ diff --git a/frontend/public/benchmarks/tts/nt-2e-q8-metal.mp3 b/frontend/public/benchmarks/tts/nt-2e-q8-metal.mp3 new file mode 100644 index 00000000..801dfebd Binary files /dev/null and b/frontend/public/benchmarks/tts/nt-2e-q8-metal.mp3 differ diff --git a/frontend/public/benchmarks/tts/omnivoice.mp3 b/frontend/public/benchmarks/tts/omnivoice.mp3 new file mode 100644 index 00000000..9187161c Binary files /dev/null and b/frontend/public/benchmarks/tts/omnivoice.mp3 differ diff --git a/frontend/public/benchmarks/tts/orpheus-q4.mp3 b/frontend/public/benchmarks/tts/orpheus-q4.mp3 new file mode 100644 index 00000000..e015365b Binary files /dev/null and b/frontend/public/benchmarks/tts/orpheus-q4.mp3 differ diff --git a/frontend/public/benchmarks/tts/piper.mp3 b/frontend/public/benchmarks/tts/piper.mp3 new file mode 100644 index 00000000..ced45e56 Binary files /dev/null and b/frontend/public/benchmarks/tts/piper.mp3 differ diff --git a/frontend/public/benchmarks/tts/qwen3-tts-vd.mp3 b/frontend/public/benchmarks/tts/qwen3-tts-vd.mp3 new file mode 100644 index 00000000..5f44cd3b Binary files /dev/null and b/frontend/public/benchmarks/tts/qwen3-tts-vd.mp3 differ diff --git a/frontend/public/benchmarks/tts/qwen3-tts.mp3 b/frontend/public/benchmarks/tts/qwen3-tts.mp3 new file mode 100644 index 00000000..6660edec Binary files /dev/null and b/frontend/public/benchmarks/tts/qwen3-tts.mp3 differ diff --git a/frontend/public/benchmarks/tts/speecht5.mp3 b/frontend/public/benchmarks/tts/speecht5.mp3 new file mode 100644 index 00000000..8378ecde Binary files /dev/null and b/frontend/public/benchmarks/tts/speecht5.mp3 differ diff --git a/frontend/public/benchmarks/tts/styletts2.mp3 b/frontend/public/benchmarks/tts/styletts2.mp3 new file mode 100644 index 00000000..5d9b4e77 Binary files /dev/null and b/frontend/public/benchmarks/tts/styletts2.mp3 differ diff --git a/frontend/public/benchmarks/tts/supertonic.mp3 b/frontend/public/benchmarks/tts/supertonic.mp3 new file mode 100644 index 00000000..ac81ab53 Binary files /dev/null and b/frontend/public/benchmarks/tts/supertonic.mp3 differ diff --git a/frontend/public/benchmarks/tts/tada-1b.mp3 b/frontend/public/benchmarks/tts/tada-1b.mp3 new file mode 100644 index 00000000..cdf872f3 Binary files /dev/null and b/frontend/public/benchmarks/tts/tada-1b.mp3 differ diff --git a/frontend/public/benchmarks/tts/tada-3b.mp3 b/frontend/public/benchmarks/tts/tada-3b.mp3 new file mode 100644 index 00000000..90b9820e Binary files /dev/null and b/frontend/public/benchmarks/tts/tada-3b.mp3 differ diff --git a/frontend/public/benchmarks/tts/vibevoice-1.5b.mp3 b/frontend/public/benchmarks/tts/vibevoice-1.5b.mp3 new file mode 100644 index 00000000..4b152345 Binary files /dev/null and b/frontend/public/benchmarks/tts/vibevoice-1.5b.mp3 differ diff --git a/frontend/public/benchmarks/tts/vibevoice.mp3 b/frontend/public/benchmarks/tts/vibevoice.mp3 new file mode 100644 index 00000000..a3125644 Binary files /dev/null and b/frontend/public/benchmarks/tts/vibevoice.mp3 differ diff --git a/frontend/public/benchmarks/tts/xtts.mp3 b/frontend/public/benchmarks/tts/xtts.mp3 new file mode 100644 index 00000000..d37dc253 Binary files /dev/null and b/frontend/public/benchmarks/tts/xtts.mp3 differ diff --git a/frontend/public/benchmarks/tts/zonos.mp3 b/frontend/public/benchmarks/tts/zonos.mp3 new file mode 100644 index 00000000..2ac5a698 Binary files /dev/null and b/frontend/public/benchmarks/tts/zonos.mp3 differ diff --git a/frontend/public/icons/tts/Banaxi-Tech.png b/frontend/public/icons/tts/Banaxi-Tech.png new file mode 100644 index 00000000..902c9ceb Binary files /dev/null and b/frontend/public/icons/tts/Banaxi-Tech.png differ diff --git a/frontend/public/icons/tts/HumeAI.png b/frontend/public/icons/tts/HumeAI.png new file mode 100644 index 00000000..0af1a24d Binary files /dev/null and b/frontend/public/icons/tts/HumeAI.png differ diff --git a/frontend/public/icons/tts/KittenML.png b/frontend/public/icons/tts/KittenML.png new file mode 100644 index 00000000..279042da Binary files /dev/null and b/frontend/public/icons/tts/KittenML.png differ diff --git a/frontend/public/icons/tts/NVIDIA.png b/frontend/public/icons/tts/NVIDIA.png new file mode 100644 index 00000000..433f77df Binary files /dev/null and b/frontend/public/icons/tts/NVIDIA.png differ diff --git a/frontend/public/icons/tts/QwenLM.png b/frontend/public/icons/tts/QwenLM.png new file mode 100644 index 00000000..5c65fe07 Binary files /dev/null and b/frontend/public/icons/tts/QwenLM.png differ diff --git a/frontend/public/icons/tts/SWivid.jpg b/frontend/public/icons/tts/SWivid.jpg new file mode 100644 index 00000000..b5d09c39 Binary files /dev/null and b/frontend/public/icons/tts/SWivid.jpg differ diff --git a/frontend/public/icons/tts/SesameAILabs.png b/frontend/public/icons/tts/SesameAILabs.png new file mode 100644 index 00000000..e1d17d91 Binary files /dev/null and b/frontend/public/icons/tts/SesameAILabs.png differ diff --git a/frontend/public/icons/tts/Zyphra.png b/frontend/public/icons/tts/Zyphra.png new file mode 100644 index 00000000..5179778b Binary files /dev/null and b/frontend/public/icons/tts/Zyphra.png differ diff --git a/frontend/public/icons/tts/canopyai.png b/frontend/public/icons/tts/canopyai.png new file mode 100644 index 00000000..ff09add4 Binary files /dev/null and b/frontend/public/icons/tts/canopyai.png differ diff --git a/frontend/public/icons/tts/coqui-ai.png b/frontend/public/icons/tts/coqui-ai.png new file mode 100644 index 00000000..02dd4cc5 Binary files /dev/null and b/frontend/public/icons/tts/coqui-ai.png differ diff --git a/frontend/public/icons/tts/facebookresearch.png b/frontend/public/icons/tts/facebookresearch.png new file mode 100644 index 00000000..58a19393 Binary files /dev/null and b/frontend/public/icons/tts/facebookresearch.png differ diff --git a/frontend/public/icons/tts/hexgrad.png b/frontend/public/icons/tts/hexgrad.png new file mode 100644 index 00000000..b813a9ec Binary files /dev/null and b/frontend/public/icons/tts/hexgrad.png differ diff --git a/frontend/public/icons/tts/k2-fsa.png b/frontend/public/icons/tts/k2-fsa.png new file mode 100644 index 00000000..5623f682 Binary files /dev/null and b/frontend/public/icons/tts/k2-fsa.png differ diff --git a/frontend/public/icons/tts/microsoft.png b/frontend/public/icons/tts/microsoft.png new file mode 100644 index 00000000..b15cce6c Binary files /dev/null and b/frontend/public/icons/tts/microsoft.png differ diff --git a/frontend/public/icons/tts/myshell-ai.png b/frontend/public/icons/tts/myshell-ai.png new file mode 100644 index 00000000..78b0c22f Binary files /dev/null and b/frontend/public/icons/tts/myshell-ai.png differ diff --git a/frontend/public/icons/tts/neuphonic.png b/frontend/public/icons/tts/neuphonic.png new file mode 100644 index 00000000..df2f4368 Binary files /dev/null and b/frontend/public/icons/tts/neuphonic.png differ diff --git a/frontend/public/icons/tts/resemble-ai.png b/frontend/public/icons/tts/resemble-ai.png new file mode 100644 index 00000000..d5980325 Binary files /dev/null and b/frontend/public/icons/tts/resemble-ai.png differ diff --git a/frontend/public/icons/tts/rhasspy.png b/frontend/public/icons/tts/rhasspy.png new file mode 100644 index 00000000..28276f8f Binary files /dev/null and b/frontend/public/icons/tts/rhasspy.png differ diff --git a/frontend/public/icons/tts/studio-dots-ai.png b/frontend/public/icons/tts/studio-dots-ai.png new file mode 100644 index 00000000..dabb4ce7 Binary files /dev/null and b/frontend/public/icons/tts/studio-dots-ai.png differ diff --git a/frontend/public/icons/tts/suno-ai.jpg b/frontend/public/icons/tts/suno-ai.jpg new file mode 100644 index 00000000..5f7fec0a Binary files /dev/null and b/frontend/public/icons/tts/suno-ai.jpg differ diff --git a/frontend/public/icons/tts/supertone-inc.png b/frontend/public/icons/tts/supertone-inc.png new file mode 100644 index 00000000..29943ad3 Binary files /dev/null and b/frontend/public/icons/tts/supertone-inc.png differ diff --git a/frontend/scripts/build-content.mjs b/frontend/scripts/build-content.mjs index 20dffae9..0820707d 100644 --- a/frontend/scripts/build-content.mjs +++ b/frontend/scripts/build-content.mjs @@ -87,6 +87,9 @@ function buildSection({ name, dir, out }) { // Social-card image (Open Graph / Twitter). Kept separate from coverImage // so it drives the link preview WITHOUT injecting a hero into the article. ogImage: frontmatter.ogImage || null, + // Opt a post out of the side index. Long, heading-dense posts want it; + // a short one is just a list of four links beside three paragraphs. + toc: frontmatter.toc === "false" || frontmatter.toc === false ? false : true, headings: extractHeadings(content), content, }; diff --git a/frontend/scripts/build-og-card.mjs b/frontend/scripts/build-og-card.mjs new file mode 100644 index 00000000..2ae18a09 --- /dev/null +++ b/frontend/scripts/build-og-card.mjs @@ -0,0 +1,74 @@ +// Copyright 2026 Rob Macrae. All rights reserved. +// SPDX-License-Identifier: LicenseRef-Proprietary + +// Renders a benchmark chart to a 1200x630 social card. +// +// The card is a screenshot of the real chart rather than a separate drawing, so +// it cannot drift from the data: re-run this after a new run lands and the card +// updates with the numbers. LinkedIn and X both crop toward the centre and +// neither executes JS, so this has to be a flat PNG at exactly 1200x630. +// +// Usage (needs `npm run dev` on :3000): +// node scripts/build-og-card.mjs open-weight-tts 0 public/benchmarks/og-open-weight-tts.png +// +// Args:
+ +import { chromium } from "@playwright/test"; +import path from "node:path"; + +const [slug, figIndex = "0", out] = process.argv.slice(2); +if (!slug || !out) { + console.error("usage: build-og-card.mjs "); + process.exit(1); +} + +const BASE = process.env.OG_BASE_URL ?? "http://localhost:3000"; +const [W, H] = [1200, 630]; + +const browser = await chromium.launch(); +// deviceScaleFactor 2 renders at 2400x1260 and downsamples, so text stays crisp +// on retina timelines rather than looking like a 1x screenshot. +const page = await browser.newPage({ viewport: { width: W, height: H }, deviceScaleFactor: 2 }); + +const url = `${BASE}/benchmarks/${slug}`; +await page.goto(url, { waitUntil: "domcontentloaded" }); +await page.locator("figure").nth(Number(figIndex)).waitFor({ timeout: 60_000 }); +// Charts measure themselves on mount and animate in; let that settle. +await page.waitForTimeout(2500); + +// Lift the chart out of the article and let it fill the frame on its own +// background. Cheaper and far more stable than trying to crop around the page +// chrome, which moves whenever the layout does. +await page.evaluate( + ({ figIndex, W, H }) => { + const PAD = 28; + const fig = document.querySelectorAll("figure")[Number(figIndex)]; + document.body.replaceChildren(fig); + Object.assign(document.body.style, { + margin: "0", padding: "0", width: `${W}px`, height: `${H}px`, + background: "#0a1120", overflow: "hidden", position: "relative", + }); + // Interactive affordances mean nothing in a static image. + fig.querySelectorAll("button").forEach((b) => { + if (b.closest("figcaption")) b.remove(); + else b.style.pointerEvents = "none"; + }); + Object.assign(fig.style, { + margin: "0", position: "absolute", top: "0", left: "0", + width: `${W - PAD * 2}px`, transformOrigin: "top left", + }); + // A chart authored for an article column is taller than a 1.9:1 card, so it + // is scaled to fit and centred. Cropping instead loses the title off the top + // and the x-axis off the bottom, which is most of what makes it readable. + const r = fig.getBoundingClientRect(); + const scale = Math.min((W - PAD * 2) / r.width, (H - PAD * 2) / r.height); + const [w, h] = [r.width * scale, r.height * scale]; + fig.style.transform = `translate(${(W - w) / 2}px, ${(H - h) / 2}px) scale(${scale})`; + }, + { figIndex, W, H } +); +await page.waitForTimeout(400); + +await page.screenshot({ path: path.resolve(out), clip: { x: 0, y: 0, width: W, height: H } }); +await browser.close(); +console.log(`[build-og-card] ${url} figure[${figIndex}] -> ${out} (${W}x${H})`); diff --git a/frontend/scripts/import-tts-export.mjs b/frontend/scripts/import-tts-export.mjs new file mode 100644 index 00000000..8b9f726d --- /dev/null +++ b/frontend/scripts/import-tts-export.mjs @@ -0,0 +1,532 @@ +// Copyright 2026 Rob Macrae. All rights reserved. +// SPDX-License-Identifier: LicenseRef-Proprietary + +// Turns a TTS benchmark export into the run JSON the page reads, and copies the +// audio in. Written as a script rather than done by hand because the export is +// re-cut whenever the sweep is re-run, and hand-transcribing thirty-odd rows of +// eighteen columns is exactly the sort of thing that silently loses four rows. +// +// benchmark.html is the source, not results.txt: only the HTML carries the +// precomputed data-sort keys, the per-cell tone classes, and the row grouping. +// The two have been seen to disagree (a run where chatterbox-q4's RTF read 1.97 +// in one and 2.03 in the other), and the HTML is what the exporter renders from. +// +// Parsing notes, both learned by losing data: +// - Attributes are parsed as a set, never positionally. Some cells carry +// `title` before `data-sort`, so a `class="..." data-sort="..."` regex +// silently yields an empty sort key for exactly the columns that need one. +// - Rows are matched as ``, not ``. Grouped rows carry a class, and +// a bare `` pattern drops every one of them. +// Two independent checks guard against a row vanishing anyway; see below. +// +// Usage: +// node scripts/import-tts-export.mjs [run-id] + +import fs from "node:fs"; +import path from "node:path"; +import crypto from "node:crypto"; + +const [exportDir, runId = "2026-08"] = process.argv.slice(2); +if (!exportDir) { + console.error("usage: import-tts-export.mjs [run-id]"); + process.exit(1); +} + +const OUT_JSON = `src/data/benchmarks/open-weight-tts/${runId}.json`; +const OUT_AUDIO = "public/benchmarks/tts"; + +/** Columns carried by the export that the page does not show. + * Passed is the denominator behind the word error rates - methodology, not a + * result, and it never separates one engine from another. Frame rate only means + * anything for the token-based engines and is blank for most rows. Libs is a + * packaging detail already folded into Disk. Lead-in is measured on a single + * clip, so it is indicative rather than a mean, and it earns its width less + * than it costs in a table this wide - the caveat it exists to explain lives in + * the methodology instead. */ +const DROP_COLUMNS = new Set(["Frame rate", "Passed", "Libs", "Lead-in", "WER med", "Avg audio"]); + +/** Configurations the comparison does not show. The NeuTTS-2E CPU builds are + * the same weights as the rows that remain, run on a slower path: they tripled + * the size of the NeuTTS band while saying nothing about the model, only about + * the hardware it was pointed at. Their measurements stay in the raw export. */ +const DROP_ROWS = new Set(["nt-2e-fp32-cpu", "nt-2e-q4-cpu", "nt-2e-q8-cpu"]); + +/** Shorter headers where the export's are longer than they need to be. + * Only one word error rate is shown, so it does not need qualifying: base.en + * is the recogniser that completed for every configuration, where medium.en + * did not. */ +const RENAME_COLUMNS = { + "Total disk": "Disk", + "WER base": "WER", + // "Peak" is the only kind of RSS reported here, so it says nothing the column + // does not already imply. + "Peak RSS": "RSS", + // x-bar: it is a mean, and "Avg" spent five characters saying so in a column + // whose values are four wide. + "Avg synth": "x\u0304 synth", +}; + +/** Where each project states its own licence: its LICENSE file, or the model + * card that declares it. Deliberately not opensource.org and friends - those + * explain what MIT is, they do not evidence that this model is under it, which + * is the whole point of the link. + * + * Resolved and verified per repo rather than constructed: GitHub's licence API + * finds the file whatever it is called, and the Hugging Face repos were probed + * for LICENSE before falling back to the card. + * + * Two point at a model card rather than the repo's LICENSE, because the repo + * licenses the *code* and this benchmark measures *weights*, and for these two + * they differ: F5-TTS is MIT as code and cc-by-nc-4.0 as weights, NeMo is + * Apache-2.0 as a framework while the FastPitch checkpoint is cc-by-4.0. The + * card is what evidences the licence the table prints. + * + * Three rows have none. Dots-TTS is listed "?" here while its repo declares + * Apache-2.0, so there is nothing consistent to point at; OmniVoice has no + * upstream card at all; BananaMind TTS has no findable home. + */ +const LICENCE_PROOF_URLS = { + // Code is MIT, weights are OpenRAIL-M - the table states the weights licence, + // so the proof has to be the weights' own LICENSE rather than the repo's. + "supertonic": "https://huggingface.co/Supertone/supertonic-3/blob/main/LICENSE", + "bark": "https://github.com/suno-ai/bark/blob/main/LICENSE", + "chatterbox": "https://github.com/resemble-ai/chatterbox/blob/master/LICENSE", + "chatterbox-q4": "https://github.com/resemble-ai/chatterbox/blob/master/LICENSE", + "chatterbox-q8": "https://github.com/resemble-ai/chatterbox/blob/master/LICENSE", + "chatterbox-turbo": "https://github.com/resemble-ai/chatterbox/blob/master/LICENSE", + "cosyvoice3": "https://github.com/QwenAudio/CosyVoice/blob/main/LICENSE", + "cosyvoice3-rl": "https://github.com/QwenAudio/CosyVoice/blob/main/LICENSE", + "csm": "https://github.com/SesameAILabs/csm/blob/main/LICENSE", + "f5-tts": "https://huggingface.co/SWivid/F5-TTS", + "fastpitch": "https://huggingface.co/nvidia/tts_en_fastpitch", + "kittentts-mini": "https://huggingface.co/KittenML/kitten-tts-mini-0.8/blob/main/README.md", + "kittentts-micro": "https://huggingface.co/KittenML/kitten-tts-micro-0.8/blob/main/README.md", + "kittentts-nano": "https://huggingface.co/KittenML/kitten-tts-nano-0.8-fp32/blob/main/README.md", + "kittentts-nano-int8": "https://huggingface.co/KittenML/kitten-tts-nano-0.8-int8/blob/main/README.md", + "kokoro": "https://huggingface.co/hexgrad/Kokoro-82M/blob/main/README.md", + "melotts": "https://github.com/myshell-ai/MeloTTS/blob/main/LICENSE", + "mms-tts": "https://huggingface.co/facebook/mms-tts/blob/main/README.md", + "nt-2e-fp32-mps": "https://huggingface.co/neuphonic/neutts-2e/blob/main/LICENSE", + "nt-2e-q4-metal": "https://huggingface.co/neuphonic/neutts-2e/blob/main/LICENSE", + "nt-2e-q8-metal": "https://huggingface.co/neuphonic/neutts-2e/blob/main/LICENSE", + "parler-tts": "https://github.com/huggingface/parler-tts/blob/main/LICENSE", + "piper": "https://github.com/rhasspy/piper/blob/master/LICENSE.md", + "qwen3-tts": "https://github.com/QwenLM/Qwen3-TTS/blob/main/LICENSE", + // The model cards rather than the repositories: both declare the licence the + // weights carry, which is what this column states. Supertonic is the standing + // reminder that the two can differ. + "qwen3-tts-vd": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", + "orpheus-q4": "https://huggingface.co/canopylabs/orpheus-3b-0.1-ft", + "speecht5": "https://huggingface.co/microsoft/speecht5_tts/blob/main/README.md", + "styletts2": "https://github.com/yl4579/StyleTTS2/blob/main/LICENSE", + "dots-tts": "https://github.com/studio-dots-ai/dots.tts/blob/main/LICENSE", + "bananamind-tts": "https://huggingface.co/Banaxi-Tech/BananaMind-TTS-V2/blob/main/LICENSE", + "omnivoice": "https://github.com/k2-fsa/OmniVoice/blob/master/LICENSE", + "tada-1b": "https://huggingface.co/HumeAI/tada-1b/blob/main/LICENSE", + "tada-3b": "https://huggingface.co/HumeAI/tada-3b-ml/blob/main/LICENSE", + "vibevoice": "https://github.com/microsoft/VibeVoice/blob/main/LICENSE", + "vibevoice-1.5b": "https://huggingface.co/microsoft/VibeVoice-1.5B/blob/main/README.md", + "xtts": "https://huggingface.co/coqui/XTTS-v2/blob/main/LICENSE.txt", + "zonos": "https://github.com/Zyphra/Zonos/blob/main/LICENSE", +}; + +/** Where each model actually lives. Kept here rather than in the export, which + * carries no URLs at all. A configuration with no entry renders as plain text: + * several of these have no upstream model card to link to (the export says so + * outright for OmniVoice), and a plausible-looking wrong link is worse than + * none. */ +const MODEL_URLS = { + "supertonic": "https://github.com/supertone-inc/supertonic", + "piper": "https://github.com/rhasspy/piper", + "kokoro": "https://huggingface.co/hexgrad/Kokoro-82M", + "bark": "https://github.com/suno-ai/bark", + "mms-tts": "https://huggingface.co/facebook/mms-tts", + "f5-tts": "https://github.com/SWivid/F5-TTS", + "parler-tts": "https://github.com/huggingface/parler-tts", + "xtts": "https://huggingface.co/coqui/XTTS-v2", + "cosyvoice3": "https://github.com/QwenAudio/CosyVoice", + "cosyvoice3-rl": "https://github.com/QwenAudio/CosyVoice", + "chatterbox": "https://github.com/resemble-ai/chatterbox", + "chatterbox-turbo": "https://github.com/resemble-ai/chatterbox", + "chatterbox-q4": "https://github.com/resemble-ai/chatterbox", + "chatterbox-q8": "https://github.com/resemble-ai/chatterbox", + "styletts2": "https://github.com/yl4579/StyleTTS2", + "melotts": "https://github.com/myshell-ai/MeloTTS", + "speecht5": "https://huggingface.co/microsoft/speecht5_tts", + "kittentts-mini": "https://huggingface.co/KittenML/kitten-tts-mini-0.8", + "kittentts-micro": "https://huggingface.co/KittenML/kitten-tts-micro-0.8", + "kittentts-nano": "https://huggingface.co/KittenML/kitten-tts-nano-0.8-fp32", + "kittentts-nano-int8": "https://huggingface.co/KittenML/kitten-tts-nano-0.8-int8", + "csm": "https://github.com/SesameAILabs/csm", + "zonos": "https://github.com/Zyphra/Zonos", + "fastpitch": "https://github.com/NVIDIA-NeMo/Speech", + "qwen3-tts": "https://github.com/QwenLM/Qwen3-TTS", + // VoiceDesign has its own checkpoint, so it points at that rather than at the + // family repository the base model uses. + "qwen3-tts-vd": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", + "orpheus-q4": "https://github.com/canopyai/Orpheus-TTS", + "omnivoice": "https://github.com/k2-fsa/OmniVoice", + "bananamind-tts": "https://huggingface.co/Banaxi-Tech/BananaMind-TTS-V2", + "dots-tts": "https://github.com/studio-dots-ai/dots.tts", + "tada-1b": "https://huggingface.co/HumeAI/tada-1b", + "tada-3b": "https://huggingface.co/HumeAI/tada-3b-ml", + // The quantized and device variants are the same upstream weights. + "nt-2e-fp32-mps": "https://huggingface.co/neuphonic/neutts-2e", + "nt-2e-q4-metal": "https://huggingface.co/neuphonic/neutts-2e", + "nt-2e-q8-metal": "https://huggingface.co/neuphonic/neutts-2e", + // 1.02B "despite a 0.5b filename", per the export - no model page matches it, + // so this one points at the project rather than a specific checkpoint. + "vibevoice": "https://github.com/microsoft/VibeVoice", + "vibevoice-1.5b": "https://huggingface.co/microsoft/VibeVoice-1.5B", + // Deliberately absent: omnivoice, which the export says has no upstream model + // card, and bananamind-tts, which nothing findable matches. They render as + // plain text rather than pointing somewhere plausible but wrong. +}; + +/** Facts the export could not read off a model card, established since and + * cited here so the next export cannot quietly revert them. + * + * The export prints "-" or "?" when it cannot find something. That is a + * statement about the card it could reach, not about the model, so a value + * confirmed from the project's own materials belongs in the table. + * + * `sort` is given wherever the column sorts on something other than the text - + * Released is YYYYMM, Params is millions - because an override that changes + * only the text sorts under the old blank, invisibly. + */ +const CELL_OVERRIDES = { + omnivoice: { + // k2-fsa's Space declares `license: apache-2.0`; the repo carries the text. + Licence: { v: "Apache-2.0" }, + // arXiv:2604.00688, "OmniVoice: Towards Omnilingual Zero-Shot Text-to-Speech + // with Diffusion Language Models", submitted 1 Apr 2026. + Released: { v: "2026-04", sort: "202604" }, + Params: { v: "0.8B", sort: "800.0" }, + // The old note said the licence was unverified, which the row now + // contradicts two columns to its left. + Notes: { v: "Speech is fluent. Size and date come from the paper (arXiv:2604.00688) rather than a model card." }, + }, + // The export dates this 2026-03, but the configuration measured is the 99M + // v3 checkpoint - four ONNX graphs, native 44.1 kHz, 31 languages - and that + // shipped at the end of April 2026. March belongs to an earlier checkpoint; + // v1 was 66M and v2, from January, covered five languages. + supertonic: { Released: { v: "2026-04", sort: "202604" } }, + // Banaxi-Tech's card states "9.5M params"; it already declared apache-2.0, + // which is what the export had, so only the size was missing. + "bananamind-tts": { Params: { v: "9.5M", sort: "9.5" } }, + // Its repo has carried an Apache LICENSE all along - this was the mismatch the + // licence cross-check flagged when the links were first added. + "dots-tts": { Licence: { v: "Apache-2.0" } }, + // Both cards declare `license: llama3.2`, and Hume's Space says "the model is + // licensed under the Llama 3.2 Community License Agreement". + "tada-1b": { Licence: { v: "Llama 3.2 Community" } }, + "tada-3b": { Licence: { v: "Llama 3.2 Community" } }, +}; + +/** Cell text rewrites, by column. NeuTTS's licence is a sentence rather than an + * SPDX id, and spelled out it is the widest cell in the column. */ +const REWRITE = { + // "stoch-ff" was the widest value in a column of six-character codes. + Class: (v) => (v === "stoch-ff" ? "st-ff" : v), + // Replacement is a function, not a string: "$5m" as a string literal is a + // capture-group reference to any future group 5 in that pattern, and would + // silently start substituting instead of printing. + Licence: (v) => v.replace(/under \$5M/i, () => "<$5M"), +}; + +/** Each vendor's own capitalisation. The export uses lowercase run ids; showing + * those verbatim misspells every product on the page. Anything not listed falls + * back to the raw id, which is visible enough to get noticed and fixed. */ +const DISPLAY = { + // 99M parameters across four ONNX graphs at 44.1 kHz is the v3 checkpoint; + // v1 was 66M. See the note flagged with this import about its release date. + "supertonic": "Supertonic 3", + "piper": "Piper", + "bark": "Bark", + "chatterbox": "Chatterbox", + "chatterbox-q4": "Chatterbox Q4", + "chatterbox-q8": "Chatterbox Q8", + "chatterbox-turbo": "Chatterbox Turbo", + "cosyvoice3": "CosyVoice3", + "cosyvoice3-rl": "CosyVoice3 RL", + "csm": "CSM", + "f5-tts": "F5-TTS", + "fastpitch": "FastPitch", + "kittentts-mini": "KittenTTS Mini", + "kittentts-micro": "KittenTTS Micro", + "kittentts-nano": "KittenTTS Nano", + "kittentts-nano-int8": "KittenTTS Nano INT8", + "kokoro": "Kokoro", + "melotts": "MeloTTS", + "nt-2e-fp32-mps": "NeuTTS-2E FP32", + "nt-2e-q4-metal": "NeuTTS-2E Q4", + "nt-2e-q8-metal": "NeuTTS-2E Q8", + "xtts": "XTTS", + "omnivoice": "OmniVoice", + "parler-tts": "Parler-TTS", + "qwen3-tts": "Qwen3-TTS", + "qwen3-tts-vd": "Qwen3-TTS VoiceDesign", + "orpheus-q4": "Orpheus 3B Q4", + "speecht5": "SpeechT5", + "styletts2": "StyleTTS2", + "vibevoice": "VibeVoice", + "vibevoice-1.5b": "VibeVoice 1.5B", + "zonos": "Zonos", + "bananamind-tts": "BananaMind TTS", + "mms-tts": "MMS-TTS", + // No verifiable model card for these three (licence reads "?"), so they get + // conservative title case rather than invented internal capitals. + "dots-tts": "Dots-TTS", + "tada-1b": "Tada 1B", + "tada-3b": "Tada 3B", +}; + +const html = fs + .readFileSync(path.join(exportDir, "benchmark.html"), "utf8") + // Samples are embedded as data URIs. They are megabytes of base64 that make + // every subsequent pattern quadratic, and the mp3s are on disk anyway. + .replace(/data:audio\/mpeg;base64,[A-Za-z0-9+/=]+/g, "AUDIO"); + +/** Attributes as a map, order-independent. */ +function attrs(tag) { + const out = {}; + for (const m of tag.matchAll(/([\w-]+)\s*=\s*"([^"]*)"/g)) out[m[1]] = m[2]; + return out; +} +const strip = (s) => + s + .replace(/]*>[\s\S]*?<\/button>/gi, "") // the play control is not cell text + .replace(/<[^>]+>/g, "") + .replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">") + .replace(/"/g, '"').replace(/'/g, "'") + .replace(/·/g, "·").replace(/ /g, " ") + .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n))) + .replace(/\s+/g, " ") + .trim(); + +const thead = html.slice(0, html.indexOf("")); +const headers = [...thead.matchAll(/]*>([\s\S]*?)<\/th>/g)] + .map((m) => strip(m[1])) + .filter(Boolean); + +const tbody = html.slice(html.indexOf("")); +const rawRows = [...tbody.matchAll(/]*)>([\s\S]*?)<\/tr>/g)]; + +const readme = fs.readFileSync(path.join(exportDir, "README.txt"), "utf8"); + +// Two independent checks, because the failure that matters is a row silently +// vanishing, and a plain count cannot tell "the exporter left it out" apart from +// "my pattern missed it". +// +// 1. Did the row pattern match every row the markup contains? Counting bare +// ` tags but ${rawRows.length} rows matched`); + process.exit(1); +} + +// 2. Is every configuration in the table also in results.txt? The two are +// generated from the same run, so a name here that is missing there means the +// parse is producing junk. The reverse is expected and only reported: the +// exporter leaves failed and sample-less runs out of the comparison while +// still listing their measurements. +const resultsTxt = fs.readFileSync(path.join(exportDir, "data", "results.txt"), "utf8"); +const measured = new Set( + resultsTxt.split("\n").slice(2) + .map((l) => l.trim().split(/\s+/)[0]) + .filter((n) => n && !/^-+$/.test(n)) +); +const parsedIds = rawRows.map(([, , inner]) => inner.match(/data-sort="([^"]+)"/)?.[1] ?? ""); +const unknown = parsedIds.filter((id) => !measured.has(id)); +if (unknown.length) { + console.error(`FATAL: parsed configurations absent from results.txt: ${unknown.join(", ")}`); + process.exit(1); +} +const excluded = [...measured].filter((id) => !parsedIds.includes(id)); +if (excluded.length) { + console.log(` i measured but not in the comparison, per the exporter: ${excluded.join(", ")}`); +} +// Prose count is informational only: it counts everything measured, which is +// not the same as everything compared. +const claimed = Number(readme.match(/(\d+)\s+configurations/)?.[1] ?? 0); +if (claimed && claimed !== rawRows.length) { + console.log(` i README counts ${claimed} measured; ${rawRows.length} are in the comparison`); +} + +const kept = rawRows.filter((r) => !DROP_ROWS.has(r[2].match(/data-sort="([^"]+)"/)?.[1] ?? "")); +if (kept.length !== rawRows.length) { + console.log(` i not shown: ${[...DROP_ROWS].join(", ")}`); +} +rawRows.length = 0; +rawRows.push(...kept); + +const keep = headers.map((h) => !DROP_COLUMNS.has(h)); +const columns = headers.filter((_, i) => keep[i]).map((h) => RENAME_COLUMNS[h] ?? h); + +const audioAvailable = new Set( + fs.readdirSync(path.join(exportDir, "samples")).filter((f) => f.endsWith(".mp3")) +); + +const rows = rawRows.map(([, rowAttrs, inner]) => { + // Resolved up front: the cell mapper below needs it, and it is derived from + // the first cell rather than from the loop variable. + const config = (inner.match(/]*)>([\s\S]*?)<\/td>/g)].map((m) => { + const a = attrs(m[0].slice(0, m[0].indexOf(">") + 1)); + const cls = a.class ?? ""; + const tone = ["good", "bad", "warn", "pending"].find((t) => cls.split(/\s+/).includes(t)) ?? ""; + const v = strip(m[2]); + // The export gives unmeasured cells a sentinel sort key - -1 for PESQ and + // Params, 99 for WER med and Lead-in - so that they pile up at one end of a + // sort. On a page where the reader can sort by any column that is actively + // wrong: it ranks three engines with no PESQ as the worst-sounding in the + // table, and thirteen with no medium-model score as the least accurate. + // Blanking the key lets the table sink them at both ends instead, which is + // what "not measured" should do. "?" is left alone: on Licence it is a real + // category meaning the card could not be verified, not a missing number. + const missing = v === "·" || v === "-" || v === ""; + return { + v, + sort: missing ? "" : a["data-sort"] ?? "", + tone, + align: cls.split(/\s+/).includes("l") ? "left" : "", + restricted: cls.split(/\s+/).includes("restricted"), + }; + }); + if (tds.length !== headers.length) { + console.error(`FATAL: row "${tds[0]?.v}" has ${tds.length} cells, expected ${headers.length}`); + process.exit(1); + } + + // The name cell can carry a qualifier the id does not, e.g. "zonos (partial)". + const qualifier = tds[0].v.replace(config, "").trim(); + const display = (DISPLAY[config] ?? config) + (qualifier ? ` ${qualifier}` : ""); + if (!DISPLAY[config]) console.warn(` ! no display name for "${config}" - using the raw id`); + + const cells = tds + .map((c, i) => { + const rewrite = REWRITE[headers[i]]; + const cell = rewrite ? { ...c, v: rewrite(c.v) } : c; + const override = CELL_OVERRIDES[config]?.[headers[i]]; + const withOverride = override + ? { ...cell, v: override.v, sort: override.sort ?? override.v } + : cell; + if (headers[i] === "Licence" && LICENCE_PROOF_URLS[config]) { + return { ...withOverride, href: LICENCE_PROOF_URLS[config] }; + } + if (override) return withOverride; + return cell; + }) + .filter((_, i) => keep[i]); + cells[0] = { ...cells[0], v: display, ...(MODEL_URLS[config] ? { href: MODEL_URLS[config] } : {}) }; + + const sample = `${config}.mp3`; + return { + config, + display, + group: (attrs(``).class ?? "").trim(), + // Bare filename here; the content hash is appended after the copy below. + sample: audioAvailable.has(sample) ? sample : null, + cells, + }; +}); + +const rtfCol = columns.indexOf("RTF"); +const rtfOf = (r) => Number(r.cells[rtfCol].sort); +const RTF_CUTOFF = 2; + +const run = { + benchmark: "open-weight-tts", + run: runId, + label: readme.match(/Exported (\d{4}-\d{2}-\d{2})/)?.[1] ?? runId, + machine: "Apple M2, macOS", + corpus: "84 phrases", + // Kept as data rather than hardcoded in the component: it drives the + // real-time filter, and where the line sits is an editorial choice. + rtfCutoff: RTF_CUTOFF, + caption: + "Sorted by compute per phrase, fastest first. Every configuration measured is listed, " + + "including the ones far too slow to keep up with their own speech; the filter above hides " + + `anything above ${RTF_CUTOFF}x real time.`, + columns, + rows, +}; + +fs.mkdirSync(OUT_AUDIO, { recursive: true }); +const short = (buf) => crypto.createHash("sha256").update(buf).digest("hex").slice(0, 8); + +let copied = 0, changed = 0; +for (const r of rows) { + if (!r.sample) { console.warn(` ! no audio for ${r.config}`); continue; } + const src = path.join(exportDir, "samples", r.sample); + const dst = path.join(OUT_AUDIO, r.sample); + const incoming = fs.readFileSync(src); + const existing = fs.existsSync(dst) ? fs.readFileSync(dst) : null; + const replaced = !existing || !existing.equals(incoming); + if (replaced) { fs.writeFileSync(dst, incoming); changed++; } + copied++; + + // A re-cut clip usually keeps its filename, so browsers and the CDN would go + // on serving the old audio against a table of new numbers - the sort of wrong + // that is invisible because everything still plays. Fingerprinting the URL + // with the content hash makes changed bytes a different URL, while unchanged + // clips keep theirs and stay cached. + r.sample = `${r.sample}?v=${short(incoming)}`; + if (replaced) console.log(` ~ replaced ${path.basename(dst)}`); +} + +// Audio for configurations no longer listed would otherwise accumulate forever. +const wanted = new Set(rows.map((r) => r.sample?.split("?")[0]).filter(Boolean)); +for (const f of fs.readdirSync(OUT_AUDIO)) { + if (f.endsWith(".mp3") && !wanted.has(f)) { + fs.unlinkSync(path.join(OUT_AUDIO, f)); + console.log(` - removed stale audio ${f}`); + } +} + +// Written only now: the copy loop above appends each clip's content hash to +// row.sample, and serialising before that ran wrote the rows without their +// version tags - the audio was correctly replaced and every URL still pointed +// at the unversioned name, which is precisely the stale-cache case this exists +// to prevent. +fs.mkdirSync(path.dirname(OUT_JSON), { recursive: true }); +fs.writeFileSync(OUT_JSON, JSON.stringify(run, null, 2) + "\n"); + +// The control plane draws ballots from its own copy of the configuration list, +// and it has to match the table: a ballot offering a clip the page no longer +// shows is one the server then refuses, and a row missing from the pool can +// never be rated. That list was being regenerated by hand after every import, +// which is exactly the sort of step that gets skipped - so the import does it. +const POOL_FILE = "../controlplane/src/tts/handler.ts"; +if (fs.existsSync(POOL_FILE)) { + const ids = rows.filter((r) => r.sample).map((r) => r.config); + const lines = []; + let line = " "; + for (const id of ids) { + const add = `'${id}', `; + if ((line + add).length > 78) { lines.push(line.trimEnd()); line = " "; } + line += add; + } + lines.push(line.trimEnd()); + const block = `const CONFIGS = [\n${lines.join("\n")}\n] as const;`; + const before = fs.readFileSync(POOL_FILE, "utf8"); + const after = before.replace(/const CONFIGS = \[[\s\S]*?\] as const;/, block); + if (after === before) { + console.log(" = ballot pool already matches"); + } else if (!/const CONFIGS = \[/.test(before)) { + console.warn(" ! could not find CONFIGS in the control plane - ballot pool NOT updated"); + } else { + fs.writeFileSync(POOL_FILE, after); + console.log(` ~ ballot pool synced to ${ids.length} configurations`); + } +} + +const within = rows.filter((r) => rtfOf(r) <= RTF_CUTOFF).length; +console.log( + `[import-tts-export] ${rows.length} configurations, ${columns.length} columns ` + + `(dropped ${[...DROP_COLUMNS].join(", ")}), ${copied} clips (${changed} changed)\n` + + ` ${within} within ${RTF_CUTOFF}x real time, ${rows.length - within} above it\n` + + ` -> ${OUT_JSON}` +); diff --git a/frontend/src/app/(benchmarks)/benchmarks/[slug]/page.tsx b/frontend/src/app/(benchmarks)/benchmarks/[slug]/page.tsx index dfbc7b76..f3a45604 100644 --- a/frontend/src/app/(benchmarks)/benchmarks/[slug]/page.tsx +++ b/frontend/src/app/(benchmarks)/benchmarks/[slug]/page.tsx @@ -7,11 +7,11 @@ import { getPost, getAllPosts } from "@/lib/benchmarks"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; import rehypeSlug from "rehype-slug"; -import Link from "next/link"; import { notFound } from "next/navigation"; import type { Metadata } from "next"; import { ScrollVideo } from "@/components/ScrollVideo"; import { BenchmarkToc } from "@/components/BenchmarkToc"; +import { BenchmarkTabs } from "@/components/BenchmarkTabs"; import { MarkdownChart } from "@/components/charts/MarkdownChart"; import { SortableTable } from "@/components/SortableTable"; @@ -22,6 +22,18 @@ function isChartFence(className?: string): boolean { return typeof className === "string" && className.split(" ").includes("language-chart"); } +/** The fence language of a 
's  child, as a class string. hast keeps
+ *  className as an array, which isChartFence does not accept. */
+function fenceLanguageOf(node: unknown): string {
+  const kids = (node as { children?: unknown[] } | undefined)?.children;
+  const first = kids?.[0] as
+    | { tagName?: string; properties?: { className?: unknown } }
+    | undefined;
+  if (first?.tagName !== "code") return "";
+  const cls = first.properties?.className;
+  return Array.isArray(cls) ? cls.join(" ") : typeof cls === "string" ? cls : "";
+}
+
 const MODULE_REVISION = "benchmarks-v1-post";
 console.log(`[benchmarks-post] REVISION: ${MODULE_REVISION} loaded at ${new Date().toISOString()}`);
 
@@ -77,6 +89,8 @@ export default async function BenchmarkPage({ params }: Props) {
   const post = getPost(slug);
   if (!post) notFound();
 
+  const showToc = post.toc !== false;
+
   // Lead the side menu with the article title, then its headings.
   const tocItems = [
     { text: post.title, slug: post.slug, depth: 1 },
@@ -84,12 +98,16 @@ export default async function BenchmarkPage({ params }: Props) {
   ];
 
   return (
-    

+