From 5eb71dbb261e747359b459e572f0754ec53cd9bc Mon Sep 17 00:00:00 2001 From: Eason WaveKat Date: Fri, 15 May 2026 11:18:53 +1200 Subject: [PATCH] feat(audio-lab): live ASR transcript panel (M2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Frontend half of the ASR integration: - New AsrConfigPanel mirrors TurnConfigPanel — backend + preset + label. - New AsrTranscript card renders finals (with [mm:ss.s–mm:ss.s] prefix) plus a dimmed trailing partial that overwrites until the final lands. Footer shows last confidence, count of finals, average segment duration. "loading model…" until the backend's `ready` event arrives. Copy-all button concatenates final text to the clipboard. - App.tsx wires list_asr_backends on connect, persists asr configs to localStorage, pushes set_asr_configs on change + before start / load_file, resets transcripts on new session. - websocket.ts: new AsrConfig / AsrEventKind types, asr_backends + asr server messages, list_asr_backends + set_asr_configs client messages. Log panel batches `partial` events (matching how `vad` is batched) and inlines finals / warnings. cargo isn't touched — backend already merged on feat/asr-backend. npm run lint clean (no new warnings); npm run build clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- tools/audio-lab/frontend/package-lock.json | 4 +- tools/audio-lab/frontend/src/App.tsx | 121 ++++++++++ .../src/components/AsrConfigPanel.tsx | 207 ++++++++++++++++++ .../frontend/src/components/AsrTranscript.tsx | 154 +++++++++++++ tools/audio-lab/frontend/src/lib/websocket.ts | 60 ++++- 5 files changed, 540 insertions(+), 6 deletions(-) create mode 100644 tools/audio-lab/frontend/src/components/AsrConfigPanel.tsx create mode 100644 tools/audio-lab/frontend/src/components/AsrTranscript.tsx diff --git a/tools/audio-lab/frontend/package-lock.json b/tools/audio-lab/frontend/package-lock.json index f2ca16f..5bc3432 100644 --- a/tools/audio-lab/frontend/package-lock.json +++ b/tools/audio-lab/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "frontend", - "version": "0.0.3", + "version": "0.0.14", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "frontend", - "version": "0.0.3", + "version": "0.0.14", "dependencies": { "@base-ui/react": "^1.3.0", "@fontsource-variable/geist": "^5.2.8", diff --git a/tools/audio-lab/frontend/src/App.tsx b/tools/audio-lab/frontend/src/App.tsx index d7b3b50..6124e68 100644 --- a/tools/audio-lab/frontend/src/App.tsx +++ b/tools/audio-lab/frontend/src/App.tsx @@ -19,6 +19,11 @@ import { STATE_COLORS as TURN_STATE_COLORS } from "@/lib/turnColors"; import { TurnConfigPanel } from "@/components/TurnConfigPanel"; import { PipelineConfigPanel } from "@/components/PipelineConfigPanel"; import { PipelineTimeline } from "@/components/PipelineTimeline"; +import { AsrConfigPanel } from "@/components/AsrConfigPanel"; +import { + AsrTranscript, + type AsrTranscriptState, +} from "@/components/AsrTranscript"; import { type Viewport, createDefaultViewport, @@ -32,6 +37,7 @@ import { type TurnConfig, type PipelineConfig, type PipelineResultPoint, + type AsrConfig, type ParamInfo, type ServerMessage, type ConnectionState, @@ -192,6 +198,17 @@ function App() { }); const [pipelineResults, setPipelineResults] = useState>({}); + const [asrBackends, setAsrBackends] = useState>({}); + const [asrConfigs, setAsrConfigs] = useState(() => { + try { + const saved = localStorage.getItem("lab-asr-configs"); + if (saved !== null) return JSON.parse(saved) as AsrConfig[]; + } catch { /* ignore */ } + return []; + }); + const [asrTranscripts, setAsrTranscripts] = useState>({}); + const [asrOpen, setAsrOpen] = useState(true); + // Preprocessed data per config const [preprocessedSamples, setPreprocessedSamples] = useState>({}); const [preprocessedSpectrumData, setPreprocessedSpectrumData] = useState< @@ -217,6 +234,18 @@ function App() { localStorage.setItem("lab-pipeline-configs", JSON.stringify(pipelineConfigs)); }, [pipelineConfigs]); + // Persist asr configs to localStorage + useEffect(() => { + localStorage.setItem("lab-asr-configs", JSON.stringify(asrConfigs)); + }, [asrConfigs]); + + // Push asr config changes to the backend so the next start picks them up + useEffect(() => { + const socket = socketRef.current; + if (!socket || !connected) return; + socket.send({ type: "set_asr_configs", configs: asrConfigs }); + }, [asrConfigs, connected]); + // Resolve playback samples based on selected source const playbackSamples = playbackSource === "original" ? samples : (preprocessedSamples[playbackSource] ?? []); @@ -261,6 +290,7 @@ function App() { socket.send({ type: "list_devices" }); socket.send({ type: "list_backends" }); socket.send({ type: "list_turn_backends" }); + socket.send({ type: "list_asr_backends" }); }, []); const handleMessage = useCallback((msg: ServerMessage) => { @@ -433,6 +463,59 @@ function App() { })); break; + case "asr_backends": + setAsrBackends(msg.backends); + break; + + case "asr": + setAsrTranscripts((prev) => { + const existing: AsrTranscriptState = prev[msg.config_id] ?? { + ready: false, + finals: [], + partial: null, + warning: null, + }; + switch (msg.kind) { + case "ready": + return { + ...prev, + [msg.config_id]: { ...existing, ready: true, warning: null }, + }; + case "partial": + return { + ...prev, + [msg.config_id]: { ...existing, partial: msg.text ?? null }, + }; + case "final": + return { + ...prev, + [msg.config_id]: { + ...existing, + partial: null, + finals: [ + ...existing.finals, + { + ts_ms: msg.ts_ms ?? 0, + end_ms: msg.end_ms ?? msg.ts_ms ?? 0, + text: msg.text ?? "", + confidence: msg.confidence ?? 1, + }, + ], + }, + }; + case "warning": + return { + ...prev, + [msg.config_id]: { ...existing, warning: msg.message ?? null }, + }; + case "speech_started": + case "speech_ended": + default: + return prev; + } + }); + break; + case "done": recordingRef.current = false; setRecording(false); @@ -488,6 +571,7 @@ function App() { setTurnResults({}); setTurnTiming({}); setPipelineResults({}); + setAsrTranscripts({}); setPlaybackSource("original"); setTotalDurationMs(0); setSampleRate(null); @@ -504,6 +588,7 @@ function App() { socket.send({ type: "set_configs", configs }); socket.send({ type: "set_turn_configs", configs: turnConfigs }); socket.send({ type: "set_pipeline_configs", configs: pipelineConfigs }); + socket.send({ type: "set_asr_configs", configs: asrConfigs }); socket.send({ type: "start_recording", device_index: parseInt(selectedDevice), @@ -537,6 +622,7 @@ function App() { setTurnResults({}); setTurnTiming({}); setPipelineResults({}); + setAsrTranscripts({}); setPlaybackSource("original"); setTotalDurationMs(0); setSampleRate(null); @@ -545,6 +631,7 @@ function App() { socket.send({ type: "set_configs", configs }); socket.send({ type: "set_turn_configs", configs: turnConfigs }); socket.send({ type: "set_pipeline_configs", configs: pipelineConfigs }); + socket.send({ type: "set_asr_configs", configs: asrConfigs }); socket.send({ type: "load_file", path, channel }); setLoadingFile(true); }; @@ -979,6 +1066,10 @@ function App() { /> ))} + {asrConfigs.length > 0 && ( + + )} + {/* Preprocessed Waveforms/Spectrograms/VAD - only for configs with showPreprocessed enabled */} {vadOpen && configs.filter((c) => showPreprocessed[c.id]).map((config) => { const configIndex = configs.findIndex((c) => c.id === config.id); @@ -1149,6 +1240,36 @@ function App() { )} + {/* ASR Config Panel */} +
+
+ +
+ {asrOpen && ( + { + const backendNames = Object.keys(asrBackends); + if (backendNames.length === 0) return; + const backend = backendNames[0]; + const params: Record = {}; + for (const p of asrBackends[backend]) { + params[p.name] = p.default; + } + setAsrConfigs([{ id: "asr-1", label: "asr-1", backend, params }]); + }} + /> + )} +
+ {/* Logs */} diff --git a/tools/audio-lab/frontend/src/components/AsrConfigPanel.tsx b/tools/audio-lab/frontend/src/components/AsrConfigPanel.tsx new file mode 100644 index 0000000..fa25924 --- /dev/null +++ b/tools/audio-lab/frontend/src/components/AsrConfigPanel.tsx @@ -0,0 +1,207 @@ +import { useMemo } from "react"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import type { AsrConfig, ParamInfo } from "@/lib/websocket"; + +export type { AsrConfig }; + +interface AsrConfigPanelProps { + configs: AsrConfig[]; + backends: Record; + onConfigsChange: (configs: AsrConfig[]) => void; + onResetDefaults: () => void; +} + +export function AsrConfigPanel({ + configs, + backends, + onConfigsChange, + onResetDefaults, +}: AsrConfigPanelProps) { + const nextId = useMemo(() => { + let max = 0; + for (const c of configs) { + const match = c.id.match(/^asr-(\d+)$/); + if (match) { + max = Math.max(max, parseInt(match[1], 10)); + } + } + return max + 1; + }, [configs]); + + const addConfig = () => { + const backendNames = Object.keys(backends); + if (backendNames.length === 0) return; + + const backend = backendNames[0]; + const params: Record = {}; + for (const p of backends[backend]) { + params[p.name] = p.default; + } + + const id = `asr-${nextId}`; + onConfigsChange([ + ...configs, + { id, label: `asr-${nextId}`, backend, params }, + ]); + }; + + const removeConfig = (id: string) => { + onConfigsChange(configs.filter((c) => c.id !== id)); + }; + + const cloneConfig = (config: AsrConfig) => { + const id = `asr-${nextId}`; + onConfigsChange([ + ...configs, + { ...config, id, label: `${config.label} (copy)`, params: { ...config.params } }, + ]); + }; + + const updateConfig = (id: string, updates: Partial) => { + onConfigsChange( + configs.map((c) => { + if (c.id !== id) return c; + const updated = { ...c, ...updates }; + + if (updates.backend && updates.backend !== c.backend) { + const newParams: Record = {}; + for (const p of backends[updates.backend] ?? []) { + newParams[p.name] = p.default; + } + updated.params = newParams; + } + + return updated; + }) + ); + }; + + const updateParam = (configId: string, paramName: string, value: unknown) => { + onConfigsChange( + configs.map((c) => { + if (c.id !== configId) return c; + return { ...c, params: { ...c.params, [paramName]: value } }; + }) + ); + }; + + return ( +
+
+ + +
+ +
+ {configs.map((config) => ( + + +
+ + updateConfig(config.id, { label: e.target.value })} + /> + +
+ + +
+
+ + +
+ + +
+ + {(backends[config.backend] ?? []).map((param) => ( +
+ + {param.param_type.type === "Select" && ( + + )} + {param.param_type.type === "Float" && ( + { + const val = parseFloat(e.target.value); + if (!isNaN(val)) { + updateParam(config.id, param.name, val); + } + }} + className="h-8 text-xs w-24" + /> + )} +
+ ))} + + + ))} +
+
+ ); +} diff --git a/tools/audio-lab/frontend/src/components/AsrTranscript.tsx b/tools/audio-lab/frontend/src/components/AsrTranscript.tsx new file mode 100644 index 0000000..ac0a45d --- /dev/null +++ b/tools/audio-lab/frontend/src/components/AsrTranscript.tsx @@ -0,0 +1,154 @@ +import { useEffect, useMemo, useRef } from "react"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { AsrConfig } from "@/lib/websocket"; + +export interface AsrTranscriptFinal { + ts_ms: number; + end_ms: number; + text: string; + confidence: number; +} + +export interface AsrTranscriptState { + ready: boolean; + finals: AsrTranscriptFinal[]; + partial: string | null; + warning: string | null; +} + +interface AsrTranscriptProps { + configs: AsrConfig[]; + states: Record; +} + +function formatMs(ms: number): string { + const totalSeconds = ms / 1000; + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds - minutes * 60; + return `${minutes.toString().padStart(2, "0")}:${seconds.toFixed(1).padStart(4, "0")}`; +} + +export function AsrTranscript({ configs, states }: AsrTranscriptProps) { + if (configs.length === 0) return null; + + return ( +
+ {configs.map((config) => ( + + ))} +
+ ); +} + +interface AsrTranscriptCardProps { + config: AsrConfig; + state: AsrTranscriptState | undefined; +} + +function AsrTranscriptCard({ config, state }: AsrTranscriptCardProps) { + const ready = state?.ready ?? false; + const finals = useMemo(() => state?.finals ?? [], [state?.finals]); + const partial = state?.partial ?? null; + const warning = state?.warning ?? null; + const preset = + typeof config.params.preset === "string" ? config.params.preset : "—"; + + const stats = useMemo(() => { + if (finals.length === 0) return null; + const lastFinal = finals[finals.length - 1]; + const avgDuration = + finals.reduce((sum, f) => sum + (f.end_ms - f.ts_ms), 0) / finals.length; + return { + count: finals.length, + lastConfidence: lastFinal.confidence, + avgDurationMs: avgDuration, + }; + }, [finals]); + + const scrollRef = useRef(null); + const autoScrollRef = useRef(true); + + useEffect(() => { + const el = scrollRef.current; + if (!el || !autoScrollRef.current) return; + el.scrollTop = el.scrollHeight; + }, [finals.length, partial]); + + const onScroll = () => { + const el = scrollRef.current; + if (!el) return; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 20; + autoScrollRef.current = atBottom; + }; + + const copyAll = () => { + const text = finals.map((f) => f.text).join("\n"); + navigator.clipboard.writeText(text).catch(() => {}); + }; + + return ( + + +
+ + ASR: {config.label}{" "} + + · {config.backend} · {preset} + + + +
+ + +
+ {!ready && !warning && ( +
loading model…
+ )} + {warning && ( +
⚠ {warning}
+ )} + {finals.map((f, i) => ( +
+ + [{formatMs(f.ts_ms)}–{formatMs(f.end_ms)}] + {" "} + {f.text} +
+ ))} + {partial && ( +
+ partial: {partial} +
+ )} +
+
+ {stats ? ( + + conf {stats.lastConfidence.toFixed(2)} · {stats.count} finals · avg{" "} + {(stats.avgDurationMs / 1000).toFixed(1)}s/segment + + ) : ( + {ready ? "waiting for speech…" : "—"} + )} +
+ + + ); +} diff --git a/tools/audio-lab/frontend/src/lib/websocket.ts b/tools/audio-lab/frontend/src/lib/websocket.ts index f885659..dc607be 100644 --- a/tools/audio-lab/frontend/src/lib/websocket.ts +++ b/tools/audio-lab/frontend/src/lib/websocket.ts @@ -24,6 +24,21 @@ export interface TurnConfig { params: Record; } +export interface AsrConfig { + id: string; + label: string; + backend: string; + params: Record; +} + +export type AsrEventKind = + | "ready" + | "speech_started" + | "speech_ended" + | "partial" + | "final" + | "warning"; + export interface PipelineConfig { id: string; label: string; @@ -64,6 +79,8 @@ export type ServerMessage = | { type: "turn_backends"; backends: Record } | { type: "turn"; config_id: string; timestamp_ms: number; state: string; confidence: number; latency_ms: number; stage_times: Array<{ name: string; us: number }> } | { type: "pipeline"; config_id: string; timestamp_ms: number; event: string; turn_state?: string; turn_confidence?: number; turn_latency_ms?: number; audio_duration_ms?: number } + | { type: "asr_backends"; backends: Record } + | { type: "asr"; config_id: string; kind: AsrEventKind; ts_ms?: number; end_ms?: number; text?: string; confidence?: number; message?: string } | { type: "done" } | { type: "error"; message: string }; @@ -78,7 +95,9 @@ export type ClientMessage = | { type: "set_spectrum_bins"; bins: number } | { type: "list_turn_backends" } | { type: "set_turn_configs"; configs: TurnConfig[] } - | { type: "set_pipeline_configs"; configs: PipelineConfig[] }; + | { type: "set_pipeline_configs"; configs: PipelineConfig[] } + | { type: "list_asr_backends" } + | { type: "set_asr_configs"; configs: AsrConfig[] }; export type MessageHandler = (msg: ServerMessage) => void; @@ -106,6 +125,7 @@ interface StreamBatch { preprocessedAudioFrames: Map; preprocessedSpectrumFrames: Map; vad: Map; + asrPartials: Map; } export class VadLabSocket { @@ -247,8 +267,14 @@ export class VadLabSocket { } private logServerMessage(msg: ServerMessage) { - if (msg.type === "audio" || msg.type === "vad" || msg.type === "spectrum" || - msg.type === "preprocessed_audio" || msg.type === "preprocessed_spectrum") { + if ( + msg.type === "audio" || + msg.type === "vad" || + msg.type === "spectrum" || + msg.type === "preprocessed_audio" || + msg.type === "preprocessed_spectrum" || + (msg.type === "asr" && msg.kind === "partial") + ) { this.addToBatch(msg); } else { // Flush any pending batch before logging a non-streaming message @@ -257,7 +283,7 @@ export class VadLabSocket { } } - private addToBatch(msg: ServerMessage & { type: "audio" | "vad" | "spectrum" | "preprocessed_audio" | "preprocessed_spectrum" }) { + private addToBatch(msg: ServerMessage & { type: "audio" | "vad" | "spectrum" | "preprocessed_audio" | "preprocessed_spectrum" | "asr" }) { if (!this.streamBatch) { this.streamBatch = { audioFrames: 0, @@ -267,6 +293,7 @@ export class VadLabSocket { preprocessedAudioFrames: new Map(), preprocessedSpectrumFrames: new Map(), vad: new Map(), + asrPartials: new Map(), }; this.startBatchTimer(); } @@ -288,6 +315,11 @@ export class VadLabSocket { msg.config_id, (batch.preprocessedSpectrumFrames.get(msg.config_id) ?? 0) + 1 ); + } else if (msg.type === "asr") { + batch.asrPartials.set( + msg.config_id, + (batch.asrPartials.get(msg.config_id) ?? 0) + 1 + ); } else { const existing = batch.vad.get(msg.config_id); if (existing) { @@ -363,6 +395,10 @@ export class VadLabSocket { } } + for (const [configId, count] of batch.asrPartials) { + parts.push(`asr [${configId}]: ${count} partials`); + } + if (parts.length > 0) { this.emitLog("recv", parts.join(" | ")); } @@ -382,6 +418,20 @@ function summarizeServer(msg: ServerMessage): string { case "turn_backends": return `turn_backends (${Object.keys(msg.backends).length})`; case "turn": return `turn [${msg.config_id}] t=${msg.timestamp_ms.toFixed(0)}ms state=${msg.state} conf=${msg.confidence.toFixed(2)} lat=${msg.latency_ms}ms`; case "pipeline": return `pipeline [${msg.config_id}] t=${msg.timestamp_ms.toFixed(0)}ms ${msg.event}${msg.turn_state ? ` ${msg.turn_state} ${((msg.turn_confidence ?? 0) * 100).toFixed(0)}%` : ""}`; + case "asr_backends": return `asr_backends (${Object.keys(msg.backends).length})`; + case "asr": { + const where = msg.ts_ms !== undefined ? ` t=${msg.ts_ms.toFixed(0)}ms` : ""; + if (msg.kind === "final") { + return `asr [${msg.config_id}] final${where} "${msg.text ?? ""}" conf=${(msg.confidence ?? 0).toFixed(2)}`; + } + if (msg.kind === "partial") { + return `asr [${msg.config_id}] partial${where} "${msg.text ?? ""}"`; + } + if (msg.kind === "warning") { + return `asr [${msg.config_id}] warning: ${msg.message ?? ""}`; + } + return `asr [${msg.config_id}] ${msg.kind}${where}`; + } case "done": return "done"; case "error": return `error: ${msg.message}`; } @@ -399,5 +449,7 @@ function summarizeClient(msg: ClientMessage): string { case "list_turn_backends": return "list_turn_backends"; case "set_turn_configs": return `set_turn_configs (${msg.configs.length})`; case "set_pipeline_configs": return `set_pipeline_configs (${msg.configs.length})`; + case "list_asr_backends": return "list_asr_backends"; + case "set_asr_configs": return `set_asr_configs (${msg.configs.length})`; } }