From 86cec9aed09008bcb7cb0213a3f87cf42e52cac7 Mon Sep 17 00:00:00 2001 From: Botond Berde <42468304+berdebotond@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:45:03 +0100 Subject: [PATCH] feat(audio): add opt-in browser dictation Co-Authored-By: Codex --- backend/README.md | 14 ++ backend/chainlit/config.py | 4 + backend/chainlit/translations/en-US.json | 5 +- backend/tests/test_config.py | 15 ++ .../chat/MessageComposer/DictationButton.tsx | 57 +++++++ .../components/chat/MessageComposer/index.tsx | 31 +++- .../src/hooks/useBrowserSpeechRecognition.ts | 118 +++++++++++++ .../tests/browserSpeechRecognition.spec.tsx | 157 ++++++++++++++++++ libs/react-client/src/types/config.ts | 1 + 9 files changed, 398 insertions(+), 4 deletions(-) create mode 100644 frontend/src/components/chat/MessageComposer/DictationButton.tsx create mode 100644 frontend/src/hooks/useBrowserSpeechRecognition.ts create mode 100644 frontend/tests/browserSpeechRecognition.spec.tsx diff --git a/backend/README.md b/backend/README.md index e19a982147..3dd12d3e7a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -102,6 +102,20 @@ chainlit run demo.py -w Quick Start +### Browser dictation + +To dictate text into the message composer without Python audio callbacks, set the following in `.chainlit/config.toml`: + +```toml +[features.audio] +enabled = true +mode = "browser" +``` + +Click the microphone, allow microphone access, and speak. You can stop recording, edit the recognized text, and send it as a normal message. The existing audio streaming mode remains the default (`mode = "realtime"`). + +Browser dictation requires Web Speech API support, such as `SpeechRecognition` or `webkitSpeechRecognition`. Some browsers send audio to their vendor's recognition service, so this mode does not guarantee offline processing. The microphone button is disabled when the browser does not support recognition. + ## 📚 More Examples - Cookbook You can find various examples of Chainlit apps [here](https://github.com/Chainlit/cookbook) that leverage tools and services such as OpenAI, Anthropiс, LangChain, LlamaIndex, ChromaDB, Pinecone and more. diff --git a/backend/chainlit/config.py b/backend/chainlit/config.py index db03d4c060..e380a709bd 100644 --- a/backend/chainlit/config.py +++ b/backend/chainlit/config.py @@ -142,6 +142,9 @@ [features.audio] # Enable audio features enabled = false + # "realtime" streams audio to Python callbacks; "browser" dictates into the composer. + # Browser dictation requires Web Speech API support and may use a browser vendor's cloud service. + mode = "realtime" # Sample rate of the audio sample_rate = 24000 @@ -315,6 +318,7 @@ class SpontaneousFileUploadFeature(BaseModel): class AudioFeature(BaseModel): sample_rate: int = 24000 enabled: bool = False + mode: Literal["realtime", "browser"] = "realtime" class SlackFeature(BaseModel): diff --git a/backend/chainlit/translations/en-US.json b/backend/chainlit/translations/en-US.json index f20f6a724d..aa4d4c6d3e 100644 --- a/backend/chainlit/translations/en-US.json +++ b/backend/chainlit/translations/en-US.json @@ -82,7 +82,10 @@ "speech": { "start": "Start recording", "stop": "Stop recording", - "connecting": "Connecting" + "connecting": "Connecting", + "dictate": "Dictate a message", + "unsupported": "Speech recognition is not supported by this browser", + "failed": "Speech recognition failed: {{error}}" }, "fileUpload": { "dragDrop": "Drag and drop files here", diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py index 8fd1febfd5..42b1193f6e 100644 --- a/backend/tests/test_config.py +++ b/backend/tests/test_config.py @@ -7,6 +7,7 @@ from chainlit import config as chainlit_config from chainlit.config import ( + AudioFeature, ChainlitConfig, ChainlitConfigOverrides, FeaturesSettings, @@ -19,6 +20,20 @@ from chainlit.version import __version__ +def test_audio_mode_defaults_to_realtime(): + assert AudioFeature(enabled=True).mode == "realtime" + + +def test_browser_audio_mode_is_serialized(): + features = FeaturesSettings(audio={"enabled": True, "mode": "browser"}) + assert features.model_dump()["audio"]["mode"] == "browser" + + +def test_unknown_audio_mode_is_rejected(): + with pytest.raises(ValidationError, match="mode"): + AudioFeature(mode="unknown") + + @pytest.fixture def translation_dir(tmp_path: Path) -> Path: """Minimal translation directory with a controlled set of locale files.""" diff --git a/frontend/src/components/chat/MessageComposer/DictationButton.tsx b/frontend/src/components/chat/MessageComposer/DictationButton.tsx new file mode 100644 index 0000000000..81949265a4 --- /dev/null +++ b/frontend/src/components/chat/MessageComposer/DictationButton.tsx @@ -0,0 +1,57 @@ +import { Mic, Square } from 'lucide-react'; + +import { useTranslation } from '@/components/i18n/Translator'; +import { Button } from '@/components/ui/button'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger +} from '@/components/ui/tooltip'; + +export default function DictationButton({ + disabled, + supported, + listening, + onClick +}: { + disabled: boolean; + supported: boolean; + listening: boolean; + onClick: () => void; +}) { + const { t } = useTranslation(); + const label = t( + !supported + ? 'chat.speech.unsupported' + : listening + ? 'chat.speech.stop' + : 'chat.speech.dictate' + ); + return ( + + + + + + + + {label} + + + ); +} diff --git a/frontend/src/components/chat/MessageComposer/index.tsx b/frontend/src/components/chat/MessageComposer/index.tsx index cbfb4c9a1d..591b85ce9b 100644 --- a/frontend/src/components/chat/MessageComposer/index.tsx +++ b/frontend/src/components/chat/MessageComposer/index.tsx @@ -6,6 +6,7 @@ import { useState } from 'react'; import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil'; +import { toast } from 'sonner'; import { v4 as uuidv4 } from 'uuid'; import { @@ -32,6 +33,7 @@ import { useTranslation } from 'components/i18n/Translator'; import { useQuery } from '@/hooks/query'; import { useIsMobile } from '@/hooks/use-mobile'; +import { useBrowserSpeechRecognition } from '@/hooks/useBrowserSpeechRecognition'; import { chatSettingsOpenState } from '@/state/project'; import { @@ -43,6 +45,7 @@ import { import { Attachments } from './Attachments'; import CommandButtons from './CommandButtons'; import CommandButton from './CommandPopoverButton'; +import DictationButton from './DictationButton'; import FavoriteButton from './FavoriteButton'; import Input, { InputMethods } from './Input'; import McpButton from './Mcp'; @@ -80,7 +83,7 @@ export default function MessageComposer({ } }, [commands]); const [attachments, setAttachments] = useRecoilState(attachmentsState); - const { t } = useTranslation(); + const { t, i18n } = useTranslation(); const { user } = useAuth(); const { sendMessage, replyMessage } = useChatInteract(); @@ -89,6 +92,17 @@ export default function MessageComposer({ const disabled = _disabled || !!attachments.find((a) => !a.uploaded); const { config } = useConfig(); + const browserDictation = + !!config?.features.audio.enabled && + config.features.audio.mode === 'browser'; + const dictation = useBrowserSpeechRecognition({ + enabled: browserDictation && !disabled, + language: i18n.language, + onTranscript: (text) => { + inputRef.current?.setValueExtern([value, text].filter(Boolean).join(' ')); + }, + onError: (error) => toast.error(t('chat.speech.failed', { error })) + }); const showSettingsInComposer = config?.ui?.chat_settings_location !== 'sidebar' && chatSettingsInputs.length > 0; @@ -231,6 +245,7 @@ export default function MessageComposer({ onSubmit(value, attachments, selectedCommand?.id); } + dictation.cancel(); setAttachments([]); setValue(''); // Clear the value state inputRef.current?.reset(); @@ -242,7 +257,8 @@ export default function MessageComposer({ selectedCommand, setAttachments, onSubmit, - onReply + onReply, + dictation.cancel ]); useEffect(() => { @@ -282,7 +298,16 @@ export default function MessageComposer({ />
- + {browserDictation ? ( + + ) : ( + + )} ; +} + +export interface BrowserSpeechRecognition { + lang: string; + continuous: boolean; + interimResults: boolean; + onresult: ((event: SpeechResultEvent) => void) | null; + onerror: ((event: { error: string }) => void) | null; + onend: (() => void) | null; + start: () => void; + stop: () => void; + abort: () => void; +} + +type SpeechWindow = Window & { + SpeechRecognition?: new () => BrowserSpeechRecognition; + webkitSpeechRecognition?: new () => BrowserSpeechRecognition; +}; + +export function useBrowserSpeechRecognition({ + enabled, + language, + onTranscript, + onError +}: { + enabled: boolean; + language: string; + onTranscript: (text: string) => void; + onError: (error: string) => void; +}) { + const [listening, setListening] = useState(false); + const recognitionRef = useRef(null); + const callbacks = useRef({ onTranscript, onError }); + callbacks.current = { onTranscript, onError }; + const speechWindow = window as SpeechWindow; + const Recognition = + speechWindow.SpeechRecognition || speechWindow.webkitSpeechRecognition; + + const dispose = useCallback(() => { + const recognition = recognitionRef.current; + recognitionRef.current = null; + if (recognition) { + recognition.onresult = null; + recognition.onerror = null; + recognition.onend = null; + recognition.abort(); + } + }, []); + + const cancel = useCallback(() => { + dispose(); + setListening(false); + }, [dispose]); + + useEffect(() => { + if (!enabled) cancel(); + return dispose; + }, [enabled, cancel, dispose]); + + const toggle = useCallback(() => { + if (!enabled || !Recognition) return; + if (recognitionRef.current) { + recognitionRef.current.stop(); + return; + } + + const recognition = new Recognition(); + recognitionRef.current = recognition; + recognition.lang = language; + recognition.continuous = true; + recognition.interimResults = false; + const delivered = new Set(); + recognition.onresult = (event) => { + const transcripts: string[] = []; + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i]; + if (result.isFinal && !delivered.has(i)) { + delivered.add(i); + transcripts.push(result[0].transcript.trim()); + } + } + const transcript = transcripts.filter(Boolean).join(' '); + if (transcript) callbacks.current.onTranscript(transcript); + }; + recognition.onend = () => { + recognitionRef.current = null; + recognition.onresult = null; + recognition.onerror = null; + recognition.onend = null; + setListening(false); + }; + recognition.onerror = (event) => { + cancel(); + if (event.error !== 'aborted' && event.error !== 'no-speech') { + callbacks.current.onError(event.error); + } + }; + try { + setListening(true); + recognition.start(); + } catch (error) { + cancel(); + callbacks.current.onError( + error instanceof Error ? error.message : 'unknown' + ); + } + }, [enabled, Recognition, language, cancel]); + + return { supported: !!Recognition, listening, toggle, cancel }; +} diff --git a/frontend/tests/browserSpeechRecognition.spec.tsx b/frontend/tests/browserSpeechRecognition.spec.tsx new file mode 100644 index 0000000000..223b1977ee --- /dev/null +++ b/frontend/tests/browserSpeechRecognition.spec.tsx @@ -0,0 +1,157 @@ +import { act, renderHook } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + BrowserSpeechRecognition, + SpeechResultEvent, + useBrowserSpeechRecognition +} from '@/hooks/useBrowserSpeechRecognition'; + +class Recognition implements BrowserSpeechRecognition { + static instances: Recognition[] = []; + lang = ''; + continuous = false; + interimResults = false; + onresult: BrowserSpeechRecognition['onresult'] = null; + onerror: BrowserSpeechRecognition['onerror'] = null; + onend: BrowserSpeechRecognition['onend'] = null; + start = vi.fn(); + stop = vi.fn(); + abort = vi.fn(); + constructor() { + Recognition.instances.push(this); + } +} + +const finalResult = (transcript: string) => ({ + isFinal: true, + 0: { transcript } +}); + +describe('browser speech recognition', () => { + beforeEach(() => { + Recognition.instances = []; + vi.stubGlobal('SpeechRecognition', Recognition); + vi.stubGlobal('webkitSpeechRecognition', undefined); + }); + afterEach(() => vi.unstubAllGlobals()); + + const setup = (enabled = true) => { + const onTranscript = vi.fn(); + const onError = vi.fn(); + const hook = renderHook( + ({ enabled }) => + useBrowserSpeechRecognition({ + enabled, + language: 'fr-FR', + onTranscript, + onError + }), + { initialProps: { enabled } } + ); + return { ...hook, onTranscript, onError }; + }; + + it('starts only on request and uses the selected language', () => { + const { result } = setup(); + expect(Recognition.instances).toHaveLength(0); + act(() => result.current.toggle()); + expect(Recognition.instances[0].start).toHaveBeenCalledOnce(); + expect(Recognition.instances[0].lang).toBe('fr-FR'); + expect(result.current.listening).toBe(true); + }); + + it('uses the WebKit-prefixed API when needed', () => { + vi.stubGlobal('SpeechRecognition', undefined); + vi.stubGlobal('webkitSpeechRecognition', Recognition); + const { result } = setup(); + expect(result.current.supported).toBe(true); + act(() => result.current.toggle()); + expect(Recognition.instances[0].start).toHaveBeenCalledOnce(); + }); + + it('does not start when disabled or unsupported', () => { + const { result } = setup(false); + act(() => result.current.toggle()); + expect(Recognition.instances).toHaveLength(0); + vi.stubGlobal('SpeechRecognition', undefined); + const unsupported = setup(); + expect(unsupported.result.current.supported).toBe(false); + act(() => unsupported.result.current.toggle()); + expect(Recognition.instances).toHaveLength(0); + }); + + it('delivers final results once and ignores interim results', () => { + const { result, onTranscript } = setup(); + act(() => result.current.toggle()); + const event: SpeechResultEvent = { + resultIndex: 0, + results: [ + finalResult(' Hello '), + { isFinal: false, 0: { transcript: 'wor' } } + ] + }; + act(() => Recognition.instances[0].onresult?.(event)); + expect(onTranscript).toHaveBeenCalledWith('Hello'); + event.resultIndex = 1; + event.results = [finalResult('Hello'), finalResult('world')]; + act(() => Recognition.instances[0].onresult?.(event)); + act(() => Recognition.instances[0].onresult?.(event)); + expect(onTranscript.mock.calls).toEqual([['Hello'], ['world']]); + }); + + it('keeps final results when stopping and resets when the browser ends', () => { + const { result, onTranscript } = setup(); + act(() => result.current.toggle()); + act(() => result.current.toggle()); + const recognition = Recognition.instances[0]; + expect(recognition.stop).toHaveBeenCalledOnce(); + act(() => + recognition.onresult?.({ resultIndex: 0, results: [finalResult('Done')] }) + ); + act(() => recognition.onend?.()); + expect(onTranscript).toHaveBeenCalledWith('Done'); + expect(result.current.listening).toBe(false); + act(() => result.current.toggle()); + expect(Recognition.instances).toHaveLength(2); + }); + + it.each(['cancel', 'disable', 'unmount'])( + 'aborts on %s and detaches callbacks', + (action) => { + const { result, rerender, unmount, onTranscript } = setup(); + act(() => result.current.toggle()); + const recognition = Recognition.instances[0]; + if (action === 'cancel') act(() => result.current.cancel()); + if (action === 'disable') rerender({ enabled: false }); + if (action === 'unmount') unmount(); + expect(recognition.abort).toHaveBeenCalledOnce(); + expect(recognition.onresult).toBeNull(); + expect(recognition.onend).toBeNull(); + expect(onTranscript).not.toHaveBeenCalled(); + } + ); + + it('reports microphone permission errors and allows retrying', () => { + const { result, onError } = setup(); + act(() => result.current.toggle()); + act(() => Recognition.instances[0].onerror?.({ error: 'not-allowed' })); + expect(onError).toHaveBeenCalledWith('not-allowed'); + expect(result.current.listening).toBe(false); + act(() => result.current.toggle()); + expect(Recognition.instances).toHaveLength(2); + }); + + it('recovers when starting throws', () => { + class FailingRecognition extends Recognition { + start = vi.fn(() => { + throw new Error('microphone unavailable'); + }); + } + vi.stubGlobal('SpeechRecognition', FailingRecognition); + const { result, onError } = setup(); + act(() => result.current.toggle()); + expect(onError).toHaveBeenCalledWith('microphone unavailable'); + expect(result.current.listening).toBe(false); + }); +}); diff --git a/libs/react-client/src/types/config.ts b/libs/react-client/src/types/config.ts index df06adbe39..4c602926e0 100644 --- a/libs/react-client/src/types/config.ts +++ b/libs/react-client/src/types/config.ts @@ -23,6 +23,7 @@ export interface ChatProfile { export interface IAudioConfig { enabled: boolean; sample_rate: number; + mode?: 'realtime' | 'browser'; } export interface IAuthConfig {