-
+ {browserDictation ? (
+
+ ) : (
+
+ )}
;
+}
+
+export interface BrowserSpeechRecognition {
+ lang: string;
+ continuous: boolean;
+ interimResults: boolean;
+ onresult: ((event: SpeechResultEvent) => void) | null;
+ onerror: ((event: { error: string }) => void) | null;
+ onend: (() => void) | null;
+ start: () => void;
+ stop: () => void;
+ abort: () => void;
+}
+
+type SpeechWindow = Window & {
+ SpeechRecognition?: new () => BrowserSpeechRecognition;
+ webkitSpeechRecognition?: new () => BrowserSpeechRecognition;
+};
+
+export function useBrowserSpeechRecognition({
+ enabled,
+ language,
+ onTranscript,
+ onError
+}: {
+ enabled: boolean;
+ language: string;
+ onTranscript: (text: string) => void;
+ onError: (error: string) => void;
+}) {
+ const [listening, setListening] = useState(false);
+ const recognitionRef = useRef(null);
+ const callbacks = useRef({ onTranscript, onError });
+ callbacks.current = { onTranscript, onError };
+ const speechWindow = window as SpeechWindow;
+ const Recognition =
+ speechWindow.SpeechRecognition || speechWindow.webkitSpeechRecognition;
+
+ const dispose = useCallback(() => {
+ const recognition = recognitionRef.current;
+ recognitionRef.current = null;
+ if (recognition) {
+ recognition.onresult = null;
+ recognition.onerror = null;
+ recognition.onend = null;
+ recognition.abort();
+ }
+ }, []);
+
+ const cancel = useCallback(() => {
+ dispose();
+ setListening(false);
+ }, [dispose]);
+
+ useEffect(() => {
+ if (!enabled) cancel();
+ return dispose;
+ }, [enabled, cancel, dispose]);
+
+ const toggle = useCallback(() => {
+ if (!enabled || !Recognition) return;
+ if (recognitionRef.current) {
+ recognitionRef.current.stop();
+ return;
+ }
+
+ const recognition = new Recognition();
+ recognitionRef.current = recognition;
+ recognition.lang = language;
+ recognition.continuous = true;
+ recognition.interimResults = false;
+ const delivered = new Set();
+ recognition.onresult = (event) => {
+ const transcripts: string[] = [];
+ for (let i = event.resultIndex; i < event.results.length; i++) {
+ const result = event.results[i];
+ if (result.isFinal && !delivered.has(i)) {
+ delivered.add(i);
+ transcripts.push(result[0].transcript.trim());
+ }
+ }
+ const transcript = transcripts.filter(Boolean).join(' ');
+ if (transcript) callbacks.current.onTranscript(transcript);
+ };
+ recognition.onend = () => {
+ recognitionRef.current = null;
+ recognition.onresult = null;
+ recognition.onerror = null;
+ recognition.onend = null;
+ setListening(false);
+ };
+ recognition.onerror = (event) => {
+ cancel();
+ if (event.error !== 'aborted' && event.error !== 'no-speech') {
+ callbacks.current.onError(event.error);
+ }
+ };
+ try {
+ setListening(true);
+ recognition.start();
+ } catch (error) {
+ cancel();
+ callbacks.current.onError(
+ error instanceof Error ? error.message : 'unknown'
+ );
+ }
+ }, [enabled, Recognition, language, cancel]);
+
+ return { supported: !!Recognition, listening, toggle, cancel };
+}
diff --git a/frontend/tests/browserSpeechRecognition.spec.tsx b/frontend/tests/browserSpeechRecognition.spec.tsx
new file mode 100644
index 0000000000..223b1977ee
--- /dev/null
+++ b/frontend/tests/browserSpeechRecognition.spec.tsx
@@ -0,0 +1,157 @@
+import { act, renderHook } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import {
+ BrowserSpeechRecognition,
+ SpeechResultEvent,
+ useBrowserSpeechRecognition
+} from '@/hooks/useBrowserSpeechRecognition';
+
+class Recognition implements BrowserSpeechRecognition {
+ static instances: Recognition[] = [];
+ lang = '';
+ continuous = false;
+ interimResults = false;
+ onresult: BrowserSpeechRecognition['onresult'] = null;
+ onerror: BrowserSpeechRecognition['onerror'] = null;
+ onend: BrowserSpeechRecognition['onend'] = null;
+ start = vi.fn();
+ stop = vi.fn();
+ abort = vi.fn();
+ constructor() {
+ Recognition.instances.push(this);
+ }
+}
+
+const finalResult = (transcript: string) => ({
+ isFinal: true,
+ 0: { transcript }
+});
+
+describe('browser speech recognition', () => {
+ beforeEach(() => {
+ Recognition.instances = [];
+ vi.stubGlobal('SpeechRecognition', Recognition);
+ vi.stubGlobal('webkitSpeechRecognition', undefined);
+ });
+ afterEach(() => vi.unstubAllGlobals());
+
+ const setup = (enabled = true) => {
+ const onTranscript = vi.fn();
+ const onError = vi.fn();
+ const hook = renderHook(
+ ({ enabled }) =>
+ useBrowserSpeechRecognition({
+ enabled,
+ language: 'fr-FR',
+ onTranscript,
+ onError
+ }),
+ { initialProps: { enabled } }
+ );
+ return { ...hook, onTranscript, onError };
+ };
+
+ it('starts only on request and uses the selected language', () => {
+ const { result } = setup();
+ expect(Recognition.instances).toHaveLength(0);
+ act(() => result.current.toggle());
+ expect(Recognition.instances[0].start).toHaveBeenCalledOnce();
+ expect(Recognition.instances[0].lang).toBe('fr-FR');
+ expect(result.current.listening).toBe(true);
+ });
+
+ it('uses the WebKit-prefixed API when needed', () => {
+ vi.stubGlobal('SpeechRecognition', undefined);
+ vi.stubGlobal('webkitSpeechRecognition', Recognition);
+ const { result } = setup();
+ expect(result.current.supported).toBe(true);
+ act(() => result.current.toggle());
+ expect(Recognition.instances[0].start).toHaveBeenCalledOnce();
+ });
+
+ it('does not start when disabled or unsupported', () => {
+ const { result } = setup(false);
+ act(() => result.current.toggle());
+ expect(Recognition.instances).toHaveLength(0);
+ vi.stubGlobal('SpeechRecognition', undefined);
+ const unsupported = setup();
+ expect(unsupported.result.current.supported).toBe(false);
+ act(() => unsupported.result.current.toggle());
+ expect(Recognition.instances).toHaveLength(0);
+ });
+
+ it('delivers final results once and ignores interim results', () => {
+ const { result, onTranscript } = setup();
+ act(() => result.current.toggle());
+ const event: SpeechResultEvent = {
+ resultIndex: 0,
+ results: [
+ finalResult(' Hello '),
+ { isFinal: false, 0: { transcript: 'wor' } }
+ ]
+ };
+ act(() => Recognition.instances[0].onresult?.(event));
+ expect(onTranscript).toHaveBeenCalledWith('Hello');
+ event.resultIndex = 1;
+ event.results = [finalResult('Hello'), finalResult('world')];
+ act(() => Recognition.instances[0].onresult?.(event));
+ act(() => Recognition.instances[0].onresult?.(event));
+ expect(onTranscript.mock.calls).toEqual([['Hello'], ['world']]);
+ });
+
+ it('keeps final results when stopping and resets when the browser ends', () => {
+ const { result, onTranscript } = setup();
+ act(() => result.current.toggle());
+ act(() => result.current.toggle());
+ const recognition = Recognition.instances[0];
+ expect(recognition.stop).toHaveBeenCalledOnce();
+ act(() =>
+ recognition.onresult?.({ resultIndex: 0, results: [finalResult('Done')] })
+ );
+ act(() => recognition.onend?.());
+ expect(onTranscript).toHaveBeenCalledWith('Done');
+ expect(result.current.listening).toBe(false);
+ act(() => result.current.toggle());
+ expect(Recognition.instances).toHaveLength(2);
+ });
+
+ it.each(['cancel', 'disable', 'unmount'])(
+ 'aborts on %s and detaches callbacks',
+ (action) => {
+ const { result, rerender, unmount, onTranscript } = setup();
+ act(() => result.current.toggle());
+ const recognition = Recognition.instances[0];
+ if (action === 'cancel') act(() => result.current.cancel());
+ if (action === 'disable') rerender({ enabled: false });
+ if (action === 'unmount') unmount();
+ expect(recognition.abort).toHaveBeenCalledOnce();
+ expect(recognition.onresult).toBeNull();
+ expect(recognition.onend).toBeNull();
+ expect(onTranscript).not.toHaveBeenCalled();
+ }
+ );
+
+ it('reports microphone permission errors and allows retrying', () => {
+ const { result, onError } = setup();
+ act(() => result.current.toggle());
+ act(() => Recognition.instances[0].onerror?.({ error: 'not-allowed' }));
+ expect(onError).toHaveBeenCalledWith('not-allowed');
+ expect(result.current.listening).toBe(false);
+ act(() => result.current.toggle());
+ expect(Recognition.instances).toHaveLength(2);
+ });
+
+ it('recovers when starting throws', () => {
+ class FailingRecognition extends Recognition {
+ start = vi.fn(() => {
+ throw new Error('microphone unavailable');
+ });
+ }
+ vi.stubGlobal('SpeechRecognition', FailingRecognition);
+ const { result, onError } = setup();
+ act(() => result.current.toggle());
+ expect(onError).toHaveBeenCalledWith('microphone unavailable');
+ expect(result.current.listening).toBe(false);
+ });
+});
diff --git a/libs/react-client/src/types/config.ts b/libs/react-client/src/types/config.ts
index df06adbe39..4c602926e0 100644
--- a/libs/react-client/src/types/config.ts
+++ b/libs/react-client/src/types/config.ts
@@ -23,6 +23,7 @@ export interface ChatProfile {
export interface IAudioConfig {
enabled: boolean;
sample_rate: number;
+ mode?: 'realtime' | 'browser';
}
export interface IAuthConfig {
+### Browser dictation
+
+To dictate text into the message composer without Python audio callbacks, set the following in `.chainlit/config.toml`:
+
+```toml
+[features.audio]
+enabled = true
+mode = "browser"
+```
+
+Click the microphone, allow microphone access, and speak. You can stop recording, edit the recognized text, and send it as a normal message. The existing audio streaming mode remains the default (`mode = "realtime"`).
+
+Browser dictation requires Web Speech API support, such as `SpeechRecognition` or `webkitSpeechRecognition`. Some browsers send audio to their vendor's recognition service, so this mode does not guarantee offline processing. The microphone button is disabled when the browser does not support recognition.
+
## 📚 More Examples - Cookbook
You can find various examples of Chainlit apps [here](https://github.com/Chainlit/cookbook) that leverage tools and services such as OpenAI, Anthropiс, LangChain, LlamaIndex, ChromaDB, Pinecone and more.
diff --git a/backend/chainlit/config.py b/backend/chainlit/config.py
index db03d4c060..e380a709bd 100644
--- a/backend/chainlit/config.py
+++ b/backend/chainlit/config.py
@@ -142,6 +142,9 @@
[features.audio]
# Enable audio features
enabled = false
+ # "realtime" streams audio to Python callbacks; "browser" dictates into the composer.
+ # Browser dictation requires Web Speech API support and may use a browser vendor's cloud service.
+ mode = "realtime"
# Sample rate of the audio
sample_rate = 24000
@@ -315,6 +318,7 @@ class SpontaneousFileUploadFeature(BaseModel):
class AudioFeature(BaseModel):
sample_rate: int = 24000
enabled: bool = False
+ mode: Literal["realtime", "browser"] = "realtime"
class SlackFeature(BaseModel):
diff --git a/backend/chainlit/translations/en-US.json b/backend/chainlit/translations/en-US.json
index f20f6a724d..aa4d4c6d3e 100644
--- a/backend/chainlit/translations/en-US.json
+++ b/backend/chainlit/translations/en-US.json
@@ -82,7 +82,10 @@
"speech": {
"start": "Start recording",
"stop": "Stop recording",
- "connecting": "Connecting"
+ "connecting": "Connecting",
+ "dictate": "Dictate a message",
+ "unsupported": "Speech recognition is not supported by this browser",
+ "failed": "Speech recognition failed: {{error}}"
},
"fileUpload": {
"dragDrop": "Drag and drop files here",
diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py
index 8fd1febfd5..42b1193f6e 100644
--- a/backend/tests/test_config.py
+++ b/backend/tests/test_config.py
@@ -7,6 +7,7 @@
from chainlit import config as chainlit_config
from chainlit.config import (
+ AudioFeature,
ChainlitConfig,
ChainlitConfigOverrides,
FeaturesSettings,
@@ -19,6 +20,20 @@
from chainlit.version import __version__
+def test_audio_mode_defaults_to_realtime():
+ assert AudioFeature(enabled=True).mode == "realtime"
+
+
+def test_browser_audio_mode_is_serialized():
+ features = FeaturesSettings(audio={"enabled": True, "mode": "browser"})
+ assert features.model_dump()["audio"]["mode"] == "browser"
+
+
+def test_unknown_audio_mode_is_rejected():
+ with pytest.raises(ValidationError, match="mode"):
+ AudioFeature(mode="unknown")
+
+
@pytest.fixture
def translation_dir(tmp_path: Path) -> Path:
"""Minimal translation directory with a controlled set of locale files."""
diff --git a/frontend/src/components/chat/MessageComposer/DictationButton.tsx b/frontend/src/components/chat/MessageComposer/DictationButton.tsx
new file mode 100644
index 0000000000..81949265a4
--- /dev/null
+++ b/frontend/src/components/chat/MessageComposer/DictationButton.tsx
@@ -0,0 +1,57 @@
+import { Mic, Square } from 'lucide-react';
+
+import { useTranslation } from '@/components/i18n/Translator';
+import { Button } from '@/components/ui/button';
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipProvider,
+ TooltipTrigger
+} from '@/components/ui/tooltip';
+
+export default function DictationButton({
+ disabled,
+ supported,
+ listening,
+ onClick
+}: {
+ disabled: boolean;
+ supported: boolean;
+ listening: boolean;
+ onClick: () => void;
+}) {
+ const { t } = useTranslation();
+ const label = t(
+ !supported
+ ? 'chat.speech.unsupported'
+ : listening
+ ? 'chat.speech.stop'
+ : 'chat.speech.dictate'
+ );
+ return (
+