feat(audio): add opt-in browser dictation

backend/README.md

@@ -102,6 +102,20 @@ chainlit run demo.py -w
 
 <img src="/images/quick-start.png" alt="Quick Start"></img>
 
+### Browser dictation
+
+To dictate text into the message composer without Python audio callbacks, set the following in `.chainlit/config.toml`:
+
+```toml
+[features.audio]
+enabled = true
+mode = "browser"
+```
+
+Click the microphone, allow microphone access, and speak. You can stop recording, edit the recognized text, and send it as a normal message. The existing audio streaming mode remains the default (`mode = "realtime"`).
+
+Browser dictation requires Web Speech API support, such as `SpeechRecognition` or `webkitSpeechRecognition`. Some browsers send audio to their vendor's recognition service, so this mode does not guarantee offline processing. The microphone button is disabled when the browser does not support recognition.
+
 ## 📚 More Examples - Cookbook
 
 You can find various examples of Chainlit apps [here](https://github.com/Chainlit/cookbook) that leverage tools and services such as OpenAI, Anthropiс, LangChain, LlamaIndex, ChromaDB, Pinecone and more.

backend/chainlit/config.py

@@ -142,6 +142,9 @@
 [features.audio]
     # Enable audio features
     enabled = false
+    # "realtime" streams audio to Python callbacks; "browser" dictates into the composer.
+    # Browser dictation requires Web Speech API support and may use a browser vendor's cloud service.
+    mode = "realtime"
     # Sample rate of the audio
     sample_rate = 24000
 
@@ -315,6 +318,7 @@ class SpontaneousFileUploadFeature(BaseModel):
 class AudioFeature(BaseModel):
     sample_rate: int = 24000
     enabled: bool = False
+    mode: Literal["realtime", "browser"] = "realtime"
 
 
 class SlackFeature(BaseModel):

backend/chainlit/translations/en-US.json

@@ -82,7 +82,10 @@
     "speech": {
       "start": "Start recording",
       "stop": "Stop recording",
-      "connecting": "Connecting"
+      "connecting": "Connecting",
+      "dictate": "Dictate a message",
+      "unsupported": "Speech recognition is not supported by this browser",
+      "failed": "Speech recognition failed: {{error}}"
     },
     "fileUpload": {
       "dragDrop": "Drag and drop files here",

backend/tests/test_config.py

@@ -7,6 +7,7 @@
 
 from chainlit import config as chainlit_config
 from chainlit.config import (
+    AudioFeature,
     ChainlitConfig,
     ChainlitConfigOverrides,
     FeaturesSettings,
@@ -19,6 +20,20 @@
 from chainlit.version import __version__
 
 
+def test_audio_mode_defaults_to_realtime():
+    assert AudioFeature(enabled=True).mode == "realtime"
+
+
+def test_browser_audio_mode_is_serialized():
+    features = FeaturesSettings(audio={"enabled": True, "mode": "browser"})
+    assert features.model_dump()["audio"]["mode"] == "browser"
+
+
+def test_unknown_audio_mode_is_rejected():
+    with pytest.raises(ValidationError, match="mode"):
+        AudioFeature(mode="unknown")
+
+
 @pytest.fixture
 def translation_dir(tmp_path: Path) -> Path:
     """Minimal translation directory with a controlled set of locale files."""

frontend/src/components/chat/MessageComposer/DictationButton.tsx

@@ -0,0 +1,57 @@
+import { Mic, Square } from 'lucide-react';
+
+import { useTranslation } from '@/components/i18n/Translator';
+import { Button } from '@/components/ui/button';
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger
+} from '@/components/ui/tooltip';
+
+export default function DictationButton({
+  disabled,
+  supported,
+  listening,
+  onClick
+}: {
+  disabled: boolean;
+  supported: boolean;
+  listening: boolean;
+  onClick: () => void;
+}) {
+  const { t } = useTranslation();
+  const label = t(
+    !supported
+      ? 'chat.speech.unsupported'
+      : listening
+        ? 'chat.speech.stop'
+        : 'chat.speech.dictate'
+  );
+  return (
+    <TooltipProvider>
+      <Tooltip>
+        <TooltipTrigger asChild>
+          <span>
+            <Button
+              type="button"
+              disabled={disabled || !supported}
+              variant="ghost"
+              size="icon"
+              aria-label={label}
+              aria-pressed={listening}
+              onClick={onClick}
+            >
+              {listening ? (
+                <Square className="!size-5" />
+              ) : (
+                <Mic className="!size-5" />
+              )}
+            </Button>
+          </span>
+        </TooltipTrigger>
+        <TooltipContent>{label}</TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
+  );
+}

frontend/src/components/chat/MessageComposer/index.tsx

@@ -6,6 +6,7 @@ import {
   useState
 } from 'react';
 import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil';
+import { toast } from 'sonner';
 import { v4 as uuidv4 } from 'uuid';
 
 import {
@@ -32,6 +33,7 @@ import { useTranslation } from 'components/i18n/Translator';
 
 import { useQuery } from '@/hooks/query';
 import { useIsMobile } from '@/hooks/use-mobile';
+import { useBrowserSpeechRecognition } from '@/hooks/useBrowserSpeechRecognition';
 
 import { chatSettingsOpenState } from '@/state/project';
 import {
@@ -43,6 +45,7 @@ import {
 import { Attachments } from './Attachments';
 import CommandButtons from './CommandButtons';
 import CommandButton from './CommandPopoverButton';
+import DictationButton from './DictationButton';
 import FavoriteButton from './FavoriteButton';
 import Input, { InputMethods } from './Input';
 import McpButton from './Mcp';
@@ -80,7 +83,7 @@ export default function MessageComposer({
     }
   }, [commands]);
   const [attachments, setAttachments] = useRecoilState(attachmentsState);
-  const { t } = useTranslation();
+  const { t, i18n } = useTranslation();
 
   const { user } = useAuth();
   const { sendMessage, replyMessage } = useChatInteract();
@@ -89,6 +92,17 @@ export default function MessageComposer({
   const disabled = _disabled || !!attachments.find((a) => !a.uploaded);
 
   const { config } = useConfig();
+  const browserDictation =
+    !!config?.features.audio.enabled &&
+    config.features.audio.mode === 'browser';
+  const dictation = useBrowserSpeechRecognition({
+    enabled: browserDictation && !disabled,
+    language: i18n.language,
+    onTranscript: (text) => {
+      inputRef.current?.setValueExtern([value, text].filter(Boolean).join(' '));
+    },
+    onError: (error) => toast.error(t('chat.speech.failed', { error }))
+  });
   const showSettingsInComposer =
     config?.ui?.chat_settings_location !== 'sidebar' &&
     chatSettingsInputs.length > 0;
@@ -231,6 +245,7 @@ export default function MessageComposer({
       onSubmit(value, attachments, selectedCommand?.id);
     }
 
+    dictation.cancel();
     setAttachments([]);
     setValue(''); // Clear the value state
     inputRef.current?.reset();
@@ -242,7 +257,8 @@ export default function MessageComposer({
     selectedCommand,
     setAttachments,
     onSubmit,
-    onReply
+    onReply,
+    dictation.cancel
   ]);
 
   useEffect(() => {
@@ -282,7 +298,16 @@ export default function MessageComposer({
       />
       <div className="flex items-center justify-between">
         <div className="flex items-center -ml-1.5">
-          <VoiceButton disabled={disabled} />
+          {browserDictation ? (
+            <DictationButton
+              disabled={disabled}
+              supported={dictation.supported}
+              listening={dictation.listening}
+              onClick={dictation.toggle}
+            />
+          ) : (
+            <VoiceButton disabled={disabled} />
+          )}
           <UploadButton
             disabled={disabled}
             fileSpec={fileSpec}

frontend/src/hooks/useBrowserSpeechRecognition.ts

@@ -0,0 +1,118 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+
+export interface SpeechResultEvent {
+  resultIndex: number;
+  results: ArrayLike<{
+    isFinal: boolean;
+    [index: number]: { transcript: string };
+  }>;
+}
+
+export interface BrowserSpeechRecognition {
+  lang: string;
+  continuous: boolean;
+  interimResults: boolean;
+  onresult: ((event: SpeechResultEvent) => void) | null;
+  onerror: ((event: { error: string }) => void) | null;
+  onend: (() => void) | null;
+  start: () => void;
+  stop: () => void;
+  abort: () => void;
+}
+
+type SpeechWindow = Window & {
+  SpeechRecognition?: new () => BrowserSpeechRecognition;
+  webkitSpeechRecognition?: new () => BrowserSpeechRecognition;
+};
+
+export function useBrowserSpeechRecognition({
+  enabled,
+  language,
+  onTranscript,
+  onError
+}: {
+  enabled: boolean;
+  language: string;
+  onTranscript: (text: string) => void;
+  onError: (error: string) => void;
+}) {
+  const [listening, setListening] = useState(false);
+  const recognitionRef = useRef<BrowserSpeechRecognition | null>(null);
+  const callbacks = useRef({ onTranscript, onError });
+  callbacks.current = { onTranscript, onError };
+  const speechWindow = window as SpeechWindow;
+  const Recognition =
+    speechWindow.SpeechRecognition || speechWindow.webkitSpeechRecognition;
+
+  const dispose = useCallback(() => {
+    const recognition = recognitionRef.current;
+    recognitionRef.current = null;
+    if (recognition) {
+      recognition.onresult = null;
+      recognition.onerror = null;
+      recognition.onend = null;
+      recognition.abort();
+    }
+  }, []);
+
+  const cancel = useCallback(() => {
+    dispose();
+    setListening(false);
+  }, [dispose]);
+
+  useEffect(() => {
+    if (!enabled) cancel();
+    return dispose;
+  }, [enabled, cancel, dispose]);
+
+  const toggle = useCallback(() => {
+    if (!enabled || !Recognition) return;
+    if (recognitionRef.current) {
+      recognitionRef.current.stop();
+      return;
+    }
+
+    const recognition = new Recognition();
+    recognitionRef.current = recognition;
+    recognition.lang = language;
+    recognition.continuous = true;
+    recognition.interimResults = false;
+    const delivered = new Set<number>();
+    recognition.onresult = (event) => {
+      const transcripts: string[] = [];
+      for (let i = event.resultIndex; i < event.results.length; i++) {
+        const result = event.results[i];
+        if (result.isFinal && !delivered.has(i)) {
+          delivered.add(i);
+          transcripts.push(result[0].transcript.trim());
+        }
+      }
+      const transcript = transcripts.filter(Boolean).join(' ');
+      if (transcript) callbacks.current.onTranscript(transcript);
+    };
+    recognition.onend = () => {
+      recognitionRef.current = null;
+      recognition.onresult = null;
+      recognition.onerror = null;
+      recognition.onend = null;
+      setListening(false);
+    };
+    recognition.onerror = (event) => {
+      cancel();
+      if (event.error !== 'aborted' && event.error !== 'no-speech') {
+        callbacks.current.onError(event.error);
+      }
+    };
+    try {
+      setListening(true);
+      recognition.start();
+    } catch (error) {
+      cancel();
+      callbacks.current.onError(
+        error instanceof Error ? error.message : 'unknown'
+      );
+    }
+  }, [enabled, Recognition, language, cancel]);
+
+  return { supported: !!Recognition, listening, toggle, cancel };
+}