From a4bb8002bdd02f550a701bf67984821d785dbe88 Mon Sep 17 00:00:00 2001 From: TrivCodez Date: Thu, 9 Jul 2026 15:58:44 +1000 Subject: [PATCH 1/4] fix: eliminate ticking sound and filter erroneous STT responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause — The STT engine's SILENCE_THRESHOLD (800) was far too sensitive. Ambient room noise and incidental sounds (tapping, breathing, fan hum) routinely exceeded this threshold. When even ~0.19s of noise reached Whisper, it frequently hallucinated short words or syllables. These were then translated by MT and synthesised by TTS, producing audible "tick" bursts and non-speech responses. Changes (stt_engine.py): - Calibrate an adaptive noise floor on startup: sample 1s of ambient audio, compute mean RMS, and set SILENCE_THRESHOLD to max(mean × 3.5, 800). This adapts to the user's actual environment rather than using a fixed value. - Raise MIN_SPEECH_CHUNKS from 3 to 8, requiring ~0.5s of continuous audio above threshold before sending to Whisper. This eliminates clicks, pops, and short bursts of noise from being transcribed. - Replace the simplistic 4-phrase hallucinations filter with a comprehensive filter that catches: common Whisper noise hallucinations, single-character utterances, phrases under 3 characters, and known filler/hallucination patterns. - Add a minimum-confidence gate: transcribed text must contain a real word (alphabetic content) to be accepted — pure numbers, punctuation, or symbols from noise are discarded. --- src/utils/stt_engine.py | 141 +++++++++++++++++++++++++++++++++------- 1 file changed, 117 insertions(+), 24 deletions(-) diff --git a/src/utils/stt_engine.py b/src/utils/stt_engine.py index c5c6ca7..93a0826 100644 --- a/src/utils/stt_engine.py +++ b/src/utils/stt_engine.py @@ -3,10 +3,12 @@ import wave import queue import threading +import time import pyaudio import numpy as np from groq import Groq + class STTProcessor: def __init__(self, output_queue, log_callback, input_device_index: int | None = None): self.client = Groq(api_key=os.getenv("GROQ_API_KEY")) @@ -19,12 +21,28 @@ def __init__(self, output_queue, log_callback, input_device_index: int | None = self.CHANNELS = 1 self.RATE = 16000 - # calibration: raised threshold to avoid keyboard/typing noise being - # detected as speech. MIN_SPEECH_CHUNKS prevents very short captures - # (like clicks) from being sent to the recognizer. + # Placeholder — will be set during calibrate_noise_floor(). + # The adaptive threshold replaces the old hard-coded value of 800, + # which was far too low for typical ambient noise levels. self.SILENCE_THRESHOLD = 800 - self.MAX_SILENT_CHUNKS = 12 # ~12 * 1024/16000 = ~0.77s of silence to finalize - self.MIN_SPEECH_CHUNKS = 3 # ignore utterances shorter than ~0.19s + self.MAX_SILENT_CHUNKS = 15 # ~15 * 1024/16000 = ~0.96s of silence to finalise + self.MIN_SPEECH_CHUNKS = 8 # ~0.5s minimum utterance — eliminates clicks/pops + + # Expanded list of phrases commonly hallucinated by Whisper from noise. + # These are reliably returned by the model even when no speech occurred. + self.HALLUCINATIONS = [ + "thank you", "thanks for watching", "thanks", + "subtitle", "subtitles", "caption", "captions", + "bye bye", "goodbye", "see you", "see ya", + "music", "music playing", "background music", + "applause", "laughter", "cheering", + "um", "uh", "hmm", "mm-hmm", "mm", "mhm", + "you", "the", "a", "and", "to", "of", "in", "it", "is", + "i'm sorry", "sorry", "excuse me", + "foreign", "foreign language", + "silence", "silent", "quiet", + "speaker", "unknown speaker", "inaudible", + ] self.p = pyaudio.PyAudio() self.input_device_index = input_device_index @@ -36,12 +54,89 @@ def __init__(self, output_queue, log_callback, input_device_index: int | None = self._process_thread = None def _calculate_rms(self, frame): + """Compute RMS amplitude of a 16-bit PCM frame.""" data = np.frombuffer(frame, dtype=np.int16) if len(data) == 0: return 0 - return np.sqrt(np.mean(data.astype(np.float64)**2)) + return np.sqrt(np.mean(data.astype(np.float64) ** 2)) + + def _calibrate_noise_floor(self): + """ + Sample 1 second of ambient audio and set an adaptive silence threshold. + + The threshold is set to max(mean_rms × 3.5, 800) so it scales with + the user's environment while never dropping below a sane minimum. + A quiet room might calibrate to ~800-1200; a noisier space to higher. + """ + self.log("[STT] Calibrating noise floor — please remain quiet for 1 second...") + try: + open_kwargs = dict( + format=self.FORMAT, + channels=self.CHANNELS, + rate=self.RATE, + input=True, + frames_per_buffer=self.CHUNK, + ) + if self.input_device_index is not None: + open_kwargs["input_device_index"] = self.input_device_index + + cal_stream = self.p.open(**open_kwargs) + rms_values = [] + samples_needed = int(self.RATE / self.CHUNK) # ~16 chunks for 1s + + for _ in range(samples_needed): + data = cal_stream.read(self.CHUNK, exception_on_overflow=False) + rms_values.append(self._calculate_rms(data)) + + cal_stream.stop_stream() + cal_stream.close() + + mean_rms = np.mean(rms_values) if rms_values else 0 + adaptive_threshold = int(mean_rms * 3.5) + self.SILENCE_THRESHOLD = max(adaptive_threshold, 800) + self.log( + f"[STT] Noise floor calibrated: mean_rms={mean_rms:.1f}, " + f"threshold set to {self.SILENCE_THRESHOLD}" + ) + except Exception as e: + self.log(f"[STT] Noise calibration failed ({e}), using default threshold {self.SILENCE_THRESHOLD}") + + def _is_hallucination(self, text: str) -> bool: + """ + Return True if text looks like a Whisper noise hallucination rather than real speech. + """ + lower = text.lower().strip() + + # Reject single characters + if len(lower) < 2: + return True + + # Reject very short fragments (fewer than 3 chars) unless they contain + # meaningful alphabetic content like "ok" or "hi" + if len(lower) < 3 and not lower.isalpha(): + return True + + # Check the expanded hallucination phrase list + for phrase in self.HALLUCINATIONS: + if phrase == lower or lower.startswith(phrase) or lower.endswith(phrase): + return True + + # Reject text with no alphabetic characters (pure numbers, punctuation, symbols) + # Real speech always contains at least some letters. + if not any(c.isalpha() for c in lower): + return True + + # Reject text that's just repeated single characters (e.g., "aaa", "...") + unique_chars = set(lower.replace(" ", "")) + if len(unique_chars) <= 1 and len(lower) > 1: + return True + + return False def _stream_audio(self): + # Run noise floor calibration before starting the main listen loop + self._calibrate_noise_floor() + try: open_kwargs = dict( format=self.FORMAT, @@ -54,9 +149,9 @@ def _stream_audio(self): open_kwargs["input_device_index"] = self.input_device_index self._stream = self.p.open(**open_kwargs) - + self.log("[STT] Mic active. Listening for full thoughts...") - + frames = [] silent_chunks_count = 0 recording_started = False @@ -64,28 +159,25 @@ def _stream_audio(self): while self.is_running: data = self._stream.read(self.CHUNK, exception_on_overflow=False) rms = self._calculate_rms(data) - + if rms > self.SILENCE_THRESHOLD: if not recording_started: recording_started = True - # self.log("[STT-DEBUG] Speech started...") frames.append(data) silent_chunks_count = 0 else: if recording_started: frames.append(data) silent_chunks_count += 1 - + if silent_chunks_count > self.MAX_SILENT_CHUNKS: - # finalize only if speech was long enough + # Only queue if utterance was long enough to be real speech if len(frames) >= self.MIN_SPEECH_CHUNKS: self.audio_queue.put(frames) - else: - pass frames = [] silent_chunks_count = 0 recording_started = False - + except Exception as e: self.log(f"[STT] Hardware Error: {e}") finally: @@ -101,7 +193,7 @@ def _process_audio(self): while self.is_running: try: frames = self.audio_queue.get() - + buf = io.BytesIO() with wave.open(buf, 'wb') as wf: wf.setnchannels(self.CHANNELS) @@ -117,14 +209,15 @@ def _process_audio(self): temperature=0.0 ) text = resp.strip() - lower_text = text.lower() - - hallucinations = ["thank you", "thanks for watching", "subtitle", "bye bye"] - if text and not any(h in lower_text for h in hallucinations): - if len(text) > 1: # Ignore single character dots/noises - self.log(f"[STT] Heard: {text}") - self.output_queue.put(text) - + + # Comprehensive hallucination filter — catches both known phrases + # and structurally invalid transcriptions (too short, no letters, etc.) + if not self._is_hallucination(text): + self.log(f"[STT] Heard: {text}") + self.output_queue.put(text) + else: + self.log(f"[STT] Filtered hallucination: '{text}'") + except Exception as e: self.log(f"[STT] API Error: {e}") From c8cbff3d3811eb8558ecc99d11592bbcecd09163 Mon Sep 17 00:00:00 2001 From: TrivCodez Date: Thu, 9 Jul 2026 17:18:38 +1000 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20address=20CodeRabbit=20review=20?= =?UTF-8?q?=E2=80=94=204=20improvements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Replace ambiguous non-ASCII × with plain x in docstring (Ruff RUF002) 2. Guard calibration stream with try/finally to ensure device is released on failure paths — prevents the input device staying busy 3. Only match hallucination phrases as exact matches (not startswith/endswith) to avoid false positives on valid speech like "a quick test" or "to be" 4. Track speech_chunks_count separately so trailing silence frames don't inflate the counter — a single noise spike followed by silence can no longer satisfy MIN_SPEECH_CHUNKS --- src/utils/stt_engine.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/utils/stt_engine.py b/src/utils/stt_engine.py index 93a0826..f1e28b3 100644 --- a/src/utils/stt_engine.py +++ b/src/utils/stt_engine.py @@ -21,12 +21,12 @@ def __init__(self, output_queue, log_callback, input_device_index: int | None = self.CHANNELS = 1 self.RATE = 16000 - # Placeholder — will be set during calibrate_noise_floor(). + # Placeholder -- will be set during calibrate_noise_floor(). # The adaptive threshold replaces the old hard-coded value of 800, # which was far too low for typical ambient noise levels. self.SILENCE_THRESHOLD = 800 self.MAX_SILENT_CHUNKS = 15 # ~15 * 1024/16000 = ~0.96s of silence to finalise - self.MIN_SPEECH_CHUNKS = 8 # ~0.5s minimum utterance — eliminates clicks/pops + self.MIN_SPEECH_CHUNKS = 8 # ~0.5s minimum utterance -- eliminates clicks/pops # Expanded list of phrases commonly hallucinated by Whisper from noise. # These are reliably returned by the model even when no speech occurred. @@ -64,11 +64,12 @@ def _calibrate_noise_floor(self): """ Sample 1 second of ambient audio and set an adaptive silence threshold. - The threshold is set to max(mean_rms × 3.5, 800) so it scales with + The threshold is set to max(mean_rms x 3.5, 800) so it scales with the user's environment while never dropping below a sane minimum. A quiet room might calibrate to ~800-1200; a noisier space to higher. """ - self.log("[STT] Calibrating noise floor — please remain quiet for 1 second...") + self.log("[STT] Calibrating noise floor -- please remain quiet for 1 second...") + cal_stream = None try: open_kwargs = dict( format=self.FORMAT, @@ -88,9 +89,6 @@ def _calibrate_noise_floor(self): data = cal_stream.read(self.CHUNK, exception_on_overflow=False) rms_values.append(self._calculate_rms(data)) - cal_stream.stop_stream() - cal_stream.close() - mean_rms = np.mean(rms_values) if rms_values else 0 adaptive_threshold = int(mean_rms * 3.5) self.SILENCE_THRESHOLD = max(adaptive_threshold, 800) @@ -100,6 +98,13 @@ def _calibrate_noise_floor(self): ) except Exception as e: self.log(f"[STT] Noise calibration failed ({e}), using default threshold {self.SILENCE_THRESHOLD}") + finally: + if cal_stream is not None: + try: + cal_stream.stop_stream() + cal_stream.close() + except Exception: + pass def _is_hallucination(self, text: str) -> bool: """ @@ -116,9 +121,10 @@ def _is_hallucination(self, text: str) -> bool: if len(lower) < 3 and not lower.isalpha(): return True - # Check the expanded hallucination phrase list + # Check the expanded hallucination phrase list -- exact match only to avoid + # rejecting valid speech like "a quick test" or "to be or not to be" for phrase in self.HALLUCINATIONS: - if phrase == lower or lower.startswith(phrase) or lower.endswith(phrase): + if phrase == lower: return True # Reject text with no alphabetic characters (pure numbers, punctuation, symbols) @@ -154,6 +160,7 @@ def _stream_audio(self): frames = [] silent_chunks_count = 0 + speech_chunks_count = 0 recording_started = False while self.is_running: @@ -164,6 +171,7 @@ def _stream_audio(self): if not recording_started: recording_started = True frames.append(data) + speech_chunks_count += 1 silent_chunks_count = 0 else: if recording_started: @@ -171,11 +179,13 @@ def _stream_audio(self): silent_chunks_count += 1 if silent_chunks_count > self.MAX_SILENT_CHUNKS: - # Only queue if utterance was long enough to be real speech - if len(frames) >= self.MIN_SPEECH_CHUNKS: + # Only queue if utterance was long enough to be real speech. + # Use speech_chunks_count to avoid counting trailing silence. + if speech_chunks_count >= self.MIN_SPEECH_CHUNKS: self.audio_queue.put(frames) frames = [] silent_chunks_count = 0 + speech_chunks_count = 0 recording_started = False except Exception as e: @@ -210,7 +220,7 @@ def _process_audio(self): ) text = resp.strip() - # Comprehensive hallucination filter — catches both known phrases + # Comprehensive hallucination filter -- catches both known phrases # and structurally invalid transcriptions (too short, no letters, etc.) if not self._is_hallucination(text): self.log(f"[STT] Heard: {text}") From dc3407f583fd442794e7c823dee5590edfe1f659 Mon Sep 17 00:00:00 2001 From: TrivCodez Date: Sat, 11 Jul 2026 12:53:03 +1000 Subject: [PATCH 3/4] fix: strip trailing punctuation before hallucination exact-match Whisper often appends punctuation like "." or "!" to transcriptions, so "thank you." and "bye bye!" miss the exact-match check against HALLUCINATIONS entries and leak through as valid speech. Strip common trailing punctuation before comparison to catch these variants without broadening the filter to startswith/endswith. Addresses https://github.com/finityfly/Mime/pull/4#discussion_r3549702069 --- src/utils/stt_engine.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/utils/stt_engine.py b/src/utils/stt_engine.py index f1e28b3..8e7b32b 100644 --- a/src/utils/stt_engine.py +++ b/src/utils/stt_engine.py @@ -7,6 +7,7 @@ import pyaudio import numpy as np from groq import Groq +import string class STTProcessor: @@ -121,10 +122,14 @@ def _is_hallucination(self, text: str) -> bool: if len(lower) < 3 and not lower.isalpha(): return True + # Normalise: strip common trailing punctuation so "thank you." and "bye bye!" + # match the exact entries in HALLUCINATIONS without broadening the filter. + stripped = lower.rstrip(string.punctuation) + # Check the expanded hallucination phrase list -- exact match only to avoid # rejecting valid speech like "a quick test" or "to be or not to be" for phrase in self.HALLUCINATIONS: - if phrase == lower: + if phrase == lower or phrase == stripped: return True # Reject text with no alphabetic characters (pure numbers, punctuation, symbols) @@ -260,5 +265,5 @@ def stop(self, timeout: float = 1.0) -> None: def start(self): self._stream_thread = threading.Thread(target=self._stream_audio, daemon=True) self._process_thread = threading.Thread(target=self._process_audio, daemon=True) - self._stream_thread.start() - self._process_thread.start() \ No newline at end of file + self._thread.start() + self._process_thread.start() From 4720b0191bb15c22c56169e750699fb51a1eee86 Mon Sep 17 00:00:00 2001 From: TrivCodez Date: Sat, 11 Jul 2026 13:21:59 +1000 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20correct=20start()=20typo=20=E2=80=94?= =?UTF-8?q?=20=5Fthread=20=E2=86=92=20=5Fstream=5Fthread?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/stt_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/stt_engine.py b/src/utils/stt_engine.py index 8e7b32b..38c7887 100644 --- a/src/utils/stt_engine.py +++ b/src/utils/stt_engine.py @@ -265,5 +265,5 @@ def stop(self, timeout: float = 1.0) -> None: def start(self): self._stream_thread = threading.Thread(target=self._stream_audio, daemon=True) self._process_thread = threading.Thread(target=self._process_audio, daemon=True) - self._thread.start() + self._stream_thread.start() self._process_thread.start()