Whisper: start each clip_timestamps clip at its own start time

whisper/mlx_whisper/transcribe.py

@@ -283,6 +283,10 @@ def new_segment(
     ) as pbar:
         last_speech_timestamp = 0.0
         for seek_clip_start, seek_clip_end in seek_clips:
+            # Jump to the start of the clip. Without this the decoder keeps
+            # `seek` wherever the previous clip left it and transcribes the
+            # gap in between, so the requested clips are not honoured.
+            seek = max(seek, seek_clip_start)
             while seek < seek_clip_end:
                 time_offset = float(seek * HOP_LENGTH / SAMPLE_RATE)
                 window_end_time = float((seek + N_FRAMES) * HOP_LENGTH / SAMPLE_RATE)

whisper/test.py

@@ -197,6 +197,36 @@ def test_transcribe(self):
             ),
         )
 
+    def test_transcribe_clip_timestamps(self):
+        # Every clip must be decoded starting from its own start time. If `seek`
+        # is instead carried over from the previous clip, the decoder walks into
+        # the gap between them: with the clips below it decodes a window that
+        # starts at 1.0s, in the middle of the excluded 1s-3s stretch.
+        clips = [(0.0, 1.0), (3.0, 4.0)]
+        result = mlx_whisper.transcribe(
+            TEST_AUDIO,
+            path_or_hf_repo=MLX_FP32_MODEL_PATH,
+            fp16=False,
+            clip_timestamps="0,1,3,4",
+            # Decode every window, so that each one shows up in the output and
+            # the seek positions the loop visited are all observable.
+            no_speech_threshold=None,
+            condition_on_previous_text=False,
+            temperature=0.0,
+        )
+
+        visited = [s["seek"] / audio.FRAMES_PER_SECOND for s in result["segments"]]
+        for window_start in visited:
+            self.assertTrue(
+                any(start <= window_start < end for start, end in clips),
+                f"decoded a window starting at {window_start}s, outside {clips}",
+            )
+        for start, end in clips:
+            self.assertTrue(
+                any(start <= w < end for w in visited),
+                f"clip {start}-{end} was never decoded (visited {visited})",
+            )
+
     def test_transcribe_alice(self):
         audio_file = os.path.join(
             os.path.expanduser("~"),