fix(tts/luxtts): remove spurious mid-phrase pauses and chunk long text (#937)

Documentation/TTS/LuxTts.md

@@ -71,8 +71,41 @@ The ANE path is ~0.5 dB softer, not bit-identical, but the Parakeet
 round-trip transcript matches the input text exactly. The much smaller
 jetsam-visible footprint is why iOS uses it.
 
-All shapes are fixed: ≤ 255 tokens (+1 pad slot), ≤ 1024 mel frames
-total, ≤ 555 generated frames (~5.9 s per call; chunking is phase 2).
+All graph shapes are fixed: ≤ 255 tokens (+1 pad slot), ≤ 1024 mel frames
+total, ≤ 555 generated frames (~5.9 s) per flow-matching pass.
+
+Text that fits one pass (≤ 102 target tokens, token bucket, 1024 frames,
+555-frame vocoder bucket) is rendered in one pass; the 102 cap is measured,
+not a graph limit — see `LuxTtsConstants.continuationSpanTokens`. Longer text is handled by `LuxTtsManager`
+with continuation prompting rather than a bigger graph: target tokens are
+split into balanced spans of ≤ 102 tokens at word/punctuation boundaries,
+each span is generated with the previous span's audio and tokens as its
+prompt, the vocoder's onset/tail padding is trimmed around sustained
+speech (a span ending in punctuation keeps its tail), and spans are joined
+with a 30 ms crossfade. Spans are also bounded by the prompt's
+frames-per-token ratio so none generates more than ~5 s (468 frames),
+which keeps every span short enough to prompt the next one untruncated;
+a prompt whose ratio exceeds 12 frames per token (transcript covering only
+part of the clip) or would force spans under 8 tokens is rejected with
+`inputTooLong`. `speed` is applied to the first span only; later spans
+inherit the rate from their prompt. For multi-span output
+`generatedFrames`/`featuresLength` are sums over spans and `samples.count`
+is shorter than `(generatedFrames − 1) × 512`.
+
+The model also drops the occasional spurious mid-phrase pause (issue #937):
+where it lands depends on the exact length/noise draw — the PyTorch
+reference does it too, and a small `speed` change moves or removes it —
+so longer text simply has more chances to draw one. Every pass is scanned
+for gaps ≥ 80 ms between runs of sustained speech (≤ −45 dB relative to
+peak; natural stop closures stay ≤ 60 ms); when they outnumber the span's
+pause punctuation (`, . ; : ! ? -`, plus ellipses and quotes/brackets the
+G2P renders silently when going through the text API) the span is re-drawn
+with the next seed, up to `maxRedraws` times (default 3), keeping the
+cleanest pass. Re-draws keep the duration estimate: compressing it (speed
+× 1.03–1.06) removed some pauses but squeezed the final word into the last
+frames. This applies to single-pass text too, so a given seed is
+only bit-reproducible with `maxRedraws: 0` (CLI `--redraws 0`); the result
+reports `redraws` and `residualPauses`. Each re-draw is one extra pass.
 
 ## Quick Start
 
@@ -204,8 +237,9 @@ corpus-level gate is scored with `luxtts-g2p-dump` + `validate.py score`
 
 ## Remaining TODOs
 
-- Long-input chunking across multiple vocoder windows (> 555 generated
-  frames currently errors; mel truncation is not allowed).
+- Continuation spans are bounded by the *first* prompt's frames-per-token
+  ratio; a prompt whose pace differs a lot from the model's own pace can
+  still push a later span past the 555-frame vocoder bucket and error.
 - Optional VAD-based automatic prompt-silence trimming.
 - Non-English text (the G2P is `en-us` only; Mandarin pinyin tokens
   exist in `tokens.txt` but have no frontend).

Sources/FluidAudio/TTS/LuxTts/LuxTtsConstants.swift

@@ -45,6 +45,8 @@ public enum LuxTtsConstants {
     /// Prompt duration cap in seconds. Frames beyond this would eat too much
     /// of the 1024-frame bucket (~10.9 s total at 93.75 frames/s).
     public static let maxPromptSeconds: Double = 5.0
+    /// `maxPromptSeconds` in 24 kHz samples.
+    public static let maxPromptSamples = Int(maxPromptSeconds * Double(melSampleRate))
 
     /// Published fixed-shape vocoder buckets (generated frames).
     public static let vocoderBuckets = [282, 555]
@@ -54,4 +56,46 @@ public enum LuxTtsConstants {
 
     /// Default synthesis noise seed (matches the Python reference scripts).
     public static let defaultSeed: UInt64 = 42
+
+    /// Largest target-token span sent through one flow-matching pass; longer
+    /// text is continuation-prompted in balanced spans of at most this size.
+    /// Measured on the issue #937 text (106 tokens, six seeds): as one pass
+    /// it paused on 6 of 8 raw draws and the re-draw ladder still failed one
+    /// seed; as two ~53-token spans every seed was clean with two re-draws
+    /// in total. Shorter passes draw pauses far less often.
+    public static let continuationSpanTokens = 102
+    /// Smallest span worth rendering. A prompt whose frames-per-token ratio
+    /// forces spans below this (silence-heavy clip, transcript that does not
+    /// match) is rejected instead of turning into dozens of passes.
+    public static let minimumSpanTokens = 8
+    /// Highest plausible prompt frames-per-token ratio for continuation
+    /// synthesis. Natural speech sits around 4–6 (93.75 frames/s); a ratio
+    /// beyond this means the transcript covers only part of the clip (or
+    /// the clip is mostly silence), and every span would inherit the error.
+    public static let maxPromptFramesPerToken = 12.0
+
+    /// Re-seed attempts for a pass whose mid-speech silences outnumber the
+    /// span's pause punctuation. The model drops such pauses stochastically
+    /// (position depends on the exact length/noise draw; the PyTorch
+    /// reference does the same), so a fresh seed is the fix. One full pass
+    /// each.
+    public static let spuriousPauseRetries = 3
+    /// Silence floor (dB relative to the pass's peak) below which audio
+    /// counts as padding or pause, and the minimum gap between sustained
+    /// speech that counts as a pause. Natural stop closures stay ≤ 60 ms at
+    /// this floor; the reported pauses measure 100–160 ms.
+    public static let pauseFloorDb: Float = -45
+    /// Minimum gap between sustained-speech runs that counts as a pause.
+    public static let pauseMinimumSeconds = 0.08
+    /// Seed stride between continuation spans; leaves room for re-seeds.
+    public static let continuationSeedStride: UInt64 = 64
+
+    /// Generated-frame budget per continuation span. A span must fit inside
+    /// `maxPromptSeconds` so its untruncated audio can prompt the next span
+    /// with a transcript that still matches it.
+    public static let continuationSpanFrameBudget =
+        Int(maxPromptSeconds * Double(melSampleRate)) / hopLength
+
+    /// Overlap used when joining continuation-prompted spans.
+    public static let continuationCrossfadeSeconds = 0.03
 }

Sources/FluidAudio/TTS/LuxTts/LuxTtsContinuation.swift

@@ -0,0 +1,278 @@
+import Accelerate
+import Foundation
+
+/// Host-side long-utterance helpers for LuxTTS.
+enum LuxTtsContinuation {
+
+    private static let analysisWindowsPerSecond = 100
+    /// Activity shorter than this is a click or breath, not speech. Every
+    /// vocoder pass opens with a ~20 ms transient followed by 150–300 ms of
+    /// silence; anchoring on sustained speech keeps both trim and pause
+    /// detection from latching onto it.
+    private static let sustainedSpeechSeconds = 0.05
+    private static let onsetPrerollSeconds = 0.03
+    private static let tailPostrollSeconds = 0.03
+
+    /// Runs of sustained speech, in `windowSize`-sample analysis windows.
+    struct SpeechRuns {
+        let runs: [Range<Int>]
+        let windowSize: Int
+    }
+
+    /// Whether the whole text can go through one pass unchanged: at most
+    /// `continuationSpanTokens`, and within the token bucket, the 1024-frame
+    /// graph, and the largest vocoder bucket.
+    static func fitsSinglePass(
+        textTokenCount: Int, promptFrames: Int, promptTokenCount: Int, speed: Double
+    ) -> Bool {
+        guard textTokenCount <= LuxTtsConstants.continuationSpanTokens,
+            promptTokenCount + textTokenCount + 1 <= LuxTtsConstants.maxTokens
+        else { return false }
+        guard promptFrames > 0, promptTokenCount > 0, speed > 0 else { return true }
+        let featuresLength = LuxTtsSolver.featuresLength(
+            promptFrames: promptFrames,
+            promptTokenCount: promptTokenCount,
+            textTokenCount: textTokenCount,
+            speed: speed)
+        return featuresLength <= LuxTtsConstants.maxFrames
+            && featuresLength - promptFrames <= (LuxTtsConstants.vocoderBuckets.max() ?? 0)
+    }
+
+    /// Largest span (in target tokens) that fits the token bucket next to
+    /// this prompt's transcript and generates at most
+    /// `continuationSpanFrameBudget` frames for its frames-per-token ratio
+    /// at `speed`, capped at `continuationSpanTokens`.
+    static func maxSpanTokens(promptFrames: Int, promptTokenCount: Int, speed: Double) -> Int {
+        let cap = min(
+            LuxTtsConstants.continuationSpanTokens,
+            LuxTtsConstants.maxTokens - 1 - promptTokenCount)
+        guard promptFrames > 0, promptTokenCount > 0, speed > 0 else { return max(1, cap) }
+        let framesPerToken = Double(promptFrames) / Double(promptTokenCount)
+        let budgetTokens = Int(
+            (Double(LuxTtsConstants.continuationSpanFrameBudget) * speed / framesPerToken)
+                .rounded(.down))
+        return max(1, min(cap, budgetTokens))
+    }
+
+    /// Split a token sequence into balanced spans, preferring word/pause
+    /// boundaries nearest each ideal split point.
+    static func chunks(
+        tokenIds: [Int], maxTokens: Int, boundaryTokenIds: Set<Int>
+    ) -> [[Int]] {
+        precondition(maxTokens > 0, "maxTokens must be positive")
+        guard tokenIds.count > maxTokens else { return tokenIds.isEmpty ? [] : [tokenIds] }
+
+        let chunkCount = (tokenIds.count + maxTokens - 1) / maxTokens
+        var chunks: [[Int]] = []
+        chunks.reserveCapacity(chunkCount)
+
+        var start = 0
+        for chunkIndex in 0..<(chunkCount - 1) {
+            let remainingChunks = chunkCount - chunkIndex
+            let remainingTokens = tokenIds.count - start
+            let idealLength = Int(
+                (Double(remainingTokens) / Double(remainingChunks)).rounded())
+            let idealEnd = start + idealLength
+            let minimumEnd = max(
+                start + 1, tokenIds.count - (remainingChunks - 1) * maxTokens)
+            let maxEnd = min(start + maxTokens, tokenIds.count - (remainingChunks - 1))
+            let end = nearestBoundaryEnd(
+                in: tokenIds,
+                minimumEnd: minimumEnd,
+                idealEnd: idealEnd,
+                maxEnd: maxEnd,
+                searchRadius: max(1, maxTokens / 4),
+                boundaryTokenIds: boundaryTokenIds)
+
+            chunks.append(Array(tokenIds[start..<end]))
+            start = end
+        }
+        chunks.append(Array(tokenIds[start...]))
+        return chunks
+    }
+
+    /// Pauses the text itself calls for: pause punctuation inside the span,
+    /// ignoring trailing boundary tokens (their silence falls after speech).
+    static func expectedPauseCount(
+        in tokenIds: [Int], pauseTokenIds: Set<Int>, boundaryTokenIds: Set<Int>
+    ) -> Int {
+        var end = tokenIds.count
+        while end > 0, boundaryTokenIds.contains(tokenIds[end - 1]) { end -= 1 }
+        return tokenIds[..<end].reduce(0) { $0 + (pauseTokenIds.contains($1) ? 1 : 0) }
+    }
+
+    /// Pause boundaries the English G2P renders without a token: ellipses
+    /// and quotes/brackets (`LuxTtsG2p` treats them as clause breaks).
+    /// Apostrophes are left out — they are almost always contractions.
+    static func textPauseAllowance(_ text: String) -> Int {
+        let ellipses = text.components(separatedBy: "...").count - 1
+        let silentBreaks = text.reduce(0) { count, character in
+            count + ("…\"()[]«»".contains(character) ? 1 : 0)
+        }
+        return ellipses + silentBreaks
+    }
+
+    /// Whether the span's last spoken token is pause punctuation (trailing
+    /// spaces ignored).
+    static func endsWithPausePunctuation(
+        _ tokenIds: [Int], pauseTokenIds: Set<Int>, boundaryTokenIds: Set<Int>
+    ) -> Bool {
+        let spaceTokenIds = boundaryTokenIds.subtracting(pauseTokenIds)
+        guard let last = tokenIds.last(where: { !spaceTokenIds.contains($0) }) else { return false }
+        return pauseTokenIds.contains(last)
+    }
+
+    /// Runs of at least `sustainedSpeechSeconds` above `pauseFloorDb`
+    /// relative to the clip's peak.
+    static func speechRuns(_ samples: [Float], sampleRate: Int) -> SpeechRuns {
+        guard sampleRate > 0, !samples.isEmpty else { return SpeechRuns(runs: [], windowSize: 1) }
+        let windowSize = max(1, sampleRate / analysisWindowsPerSecond)
+        let windowCount = samples.count / windowSize
+        guard windowCount > 0 else { return SpeechRuns(runs: [], windowSize: windowSize) }
+
+        var peak: Float = 0
+        vDSP_maxmgv(samples, 1, &peak, vDSP_Length(samples.count))
+        guard peak > 0 else { return SpeechRuns(runs: [], windowSize: windowSize) }
+        let floorMeanSquare = peak * peak * powf(10, LuxTtsConstants.pauseFloorDb / 10)
+        let minimumWindows = max(
+            1, Int((sustainedSpeechSeconds * Double(sampleRate) / Double(windowSize)).rounded(.up)))
+
+        var runs: [Range<Int>] = []
+        var runStart: Int?
+        samples.withUnsafeBufferPointer { buffer in
+            guard let base = buffer.baseAddress else { return }
+            for window in 0...windowCount {
+                var active = false
+                if window < windowCount {
+                    var meanSquare: Float = 0
+                    vDSP_measqv(base + window * windowSize, 1, &meanSquare, vDSP_Length(windowSize))
+                    active = meanSquare > floorMeanSquare
+                }
+                if active {
+                    if runStart == nil { runStart = window }
+                } else if let start = runStart {
+                    if window - start >= minimumWindows { runs.append(start..<window) }
+                    runStart = nil
+                }
+            }
+        }
+        return SpeechRuns(runs: runs, windowSize: windowSize)
+    }
+
+    /// Gaps of at least `pauseMinimumSeconds` between consecutive runs of
+    /// sustained speech. Leading and trailing padding never count.
+    static func innerPauseCount(_ speech: SpeechRuns, sampleRate: Int) -> Int {
+        guard speech.runs.count > 1 else { return 0 }
+        let minimumWindows = max(
+            1,
+            Int(
+                (LuxTtsConstants.pauseMinimumSeconds * Double(sampleRate) / Double(speech.windowSize))
+                    .rounded(.up)))
+        return zip(speech.runs, speech.runs.dropFirst()).reduce(0) { count, pair in
+            count + (pair.1.lowerBound - pair.0.upperBound >= minimumWindows ? 1 : 0)
+        }
+    }
+
+    static func innerPauseCount(_ samples: [Float], sampleRate: Int) -> Int {
+        innerPauseCount(speechRuns(samples, sampleRate: sampleRate), sampleRate: sampleRate)
+    }
+
+    /// Sample range to keep from a span: the vocoder's onset padding is cut
+    /// (leaving a short preroll so unvoiced consonants are not clipped) and
+    /// the tail padding likewise (leaving a short postroll). Clips without
+    /// sustained speech are kept whole.
+    static func speechSlice(
+        _ speech: SpeechRuns, sampleCount: Int, sampleRate: Int, trimLeading: Bool, trimTrailing: Bool
+    ) -> Range<Int> {
+        guard let first = speech.runs.first, let last = speech.runs.last else { return 0..<sampleCount }
+        var start = 0
+        var end = sampleCount
+        if trimLeading {
+            let preroll = Int(onsetPrerollSeconds * Double(sampleRate))
+            start = max(0, first.lowerBound * speech.windowSize - preroll)
+        }
+        if trimTrailing {
+            let postroll = Int(tailPostrollSeconds * Double(sampleRate))
+            end = min(sampleCount, last.upperBound * speech.windowSize + postroll)
+        }
+        return start..<max(start, end)
+    }
+
+    static func trimmingLeadingPadding(_ samples: [Float], sampleRate: Int) -> [Float] {
+        let slice = speechSlice(
+            speechRuns(samples, sampleRate: sampleRate), sampleCount: samples.count,
+            sampleRate: sampleRate, trimLeading: true, trimTrailing: false)
+        return Array(samples[slice])
+    }
+
+    static func trimmingTrailingPadding(_ samples: [Float], sampleRate: Int) -> [Float] {
+        let slice = speechSlice(
+            speechRuns(samples, sampleRate: sampleRate), sampleCount: samples.count,
+            sampleRate: sampleRate, trimLeading: false, trimTrailing: true)
+        return Array(samples[slice])
+    }
+
+    /// Join two mono clips with a linear crossfade.
+    static func appendWithCrossfade(
+        _ next: ArraySlice<Float>, to output: inout [Float], crossfadeSamples: Int
+    ) {
+        guard !output.isEmpty else {
+            output = Array(next)
+            return
+        }
+        guard !next.isEmpty else { return }
+
+        let overlap = min(crossfadeSamples, output.count, next.count)
+        guard overlap > 0 else {
+            output.append(contentsOf: next)
+            return
+        }
+
+        let outputStart = output.count - overlap
+        let nextStart = next.startIndex
+        if overlap == 1 {
+            output[outputStart] = (output[outputStart] + next[nextStart]) * 0.5
+        } else {
+            for index in 0..<overlap {
+                let nextWeight = Float(index) / Float(overlap - 1)
+                output[outputStart + index] =
+                    output[outputStart + index] * (1 - nextWeight)
+                    + next[nextStart + index] * nextWeight
+            }
+        }
+        output.append(contentsOf: next.dropFirst(overlap))
+    }
+
+    static func appendWithCrossfade(
+        _ next: [Float], to output: inout [Float], crossfadeSamples: Int
+    ) {
+        appendWithCrossfade(next[...], to: &output, crossfadeSamples: crossfadeSamples)
+    }
+
+    private static func nearestBoundaryEnd(
+        in tokenIds: [Int],
+        minimumEnd: Int,
+        idealEnd: Int,
+        maxEnd: Int,
+        searchRadius: Int,
+        boundaryTokenIds: Set<Int>
+    ) -> Int {
+        var bestEnd: Int?
+        var bestDistance = Int.max
+        let searchStart = max(minimumEnd, idealEnd - searchRadius)
+        let searchEnd = min(maxEnd, idealEnd + searchRadius)
+        guard searchStart <= searchEnd else {
+            return min(max(idealEnd, minimumEnd), maxEnd)
+        }
+
+        for end in searchStart...searchEnd {
+            guard boundaryTokenIds.contains(tokenIds[end - 1]) else { continue }
+            let distance = abs(end - idealEnd)
+            if distance < bestDistance {
+                bestEnd = end
+                bestDistance = distance
+            }
+        }
+        return bestEnd ?? min(max(idealEnd, minimumEnd), maxEnd)
+    }
+}