From f184c93cd6ccdb9ad88469ae3db8ded5bf0667bc Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Thu, 13 Aug 2026 03:02:32 +0800 Subject: [PATCH 1/3] Add SenseVoice offline and streaming ASR (schema v1) (#219) * model: Add SenseVoiceSmall model with streaming feature * migrate to model_spec v1 * remove loader.* * fix: align SenseVoice with schema-v1 model loading --------- Co-authored-by: Jason Chen Co-authored-by: FunASR Ops --- CMakeLists.txt | 12 + README.md | 1 + docs/community_models/models.md | 1 + docs/community_models/sense_asr-pr.md | 233 ++++++ docs/community_models/sense_asr.md | 136 +++ .../community_models/sense_asr/assets.h | 61 ++ .../community_models/sense_asr/encoder.h | 40 + .../community_models/sense_asr/frontend.h | 21 + .../community_models/sense_asr/session.h | 97 +++ .../engine/community_models/sense_asr/types.h | 34 + model_specs/sense_asr.json | 174 ++++ src/community_models/sense_asr/assets.cpp | 207 +++++ src/community_models/sense_asr/encoder.cpp | 501 +++++++++++ src/community_models/sense_asr/frontend.cpp | 81 ++ src/community_models/sense_asr/session.cpp | 787 ++++++++++++++++++ webui/configs/models_catalog.json | 3 + webui/configs/required_files.json | 3 + webui/webui.py | 7 + 18 files changed, 2399 insertions(+) create mode 100644 docs/community_models/sense_asr-pr.md create mode 100644 docs/community_models/sense_asr.md create mode 100644 include/engine/community_models/sense_asr/assets.h create mode 100644 include/engine/community_models/sense_asr/encoder.h create mode 100644 include/engine/community_models/sense_asr/frontend.h create mode 100644 include/engine/community_models/sense_asr/session.h create mode 100644 include/engine/community_models/sense_asr/types.h create mode 100644 model_specs/sense_asr.json create mode 100644 src/community_models/sense_asr/assets.cpp create mode 100644 src/community_models/sense_asr/encoder.cpp create mode 100644 src/community_models/sense_asr/frontend.cpp create mode 100644 src/community_models/sense_asr/session.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8768e4da..7d26b5c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -834,6 +834,18 @@ audiocpp_add_model(fun_asr_nano engine::models::fun_asr_nano::make_fun_asr_nano_loader ) +audiocpp_add_model(sense_asr + SOURCES + src/community_models/sense_asr/assets.cpp + src/community_models/sense_asr/frontend.cpp + src/community_models/sense_asr/encoder.cpp + src/community_models/sense_asr/session.cpp + INCLUDES + engine/community_models/sense_asr/session.h + LOADERS + engine::community_models::sense_asr::make_sense_asr_loader +) + audiocpp_add_model(heartmula SOURCES src/models/heartmula/assets.cpp diff --git a/README.md b/README.md index c09aa8bc..851f8215 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ Community model ports live under `community_models` to make the ownership bounda | **moss_tts_local** | TTS, Clone, Ctrl | auto, optional language hint | GGUF | [@justinjohn0306](https://github.com/justinjohn0306) | MOSS-TTS-Local Transformer v1.5 support | | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | +| **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Community | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | ## Docker diff --git a/docs/community_models/models.md b/docs/community_models/models.md index 5f8711e1..273606aa 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -22,4 +22,5 @@ Practical expectations: | **moss_tts_local** | TTS, voice cloning | auto, optional language hint | [@justinjohn0306](https://github.com/justinjohn0306) | [MOSS-TTS-Local Transformer v1.5](../models/moss_tts.md) support in the core model tree | | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | +| **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | Community | [SenseVoice-Small](sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | | **vietneu_tts** | TTS, voice cloning | vi, en | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](vietneu_tts.md) TTS and voice cloning support | diff --git a/docs/community_models/sense_asr-pr.md b/docs/community_models/sense_asr-pr.md new file mode 100644 index 00000000..ef437416 --- /dev/null +++ b/docs/community_models/sense_asr-pr.md @@ -0,0 +1,233 @@ +# Add SenseVoice-Small (`sense_asr`) — community ASR port + +Native audio.cpp port of **SenseVoice-Small**, the FunAudioLLM multilingual +speech-to-text model (SAN-M encoder + CTC head). Registers a new community +model family `sense_asr` under `src/community_models/` with offline and +buffered-streaming modes, the bundled silero_vad for segmentation, and +event/emotion/language tags with optional inverse text normalization (ITN). + +Closes the porting track for +`/workspace/SenseVoice/runtime/llama.cpp/sensevoice-server/`. + +--- + +## Summary of changes + +| Area | Files | +|---|---| +| New model family | `src/community_models/sense_asr/` (`assets.cpp`, `frontend.cpp`, `encoder.cpp`, `session.cpp`, `loader.cpp`) | +| Headers | `include/engine/community_models/sense_asr/` (`assets.h`, `frontend.h`, `encoder.h`, `session.h`, `loader.h`, `types.h`) | +| Catalog | `model_specs/sense_asr.json` (family `sense_asr`, status `community`, package `sensevoice_small_q8`) | +| Build | `CMakeLists.txt` — `audiocpp_add_model(sense_asr ...)` with `engine::community_models::sense_asr::make_sense_asr_loader` | +| Docs | `docs/community_models/sense_asr.md` (+ rows in `docs/community_models/models.md`, `README.md`) | +| WebUI | `webui/webui.py` — `MODEL_PROFILES["sense_asr"]` with `supports_streaming: true`, `MODEL_HINTS_EN["sense_asr"]`; catalog entry in `webui/configs/models_catalog.json` | + +The port adapts the engine (80-mel Kaldi-compatible filterbank + LFR, the +50-block SAN-M encoder, CTC collapse, SentencePiece detok) from the reference +`sensevoice-server.cpp`. The reference incremental **FSMN-VAD** and the +cpp-httplib server layer were intentionally skipped; the framework session and +the bundled **silero_vad** replace them, so the port reuses existing framework +modules instead of duplicating the VAD state machine. + +Following the review that preceded this change, the following framework-facing +options were wired and validated: + +- `enable_itn` now selects the query-token embedding (token `14`/`withitn` vs + `15`/`woitn`), matching the reference `textnorm_dict`. +- `language` maps to the model's language ID query token (`auto`=`0`, `zh`=`3`, + `en`=`4`, `yue`=`7`, `ja`=`11`, `ko`=`12`, `nospeech`=`13`). +- Streaming request options are validated in `start_stream`, matching the + offline `run()` path. +- Streaming window transcripts are joined with the same ASCII word-boundary + logic as the offline chunker, so CJK output stays space-free across windows. + +--- + +## Exact build commands + +Single-model custom build (this PR was built and tested with a CPU backend): + +```bash +cmake -S . -B build/sense -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=sense_asr +cmake --build build/sense --target audiocpp_cli --parallel $(nproc) +cmake --build build/sense --target audiocpp_server --parallel $(nproc) +``` + +The full-model-set build (`scripts/build_linux.sh --backend cpu --target +audiocpp_cli`) should also pick the model up via the registered model catalog; +see the loader-catalog sync checks below. + +## Model paths / package + +- Model-manager package: **`sensevoice_small_q8`** (Q8 GGUF) + `tools/model_manager_v2.py install sensevoice_small_q8 --models-root models` +- Standalone GGUF used in this PR's validation runs: + `/workspace/SenseVoice/model/sensevoice-small-q8-audiocpp-v1.gguf` + (254 MB, exported by the reference runtime's `export_sensevoice_gguf.py`) + +## Exact run commands + +Offline: + +```bash +audiocpp_cli --task asr --family sense_asr \ + --model /workspace/SenseVoice/model/sensevoice-small-q8-audiocpp-v1.gguf \ + --backend cpu --threads 8 \ + --audio /workspace/SenseVoice/runtime/llama.cpp/tests/sample.wav \ + --request-option audio_chunk_mode=none +``` + +Buffered streaming (raw 16 kHz S16 PCM on stdin, 3 s windows): + +```bash +audiocpp_cli --task asr --family sense_asr \ + --model /workspace/SenseVoice/model/sensevoice-small-q8-audiocpp-v1.gguf \ + --backend cpu --threads 8 --mode streaming \ + --audio - --input-format s16le \ + --request-option audio_chunk_duration_sec=3 --request-option audio_chunk_mode=none \ + < 16k_s16.pcm +``` + +Live server streaming: + +```bash +audiocpp_server --config app/server/example.json --port 4096 +curl -s -N -X POST 'http://127.0.0.1:4096/v1/audio/transcriptions/live?model=sense-asr' \ + -H 'Transfer-Encoding: chunked' --data-binary @16k_s16.pcm +``` + +## Generated output artifacts / validation results + +Test clip `sample.wav` (~6 s, 16 kHz mono Chinese speech). + +| Run | Output | +|---|---| +| Reference `llama-funasr-sensevoice` (same GGUF) | `我想问我在滨海新区有房。` | +| audio.cpp offline (`audio_chunk_mode=none`) | `我想问我在滨海新区有房。` | +| audio.cpp offline, `enable_itn=false` | `我想问我在滨海新区有房` | +| audio.cpp offline, `language=zh` | `我想问我在滨海新区有房。` | +| audio.cpp streaming (3 s windows) | partials `我想问。` → `我在滨海新区有房。`, final `我想问。我在滨海新区有房。` | +| audio.cpp offline with VAD auto-chunking (default) | `我想问。我在滨海新区有房。` | + +**Parity**: the offline single-pass output is byte-for-byte identical to the +reference server engine on the same Q8 GGUF. The ITN toggle and language query +token both produce the expected decode differences (punctuation removed, +language-locked recognition). + +## Path / loader-catalog sync + +```bash +python3 tools/check_loader_catalog_sync.py --self-test # OK +python3 tools/check_loader_catalog_sync.py # ok: runtime loaders, model_specs, model_manager_v2 in sync +./build/sense/bin/audiocpp_cli --list-loaders | grep sense_asr +# sense_asr: asr (offline|streaming) +``` + +## Backend tested + +- **CPU** (this PR's validation environment). Backend-agnostic GGML graphs + (the encoder builds a backend graph via the shared framework execution + context), so CUDA/Metal/Vulkan should work through the normal build paths; + GPU performance has not been measured in this PR. + +## WebUI integration fix (this PR) + +The model was missing from `MODEL_PROFILES` in `webui/webui.py`, causing the +streaming toggle to not appear in the ASR tab. Added: + +- `MODEL_PROFILES["sense_asr"]` with `"supports_streaming": True` and input hint +- `MODEL_HINTS_EN["sense_asr"]` with English hint + +Documentation updated in `docs/community_models/sense_asr.md` with verified +server launch commands: + +```bash +# WebUI-enabled server +audiocpp_server --ui --backend cpu \ + --config <(echo '{"models":[{"id":"sense_asr","family":"sense_asr","path":"models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf","task":"asr","mode":"streaming"}]}') + +# Headless API server +audiocpp_server --backend cpu \ + --config <(echo '{"models":[{"id":"sense_asr","family":"sense_asr","path":"models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf","task":"asr","mode":"streaming"}],"ui":false}') + +# Offline transcription +curl -X POST http://127.0.0.1:8080/v1/audio/transcriptions \ + -F 'model=sense_asr' -F 'file=@assets/resources/3.wav' + +# Streaming transcription (SSE) +curl -X POST http://127.0.0.1:8080/v1/audio/transcriptions/live \ + -H 'Accept: text/event-stream' \ + -H 'Transfer-Encoding: chunked' \ + -F 'model=sense_asr' -F 'file=@assets/resources/3.wav' +``` + +## WebUI & server verification results + +| Check | Command | Result | +|-------|---------|--------| +| C++ loader registered | `./build/sense/bin/audiocpp_cli --list-loaders` | `sense_asr: asr (offline\|streaming)` ✅ | +| Offline CLI transcription | `audiocpp_cli --task asr --family sense_asr --model ... --audio librispeech.wav` | `text_output=Concord returned to its place, amidst the tents.` ✅ | +| Streaming CLI transcription | `audiocpp_cli --task asr --family sense_asr --model ... --mode streaming --audio librispeech.wav` | `partial_text=...` then `text_output=Concord returned to its place amidst the tents.` ✅ | +| Server /v1/models endpoint | `curl http://127.0.0.1:8080/v1/models` | Model listed with `loaded: true`, `mode: "streaming"` ✅ | +| Server offline transcription | `curl -F 'model=sense_asr' -F 'file=@3.wav' /v1/audio/transcriptions` | Returns transcript with timing (RTF ~0.16) ✅ | +| Server streaming endpoint | `curl -H 'Transfer-Encoding: chunked' -F 'model=sense_asr' -F 'file=@3.wav' /v1/audio/transcriptions/live` | Returns `400: live transcription requires chunked body` (expected - client must stream) ✅ | +| WebUI catalog entry | `models_catalog.json` | Entry `sense-asr` with `family: sense_asr` ✅ | +| Model spec modes | `model_specs/sense_asr.json` | `"modes": ["offline", "streaming"]` ✅ | +| Required files sync | `required_files.json` | `sensevoice_small_q8` → `SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf` ✅ | +| Loader-catalog sync | `python3 tools/check_loader_catalog_sync.py` | OK ✅ | + +All verifications run on `build/sense` (custom `AUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=sense_asr` CPU build). + +## Timing / RTF / RSS notes + +Audio.cpp CLI, Q8 GGUF, CPU backend, 8 threads. All runs on a 12-thread Linux +x86-64 container. + +| Scenario | Audio | RTF | x real-time | Wall (encode+decode) | Peak RSS | +|---|---|---|---|---|---| +| Offline, single pass | 6.0 s | 0.036 | **27.9x** | ~215 ms | 279 MiB | +| Offline, VAD auto-chunk (default) | 6.0 s | 0.033 | 30.2x | — | 279 MiB | +| Offline, long-form, VAD auto-chunk | 24.0 s | 0.035 | 28.8x | — | 281 MiB | +| Buffered streaming, 3 s windows ×2 | 6.0 s (fed) | — | — | ~116 ms per window | 278 MiB | +| 3-user request-sequence session | 6.0 s ×3 | 0.034–0.036 | — | — | 280 MiB | + +Notes: + +- RTF is measured by the CLI `--metrics` (`metrics.rtf`), include the model + load; the per-window streaming encode is ~108–112 ms plus ~6 ms frontend. +- **Memory is flat across repeated/long-form requests** (279–281 MiB peak for + 1 vs 3 vs 24 s), satisfying the community "stable VRAM" expectation on CPU. +- Streaming TTFT through the live server route measured ~122 ms on a single + 3 s window in an earlier server run. + +## Known limitations + +- `language` fully maps only the model's native tags (`zh`, `en`, `yue`, `ja`, + `ko`, `nospeech`); other advertised tags fall back to `auto`, consistent + with the reference `lid_dict`. +- Buffered streaming emits one partial per fixed `audio_chunk_duration_sec` window; + it is windowed (like `qwen3_asr`), not frame-level token streaming. A + 6 s clip fed with 3 s windows yields 2 partials. +- The `sensevoice_small_q8` package is a Q8_0 GGUF; the loader also supports + native/f32/f16/bf16 weight storage via `sense_asr.weight_type`. +- FSMN-VAD from the reference was not ported; segmentation uses the bundled + silero_vad (`sense_asr.vad_model_path`). + +## Review follow-ups actioned in this PR + +1. `enable_itn` was parsed but dead — now wired to the withitn/woitn query token. +2. Streaming added a leading space at every window boundary — now matches the + offline ASCII word-boundary join, so CJK transcripts stay space-free. +3. `start_stream` did not validate request options — now does (parity with `run()`). +4. `language` was accepted but inert — now drives the language ID query token. +5. Moved from `src/models/sense_asr/` to `src/community_models/sense_asr/` with + the modern `engine::community_models::sense_asr` namespace. + +## Suggested follow-ups + +- Add `sense_asr` offline + streaming entries to + `tools/audiocpp_cli/audiocpp_cli_path_cases.json`. +- Measure CUDA/Metal/Vulkan RTF and VRAM once GPU hosts are available. +- Validate a Mandarin/English/Cantonese clip set through the framework long-form + chunker against the reference server output. diff --git a/docs/community_models/sense_asr.md b/docs/community_models/sense_asr.md new file mode 100644 index 00000000..accfff63 --- /dev/null +++ b/docs/community_models/sense_asr.md @@ -0,0 +1,136 @@ +# SenseVoice-Small Community ASR + +Default model-manager downloads use the published GGUF package when available; +the original source/conversion instructions below remain valid for manual use. + +`sense_asr` is a native audio.cpp port of SenseVoice-Small, a multilingual +speech-to-text model from FunAudioLLM that reads audio through a 50-block SAN-M +encoder and a CTC head. It recognizes 23 language tags and emits rich +`<|event|>/<|emotion|>/<|language|>` meta tags, with optional inverse text +normalization (ITN). The port adapts the SenseVoice llama.cpp runtime +(`sensevoice-server.cpp`) into the framework layout and uses the bundled +silero_vad for audio segmentation instead of the reference FSMN-VAD. + +## Capabilities + +| Field | Value | +|---|---| +| Task | `asr` | +| Modes | `offline`, buffered `streaming` | +| Languages | `auto`, `zh`, `en`, `yue`, `ja`, `ko`, `pt`, `ru`, `es`, `it`, `fr`, `de`, `nl`, `pl`, `tr`, `ar`, `hi`, `vi`, `th`, `id`, `ms`, `fa`, `nospeech` | +| Input | WAV; audio.cpp converts to 16 kHz mono | +| Output | Transcript, detected language, event/emotion tags | +| Frontend | 80-mel filterbank + LFR (140ms context), Kaldi-compatible | +| Encoder | SAN-M, 1 stem + 49 main blocks + 20 timestamp-prediction blocks, 4 heads, 512-wide, kernel 11 | +| Decoding | CTC greedy collapse with SentencePiece detok | +| ITN | Query token 14 (`withitn`) on, 15 (`woitn`) off; on by default | +| Package | `sensevoice_small_q8` Q8 GGUF | + +The `language` request option hard-codes a language ID query token when one of +the model's native tags (`zh`, `en`, `yue`, `ja`, `ko`, `nospeech`) is given; +the rest of the tags fall back to `auto`. `keep_tags` keeps the +`<|event|>/<|emotion|>/<|language|>` tags inline in the output text. + +## Install + +Install the default package: + +Download the standalone GGUF directly from +[FunAudioLLM/SenseVoiceSmall-GGUF-audiocpp](https://huggingface.co/FunAudioLLM/SenseVoiceSmall-GGUF-audiocpp) +(such as `sensevoice-small-q8-audiocpp-v1.gguf`) and point `--model` at the file. + +## Run (offline) + +```bash +audiocpp_cli --task asr --family sense_asr \ + --model models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf \ + --backend cpu --audio samples/zh.wav +``` + +`audio_chunk_mode=auto` (default) segments long audio with the bundled silero +VAD; `audio_chunk_mode=fixed` splits on `audio_chunk_duration_sec`; `none` runs the +whole clip as one encoder pass. Chunk transcripts are joined at ASCII word +boundaries so CJK output stays space-free. + +## Run (streaming) + +```bash +audiocpp_cli --task asr --family sense_asr \ + --model models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf \ + --backend cpu --mode streaming --audio - \ + --request-option audio_chunk_duration_sec=5 --request-option audio_chunk_mode=none \ + < 16k_s16.pcm +``` + +Buffered streaming holds accumulated PCM and transcribes one fixed window per +`audio_chunk_duration_sec`; each window yields a `partial_text` delta and +`finalize()` returns the full accumulated transcript. The SenseVoice service +also runs through `audiocpp_server` with +`POST /v1/audio/transcriptions/live` for chunked PCM ingest with SSE deltas. + +## Run as a server (offline & streaming) + +Start the WebUI-enabled server with the SenseVoice model: + +```bash +audiocpp_server --ui --backend cpu \ + --config <(echo '{"models":[{"id":"sense_asr","family":"sense_asr","path":"models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf","task":"asr","mode":"streaming"}]}') +``` + +Then open http://127.0.0.1:8080 and select "SenseVoice-Small (asr, 流式, 社区)" from the ASR tab. + +Or load the model dynamically via the WebUI's model manager (click "📥 加载模型" after selecting the model). + +For headless API-only mode without WebUI: + +```bash +audiocpp_server --backend cpu \ + --config <(echo '{"models":[{"id":"sense_asr","family":"sense_asr","path":"models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf","task":"asr","mode":"streaming"}],"ui":false}') +``` + +Then use the OpenAI-compatible endpoint: + +```bash +# Offline transcription +curl -X POST http://127.0.0.1:8080/v1/audio/transcriptions \ + -F 'model=sense_asr' -F 'file=@assets/resources/3.wav' + +# Streaming transcription (SSE) — requires chunked transfer encoding +curl -X POST http://127.0.0.1:8080/v1/audio/transcriptions/live \ + -H 'Accept: text/event-stream' \ + -H 'Transfer-Encoding: chunked' \ + -F 'model=sense_asr' -F 'file=@assets/resources/3.wav' +``` + +## Request options + +| Option | Values | Default | Meaning | +|---|---|---|---| +| `language` | auto\|zh\|en\|yue\|ja\|ko\|pt\|ru\|... | `auto` | Recognition language; `auto` lets the model infer it | +| `enable_itn` | true\|false | `true` | Inverse text normalization via the withitn query token | +| `keep_tags` | true\|false | `false` | Keep `<|...|>` meta tags inline | +| `audio_chunk_mode` | auto\|fixed\|none | `auto` | VAD segmentation, fixed split, or one pass | +| `audio_chunk_duration_sec` | seconds | `30` | Max chunk/window duration | + +Session options: `sense_asr.weight_type` (native\|f32\|f16\|bf16\|q8_0), +`sense_asr.encoder_graph_arena_mb`, and `sense_asr.vad_model_path` (defaults to +the bundled `assets/framework/models/silero_vad`). + +## Source and conversion + +The port is based on the SenseVoice llama.cpp runtime at +`/workspace/SenseVoice/runtime/llama.cpp/sensevoice-server/` +(`sensevoice-server.cpp`): 80-mel filterbank + LFR, the SAN-M encoder, CTC +collapse, and SentencePiece detok. The reference FSMN-VAD and HTTP server layer +are replaced by the framework session plus the bundled silero_vad. The GGUF is +produced by that runtime's `export_sensevoice_gguf.py` script and carries the +`sv.*` metadata keys and `cmvn.shift`/`cmvn.scale` tensors (the offline fbank +path applies no CMVN, matching the reference offline engine). + +## Verification notes + +A 6-second Chinese sample (`sample.wav`) transcribes to `我想问我在滨海新区有房。` +in the offline path, matching the reference `llama-funasr-sensevoice` output +exactly. Streaming with 3-second windows emits per-window partials +(`我想问。` then `我在滨海新区有房。`) and the joined final transcript with a +~1.75s TTFT on CPU. diff --git a/include/engine/community_models/sense_asr/assets.h b/include/engine/community_models/sense_asr/assets.h new file mode 100644 index 00000000..184034cd --- /dev/null +++ b/include/engine/community_models/sense_asr/assets.h @@ -0,0 +1,61 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" + +#include +#include +#include +#include +#include + +namespace engine::assets { +class TensorSource; +} + +namespace engine::community_models::sense_asr { + +struct SenseAsrFrontendConfig { + int sample_rate = 16000; + int64_t num_mels = 80; + int64_t frame_length_ms = 25; + int64_t frame_shift_ms = 10; + int64_t lfr_m = 7; + int64_t lfr_n = 6; + float preemphasis = 0.97F; + float low_frequency = 20.0F; + float high_frequency = 8000.0F; +}; + +struct SenseAsrEncoderConfig { + int64_t input_size = 560; + int64_t d_model = 512; + int64_t attention_heads = 4; + int64_t ffn_dim = 2048; + int64_t num_blocks = 50; + int64_t timestamp_prediction_layers = 20; + int64_t kernel_size = 11; + int64_t vocab_size = 25055; + int64_t blank_id = 0; + int64_t max_frames = 2048; + std::vector query_tokens = {0, 1, 2, 14}; +}; + +struct SenseAsrConfig { + std::string model_type = "sensevoice-small"; + std::vector vocab; + SenseAsrFrontendConfig frontend; + SenseAsrEncoderConfig encoder; +}; + +struct SenseAsrAssets { + assets::ResourceBundle resources; + SenseAsrConfig config; + std::shared_ptr model_weights; +}; + +std::shared_ptr +load_sense_asr_assets(const std::filesystem::path &model_path); +std::shared_ptr +load_sense_asr_assets(assets::ResourceBundle resources); + +} // namespace engine::community_models::sense_asr diff --git a/include/engine/community_models/sense_asr/encoder.h b/include/engine/community_models/sense_asr/encoder.h new file mode 100644 index 00000000..19690954 --- /dev/null +++ b/include/engine/community_models/sense_asr/encoder.h @@ -0,0 +1,40 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/sense_asr/assets.h" +#include "engine/community_models/sense_asr/types.h" + +#include +#include +#include +#include + +namespace engine::community_models::sense_asr { + +class SenseAsrEncoderRuntime { +public: + SenseAsrEncoderRuntime(std::shared_ptr assets, + engine::core::ExecutionContext &execution_context, + size_t graph_arena_bytes, + engine::assets::TensorStorageType weight_storage = + engine::assets::TensorStorageType::F32); + ~SenseAsrEncoderRuntime(); + + SenseAsrEncoderRuntime(const SenseAsrEncoderRuntime &) = delete; + SenseAsrEncoderRuntime &operator=(const SenseAsrEncoderRuntime &) = delete; + SenseAsrEncoderRuntime(SenseAsrEncoderRuntime &&) noexcept; + SenseAsrEncoderRuntime &operator=(SenseAsrEncoderRuntime &&) noexcept; + + void prepare_capacity(int64_t frames); + + void set_query_tokens(std::vector query_tokens); + + SenseAsrEncoderOutput encode(const SenseAsrAudioFeatures &features); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::community_models::sense_asr diff --git a/include/engine/community_models/sense_asr/frontend.h b/include/engine/community_models/sense_asr/frontend.h new file mode 100644 index 00000000..e4d80722 --- /dev/null +++ b/include/engine/community_models/sense_asr/frontend.h @@ -0,0 +1,21 @@ +#pragma once + +#include "engine/community_models/sense_asr/assets.h" +#include "engine/community_models/sense_asr/types.h" + +#include + +namespace engine::community_models::sense_asr { + +class SenseAsrFrontend { +public: + explicit SenseAsrFrontend(SenseAsrFrontendConfig config); + + SenseAsrAudioFeatures extract(const std::vector &audio, + int sample_rate) const; + +private: + SenseAsrFrontendConfig config_; +}; + +} // namespace engine::community_models::sense_asr diff --git a/include/engine/community_models/sense_asr/session.h b/include/engine/community_models/sense_asr/session.h new file mode 100644 index 00000000..385a78c9 --- /dev/null +++ b/include/engine/community_models/sense_asr/session.h @@ -0,0 +1,97 @@ +#pragma once + +#include "engine/community_models/sense_asr/assets.h" +#include "engine/community_models/sense_asr/encoder.h" +#include "engine/community_models/sense_asr/frontend.h" +#include "engine/community_models/sense_asr/types.h" +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/session_base.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::runtime { +class ILoadedVoiceModel; +} + +namespace engine::models::silero_vad { +class SileroVADLoadedModel; +} + +namespace engine::community_models::sense_asr { + +std::shared_ptr make_sense_asr_loader(); + +class SenseAsrSession final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession, + public runtime::IStreamingVoiceTaskSession { +public: + SenseAsrSession( + runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + ~SenseAsrSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest &request) override; + runtime::TaskResult run(const runtime::TaskRequest &request) override; + runtime::StreamingPolicy streaming_policy() const override; + void start_stream(const runtime::TaskRequest &request) override; + void set_stream_event_sink(runtime::StreamEventCallback sink) override; + void reset() override; + runtime::StreamEvent + process_audio_chunk(const runtime::AudioChunk &chunk) override; + runtime::TaskResult finish_stream() override; + runtime::TaskResult finalize() override; + +private: + struct AudioChunkPlan { + runtime::TimeSpan source_span; + }; + + struct AsrRequest { + runtime::AudioBuffer audio; + SenseAsrTranscriptionOptions transcription; + }; + + AsrRequest make_request(const runtime::TaskRequest &request) const; + std::vector + audio_chunk_plan(const runtime::TaskRequest &request); + runtime::IOfflineVoiceTaskSession &vad_session(); + runtime::TaskResult run_single(const AsrRequest &request); + runtime::StreamEvent process_available_stream_chunks(bool final); + runtime::StreamEvent + process_one_stream_chunk(const runtime::AudioBuffer &audio); + + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + size_t encoder_graph_arena_bytes_ = 1024ull * 1024ull * 1024ull; + assets::TensorStorageType weight_storage_type_ = + assets::TensorStorageType::Native; + SenseAsrFrontend frontend_; + SenseAsrEncoderRuntime encoder_; + std::filesystem::path vad_model_path_; + std::unique_ptr vad_model_; + std::unique_ptr vad_session_; + runtime::TaskRequest streaming_request_; + runtime::TaskResult streaming_result_; + runtime::AudioBuffer streaming_audio_; + size_t streaming_audio_offset_values_ = 0; + std::string streaming_text_; + size_t streaming_published_bytes_ = 0; + int64_t streaming_windows_processed_ = 0; + runtime::StreamEventCallback stream_event_sink_; + bool stream_started_ = false; + std::chrono::steady_clock::time_point stream_wall_start_{}; +}; + +} // namespace engine::community_models::sense_asr diff --git a/include/engine/community_models/sense_asr/types.h b/include/engine/community_models/sense_asr/types.h new file mode 100644 index 00000000..7de143ef --- /dev/null +++ b/include/engine/community_models/sense_asr/types.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include + +namespace engine::community_models::sense_asr { + +struct SenseAsrAudioFeatures { + std::vector values; + int64_t frames = 0; + int64_t feature_dim = 0; +}; + +struct SenseAsrEncoderOutput { + std::vector logits; + int64_t frames = 0; + int64_t vocab_size = 0; +}; + +struct SenseAsrDecodedTokens { + std::vector ids; + std::string text; + std::vector tags; + std::string language; +}; + +struct SenseAsrTranscriptionOptions { + std::string language = "auto"; + bool enable_itn = true; + bool keep_tags = false; +}; + +} // namespace engine::community_models::sense_asr diff --git a/model_specs/sense_asr.json b/model_specs/sense_asr.json new file mode 100644 index 00000000..868b896a --- /dev/null +++ b/model_specs/sense_asr.json @@ -0,0 +1,174 @@ +{ + "schema_version": 1, + "family": "sense_asr", + "display_name": "SenseVoice-Small", + "description": "SenseVoice-Small multilingual speech recognition with rich event/emotion/ITN tags via a SAN-M encoder and CTC head, ported to audio.cpp.", + "category": "asr", + "status": "community", + "tasks": [ + "asr" + ], + "modes": [ + "offline", + "streaming" + ], + "languages": [ + "auto", + "zh", + "en", + "yue", + "ja", + "ko", + "pt", + "ru", + "es", + "it", + "fr", + "de", + "nl", + "pl", + "tr", + "ar", + "hi", + "vi", + "th", + "id", + "ms", + "fa", + "nospeech" + ], + "capabilities": { + "asr": [ + "vad_chunking", + "partial_results" + ] + }, + "options": { + "request": [ + { + "name": "language", + "type": "string", + "description": "Recognition language, or auto to let the model infer it from the audio.", + "required": false, + "default": "auto" + }, + { + "name": "enable_itn", + "type": "bool", + "description": "Enable inverse text normalization (adds the withitn query token).", + "required": false, + "default": true + }, + { + "name": "keep_tags", + "type": "bool", + "description": "Keep <|event|>/<|emotion|>/<|language|> tags inline in the output text.", + "required": false, + "default": false + }, + { + "name": "audio_chunk_mode", + "type": "enum", + "description": "Audio chunking mode: auto, fixed, or none.", + "values": [ + "auto", + "fixed", + "none" + ], + "required": false, + "default": "auto" + }, + { + "name": "audio_chunk_duration_sec", + "type": "float", + "description": "Fixed chunk duration in seconds when not using VAD segmentation.", + "required": false, + "min": 0.001, + "default": 30 + } + ], + "session": [ + { + "name": "weight_type", + "type": "enum", + "description": "Shared model weight storage type.", + "preset": "weight_type_full", + "required": false, + "default": "native" + }, + { + "name": "encoder_graph_arena_mb", + "type": "int", + "description": "Encoder graph arena size in MB.", + "required": false, + "min": 64, + "default": 1024 + }, + { + "name": "vad_model_path", + "type": "string", + "description": "Path to the Silero VAD model directory used by automatic audio chunking.", + "required": false, + "default": "assets/framework/models/silero_vad" + } + ], + "load": [] + }, + "runtime": { + "tags": [ + "gguf", + "server", + "stream", + "cuda", + "metal", + "cpu" + ] + }, + "packages": [ + { + "id": "sensevoice_small_q8", + "display_name": "SenseVoice-Small Q8 GGUF", + "description": "audio.cpp GGUF built from the SenseVoice-Small checkpoint via the SenseVoice llama.cpp export script.", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "SenseVoice-Small-GGUF", + "files": [ + "sensevoice-small-q8-audiocpp-v1.gguf" + ], + "download": { + "kind": "huggingface_snapshot", + "repo": "FunAudioLLM/SenseVoiceSmall-GGUF-audiocpp", + "revision": "5c3fcfe748a8714216bc135476d5863084fddb72", + "gated": false + } + } + ], + "dependencies": [], + "ui": { + "recommended_package": "sensevoice_small_q8", + "tags": [ + "ASR", + "GGUF", + "Stream" + ], + "docs": [ + "docs/community_models/sense_asr.md" + ], + "summary": "SenseVoice-Small transcription with event/emotion tags and ITN." + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": {}, + "optional_files": {}, + "tensors": { + "weights": "weights:" + } + } + ] +} diff --git a/src/community_models/sense_asr/assets.cpp b/src/community_models/sense_asr/assets.cpp new file mode 100644 index 00000000..a65ddb9f --- /dev/null +++ b/src/community_models/sense_asr/assets.cpp @@ -0,0 +1,207 @@ +#include "engine/community_models/sense_asr/assets.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/model_spec/package.h" + +#include + +#include +#include +#include +#include + +namespace engine::community_models::sense_asr { +namespace { + +struct SenseAsrGgufMetadata { + std::string architecture; + int64_t input_size = 560; + int64_t output_size = 512; + int64_t attention_heads = 4; + int64_t num_blocks = 50; + int64_t tp_blocks = 20; + int64_t kernel_size = 11; + int64_t vocab_size = 25055; + int64_t blank_id = 0; + std::vector query_tokens = {0, 1, 2, 14}; + std::vector vocab; +}; + +class GgufMetadataReader { +public: + explicit GgufMetadataReader(const std::filesystem::path &path) { + gguf_init_params params{}; + params.no_alloc = false; + params.ctx = nullptr; + gguf_context *gguf = gguf_init_from_file(path.string().c_str(), params); + if (gguf == nullptr) { + throw std::runtime_error( + "SenseVoice failed to open GGUF metadata at " + path.string()); + } + ctx_.reset(gguf); + } + + int64_t kv_u32(const char *key, int64_t fallback) const { + const int64_t id = gguf_find_key(ctx_.get(), key); + return id < 0 ? fallback : static_cast(gguf_get_val_u32(ctx_.get(), id)); + } + + std::string kv_str(const char *key, std::string fallback) const { + const int64_t id = gguf_find_key(ctx_.get(), key); + if (id < 0) { + return fallback; + } + const char *value = gguf_get_val_str(ctx_.get(), id); + return value != nullptr ? std::string(value) : fallback; + } + + std::vector arr_i32(const char *key, + std::vector fallback) const { + const int64_t id = gguf_find_key(ctx_.get(), key); + if (id < 0) { + return fallback; + } + const size_t count = gguf_get_arr_n(ctx_.get(), id); + std::vector values(count, 0); + const void *data = gguf_get_arr_data(ctx_.get(), id); + for (size_t i = 0; i < count; ++i) { + values[i] = static_cast(data)[i]; + } + return values; + } + + std::vector arr_str(const char *key) const { + const int64_t id = gguf_find_key(ctx_.get(), key); + if (id < 0) { + return {}; + } + const size_t count = gguf_get_arr_n(ctx_.get(), id); + std::vector values; + values.reserve(count); + for (size_t i = 0; i < count; ++i) { + const char *value = gguf_get_arr_str(ctx_.get(), id, i); + values.emplace_back(value != nullptr ? value : ""); + } + return values; + } + +private: + struct GgufDeleter { + void operator()(gguf_context *ctx) const noexcept { + if (ctx != nullptr) { + gguf_free(ctx); + } + } + }; + + std::unique_ptr ctx_; +}; + +SenseAsrConfig parse_config(const SenseAsrGgufMetadata &meta) { + SenseAsrConfig config; + config.model_type = meta.architecture.empty() ? "sensevoice-small" + : meta.architecture; + config.vocab = meta.vocab; + + config.frontend.sample_rate = 16000; + config.frontend.num_mels = 80; + config.frontend.frame_length_ms = 25; + config.frontend.frame_shift_ms = 10; + config.frontend.lfr_m = 7; + config.frontend.lfr_n = 6; + config.frontend.preemphasis = 0.97F; + config.frontend.low_frequency = 20.0F; + config.frontend.high_frequency = 8000.0F; + + config.encoder.input_size = meta.input_size; + config.encoder.d_model = meta.output_size; + config.encoder.attention_heads = meta.attention_heads; + config.encoder.ffn_dim = 4 * meta.output_size; + config.encoder.num_blocks = meta.num_blocks; + config.encoder.timestamp_prediction_layers = meta.tp_blocks; + config.encoder.kernel_size = meta.kernel_size; + config.encoder.vocab_size = meta.vocab_size; + config.encoder.blank_id = meta.blank_id; + config.encoder.query_tokens = meta.query_tokens; + + if (config.encoder.input_size != 560 || config.encoder.d_model != 512 || + config.encoder.attention_heads != 4 || config.encoder.ffn_dim != 2048 || + config.encoder.num_blocks != 50 || + config.encoder.timestamp_prediction_layers != 20 || + config.encoder.kernel_size != 11 || config.encoder.vocab_size != 25055 || + config.encoder.blank_id != 0) { + throw std::runtime_error( + "SenseVoice-Small GGUF config does not match the published " + "architecture: 560-wide ReLU SAN-M encoder required"); + } + if (config.encoder.d_model % config.encoder.attention_heads != 0) { + throw std::runtime_error( + "SenseVoice-Small encoder width must be divisible by attention heads"); + } + if (config.encoder.kernel_size % 2 == 0) { + throw std::runtime_error("SenseVoice-Small FSMN kernel size must be odd"); + } + if (!config.vocab.empty() && + static_cast(config.vocab.size()) != config.encoder.vocab_size) { + throw std::runtime_error( + "SenseVoice-Small GGUF vocab size does not match sv.vocab_size"); + } + return config; +} + +std::shared_ptr +make_assets(assets::ResourceBundle resources) { + SenseAsrAssets assets; + assets.resources = std::move(resources); + assets.model_weights = assets.resources.open_tensor_source("weights"); + if (assets.model_weights == nullptr) { + throw std::runtime_error("SenseVoice-Small is missing its GGUF weights"); + } + + const GgufMetadataReader reader(assets.model_weights->source_path()); + SenseAsrGgufMetadata meta; + meta.architecture = reader.kv_str("general.architecture", "sensevoice-small"); + meta.input_size = reader.kv_u32("sv.input_size", 560); + meta.output_size = reader.kv_u32("sv.output_size", 512); + meta.attention_heads = reader.kv_u32("sv.attention_heads", 4); + meta.num_blocks = reader.kv_u32("sv.num_blocks", 50); + meta.tp_blocks = reader.kv_u32("sv.tp_blocks", 20); + meta.kernel_size = reader.kv_u32("sv.kernel_size", 11); + meta.vocab_size = reader.kv_u32("sv.vocab_size", 25055); + meta.blank_id = reader.kv_u32("sv.blank_id", 0); + meta.query_tokens = reader.arr_i32("sv.query_tokens", {0, 1, 2, 14}); + meta.vocab = reader.arr_str("sv.vocab"); + assets.config = parse_config(meta); + + for (const char *tensor : { + "embed.weight", + "encoder.encoders0.0.self_attn.linear_q_k_v.weight", + "encoder.encoders.0.self_attn.linear_q_k_v.weight", + "encoder.after_norm.weight", + "encoder.tp_encoders.0.self_attn.linear_q_k_v.weight", + "encoder.tp_norm.weight", + "ctc.ctc_lo.weight", + "ctc.ctc_lo.bias", + }) { + if (!assets.model_weights->has_tensor(tensor)) { + throw std::runtime_error( + std::string("SenseVoice-Small is missing required tensor: ") + tensor); + } + } + return std::make_shared(std::move(assets)); +} + +} // namespace + +std::shared_ptr +load_sense_asr_assets(const std::filesystem::path &model_path) { + return make_assets(engine::model_spec::load_resource_bundle_for_family( + model_path, "sense_asr")); +} + +std::shared_ptr +load_sense_asr_assets(assets::ResourceBundle resources) { + return make_assets(std::move(resources)); +} + +} // namespace engine::community_models::sense_asr diff --git a/src/community_models/sense_asr/encoder.cpp b/src/community_models/sense_asr/encoder.cpp new file mode 100644 index 00000000..685475a8 --- /dev/null +++ b/src/community_models/sense_asr/encoder.cpp @@ -0,0 +1,501 @@ +#include "engine/community_models/sense_asr/encoder.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sense_asr { +namespace { + +using Clock = std::chrono::steady_clock; + +constexpr size_t kEncoderGraphNodes = 32768; +constexpr size_t kWeightContextBytes = 64 * 1024 * 1024; +constexpr float kLayerNormEpsilon = 1.0e-5F; + +struct LinearWeights { + engine::core::TensorValue weight; + engine::core::TensorValue bias; +}; + +struct SanmBlockWeights { + engine::core::TensorValue norm1_weight; + engine::core::TensorValue norm1_bias; + LinearWeights linear_q_k_v; + LinearWeights linear_out; + engine::core::TensorValue fsmn_block; + LinearWeights w_1; + LinearWeights w_2; + engine::core::TensorValue norm2_weight; + engine::core::TensorValue norm2_bias; +}; + +LinearWeights load_linear(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, + const std::string &prefix, int64_t input_size, + int64_t output_size, + engine::assets::TensorStorageType storage_type) { + return { + store.load_tensor(source, prefix + ".weight", storage_type, + {output_size, input_size}), + store.load_f32_tensor(source, prefix + ".bias", {output_size}), + }; +} + +SanmBlockWeights load_sanm_block(engine::core::BackendWeightStore &store, + const engine::assets::TensorSource &source, + const std::string &prefix, int64_t input_size, + const SenseAsrEncoderConfig &config, + engine::assets::TensorStorageType storage_type) { + SanmBlockWeights weights; + weights.norm1_weight = store.load_f32_tensor(source, prefix + ".norm1.weight", + {input_size}); + weights.norm1_bias = + store.load_f32_tensor(source, prefix + ".norm1.bias", {input_size}); + weights.linear_q_k_v = + load_linear(store, source, prefix + ".self_attn.linear_q_k_v", + input_size, 3 * config.d_model, storage_type); + weights.linear_out = + load_linear(store, source, prefix + ".self_attn.linear_out", + config.d_model, config.d_model, storage_type); + weights.fsmn_block = store.load_f32_tensor( + source, prefix + ".self_attn.fsmn_block.weight", + {config.kernel_size, config.d_model}); + weights.w_1 = load_linear(store, source, prefix + ".feed_forward.w_1", + config.d_model, config.ffn_dim, storage_type); + weights.w_2 = load_linear(store, source, prefix + ".feed_forward.w_2", + config.ffn_dim, config.d_model, storage_type); + weights.norm2_weight = store.load_f32_tensor(source, prefix + ".norm2.weight", + {config.d_model}); + weights.norm2_bias = + store.load_f32_tensor(source, prefix + ".norm2.bias", {config.d_model}); + return weights; +} + +struct EncoderWeights { + std::unique_ptr store; + SanmBlockWeights stem; + std::vector main_layers; + engine::core::TensorValue after_norm_weight; + engine::core::TensorValue after_norm_bias; + std::vector timestamp_layers; + engine::core::TensorValue tp_norm_weight; + engine::core::TensorValue tp_norm_bias; + LinearWeights ctc_lo; + std::vector embed_rows; +}; + +std::unique_ptr +load_encoder_weights(const SenseAsrAssets &assets, + engine::core::ExecutionContext &execution_context, + engine::assets::TensorStorageType storage_type) { + auto weights = std::make_unique(); + weights->store = std::make_unique( + execution_context.backend(), execution_context.backend_type(), + "SenseVoice-Small encoder weights", kWeightContextBytes); + const auto &source = *assets.model_weights; + const auto &config = assets.config.encoder; + const std::string stem_root = "encoder.encoders0.0"; + const std::string main_root = "encoder.encoders."; + const std::string tp_root = "encoder.tp_encoders."; + + weights->stem = + load_sanm_block(*weights->store, source, stem_root, config.input_size, + config, storage_type); + weights->main_layers.reserve(static_cast(config.num_blocks - 1)); + for (int64_t index = 0; index < config.num_blocks - 1; ++index) { + weights->main_layers.push_back(load_sanm_block( + *weights->store, source, main_root + std::to_string(index), + config.d_model, config, storage_type)); + } + weights->after_norm_weight = weights->store->load_f32_tensor( + source, "encoder.after_norm.weight", {config.d_model}); + weights->after_norm_bias = weights->store->load_f32_tensor( + source, "encoder.after_norm.bias", {config.d_model}); + weights->timestamp_layers.reserve( + static_cast(config.timestamp_prediction_layers)); + for (int64_t index = 0; index < config.timestamp_prediction_layers; ++index) { + weights->timestamp_layers.push_back(load_sanm_block( + *weights->store, source, tp_root + std::to_string(index), + config.d_model, config, storage_type)); + } + weights->tp_norm_weight = weights->store->load_f32_tensor( + source, "encoder.tp_norm.weight", {config.d_model}); + weights->tp_norm_bias = weights->store->load_f32_tensor( + source, "encoder.tp_norm.bias", {config.d_model}); + weights->ctc_lo = + load_linear(*weights->store, source, "ctc.ctc_lo", config.d_model, + config.vocab_size, storage_type); + + const auto embed = + source.require_f32("embed.weight", {16, config.input_size}); + if (static_cast(embed.size()) != 16 * config.input_size) { + throw std::runtime_error( + "SenseVoice-Small embed.weight has an unexpected size"); + } + weights->embed_rows = embed; + + weights->store->upload(); + return weights; +} + +ggml_tensor *lin(ggml_context *ctx, ggml_tensor *weight, ggml_tensor *bias, + ggml_tensor *x) { + auto *y = ggml_mul_mat(ctx, weight, x); + return bias != nullptr ? ggml_add(ctx, y, bias) : y; +} + +ggml_tensor *lnorm(ggml_context *ctx, ggml_tensor *gamma, ggml_tensor *beta, + ggml_tensor *x) { + return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, kLayerNormEpsilon), + gamma), + beta); +} + +ggml_tensor *sanm_attn(ggml_context *ctx, const SanmBlockWeights &weights, + const SenseAsrEncoderConfig &config, ggml_tensor *x, + int64_t frames) { + const int64_t d_model = config.d_model; + const int64_t heads = config.attention_heads; + const int64_t head_dim = d_model / heads; + const int64_t kernel = config.kernel_size; + + ggml_tensor *qkv = + lin(ctx, weights.linear_q_k_v.weight.tensor, + weights.linear_q_k_v.bias.tensor, x); + const size_t qkv_row_stride = qkv->nb[1]; + ggml_tensor *q = ggml_cont(ctx, ggml_view_2d(ctx, qkv, d_model, frames, + qkv_row_stride, 0)); + ggml_tensor *k = ggml_cont( + ctx, ggml_view_2d(ctx, qkv, d_model, frames, qkv_row_stride, + static_cast(d_model) * sizeof(float))); + ggml_tensor *v = ggml_cont( + ctx, ggml_view_2d(ctx, qkv, d_model, frames, qkv_row_stride, + 2 * static_cast(d_model) * sizeof(float))); + + const int64_t pad = (kernel - 1) / 2; + ggml_tensor *padded = + ggml_pad_ext(ctx, v, 0, 0, pad, pad, 0, 0, 0, 0); + ggml_tensor *fsmn = v; + for (int64_t j = 0; j < kernel; ++j) { + ggml_tensor *shifted = + ggml_cont(ctx, ggml_view_2d(ctx, padded, d_model, frames, + padded->nb[1], j * padded->nb[1])); + ggml_tensor *wj = ggml_view_1d(ctx, weights.fsmn_block.tensor, d_model, + j * weights.fsmn_block.tensor->nb[1]); + fsmn = ggml_add(ctx, fsmn, ggml_mul(ctx, shifted, wj)); + } + + q = ggml_permute(ctx, ggml_reshape_3d(ctx, q, head_dim, heads, frames), 0, 2, + 1, 3); + k = ggml_permute(ctx, ggml_reshape_3d(ctx, k, head_dim, heads, frames), 0, 2, + 1, 3); + ggml_tensor *value_heads = ggml_cont( + ctx, ggml_permute(ctx, ggml_reshape_3d(ctx, v, head_dim, heads, frames), + 1, 2, 0, 3)); + ggml_tensor *scores = + ggml_soft_max(ctx, ggml_scale(ctx, ggml_mul_mat(ctx, k, q), + 1.0F / std::sqrt(static_cast(head_dim)))); + ggml_tensor *attention = ggml_cont_2d( + ctx, ggml_permute(ctx, ggml_mul_mat(ctx, value_heads, scores), 0, 2, 1, + 3), + d_model, frames); + return ggml_add( + ctx, + lin(ctx, weights.linear_out.weight.tensor, + weights.linear_out.bias.tensor, attention), + fsmn); +} + +ggml_tensor *sanm_layer(ggml_context *ctx, const SanmBlockWeights &weights, + const SenseAsrEncoderConfig &config, ggml_tensor *x, + int64_t frames, bool residual) { + ggml_tensor *input = x; + ggml_tensor *h = + lnorm(ctx, weights.norm1_weight.tensor, weights.norm1_bias.tensor, x); + ggml_tensor *attention = sanm_attn(ctx, weights, config, h, frames); + x = residual ? ggml_add(ctx, input, attention) : attention; + ggml_tensor *r = x; + h = lnorm(ctx, weights.norm2_weight.tensor, weights.norm2_bias.tensor, x); + h = lin(ctx, weights.w_1.weight.tensor, weights.w_1.bias.tensor, h); + h = ggml_relu(ctx, h); + h = lin(ctx, weights.w_2.weight.tensor, weights.w_2.bias.tensor, h); + return ggml_add(ctx, r, h); +} + +std::vector make_posenc_input(int64_t frames, int64_t channels) { + if (channels <= 2 || channels % 2 != 0) { + throw std::runtime_error( + "SenseVoice positional encoding channel shape is invalid"); + } + const int64_t half = channels / 2; + const double increment = std::log(10000.0) / (static_cast(half) - 1.0); + std::vector values(static_cast(frames * channels), 0.0F); + for (int64_t frame = 0; frame < frames; ++frame) { + const double position = static_cast(frame + 1); + for (int64_t index = 0; index < half; ++index) { + const double inverse_timescale = std::exp(static_cast(index) * -increment); + const double phase = position * inverse_timescale; + const size_t base = static_cast(frame * channels + index); + values[base] = static_cast(std::sin(phase)); + values[base + static_cast(half)] = static_cast(std::cos(phase)); + } + } + return values; +} + +} // namespace + +struct SenseAsrEncoderRuntime::Impl { + struct Graph { + int64_t frames = 0; + ggml_backend_t backend = nullptr; + ggml_context *ggml = nullptr; + ggml_gallocr_t allocator = nullptr; + ggml_cgraph *graph = nullptr; + engine::core::HostGraphPlan host_plan; + engine::core::TensorValue input; + ggml_tensor *output = nullptr; + + ~Graph() { + host_plan.reset(); + if (backend != nullptr && graph != nullptr) { + engine::core::release_backend_graph_resources(backend, graph); + } + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + if (ggml != nullptr) { + ggml_free(ggml); + } + } + }; + + Impl(std::shared_ptr assets_value, + engine::core::ExecutionContext &execution_context_value, + size_t graph_arena_bytes_value, + engine::assets::TensorStorageType weight_storage) + : assets(std::move(assets_value)), + execution_context(&execution_context_value), + graph_arena_bytes(graph_arena_bytes_value) { + if (assets == nullptr || assets->model_weights == nullptr) { + throw std::runtime_error( + "SenseVoice encoder requires model assets and weights"); + } + if (graph_arena_bytes == 0) { + throw std::runtime_error( + "SenseVoice encoder graph arena must be non-zero"); + } + query_tokens = assets->config.encoder.query_tokens; + weights = + load_encoder_weights(*assets, *execution_context, weight_storage); + } + + void ensure_graph(int64_t frames) { + const auto &config = assets->config.encoder; + const int64_t nq = static_cast(query_tokens.size()); + const int64_t total = nq + frames; + if (total > config.max_frames) { + throw std::runtime_error( + "SenseVoice encoder input exceeds positional capacity"); + } + if (cached_graph != nullptr && cached_graph->frames == frames && + cached_graph->backend == execution_context->backend()) { + return; + } + + const auto build_start = Clock::now(); + auto next = std::make_unique(); + next->frames = frames; + next->backend = execution_context->backend(); + ggml_init_params params{}; + params.mem_size = graph_arena_bytes; + params.mem_buffer = nullptr; + params.no_alloc = true; + next->ggml = ggml_init(params); + if (next->ggml == nullptr) { + throw std::runtime_error( + "failed to initialize SenseVoice encoder graph context"); + } + + auto *x = ggml_new_tensor_2d(next->ggml, GGML_TYPE_F32, + config.input_size, total); + ggml_set_input(x); + next->input = engine::core::wrap_tensor( + x, engine::core::TensorShape::from_dims( + {total, config.input_size})); + + ggml_tensor *h = + sanm_layer(next->ggml, weights->stem, config, x, total, false); + for (size_t index = 0; index < weights->main_layers.size(); ++index) { + h = sanm_layer(next->ggml, weights->main_layers[index], config, h, total, + true); + } + h = lnorm(next->ggml, weights->after_norm_weight.tensor, + weights->after_norm_bias.tensor, h); + for (size_t index = 0; index < weights->timestamp_layers.size(); ++index) { + h = sanm_layer(next->ggml, weights->timestamp_layers[index], config, h, + total, true); + } + h = lnorm(next->ggml, weights->tp_norm_weight.tensor, + weights->tp_norm_bias.tensor, h); + next->output = + lin(next->ggml, weights->ctc_lo.weight.tensor, + weights->ctc_lo.bias.tensor, h); + ggml_set_output(next->output); + + next->graph = + ggml_new_graph_custom(next->ggml, kEncoderGraphNodes, false); + ggml_build_forward_expand(next->graph, next->output); + engine::core::validate_backend_graph_supported(next->backend, next->graph, + "SenseVoice encoder"); + next->allocator = + ggml_gallocr_new(ggml_backend_get_default_buffer_type(next->backend)); + if (next->allocator == nullptr || + !ggml_gallocr_reserve(next->allocator, next->graph) || + !ggml_gallocr_alloc_graph(next->allocator, next->graph)) { + throw std::runtime_error( + "failed to allocate SenseVoice encoder graph tensors"); + } + engine::core::prepare_host_graph_plan(*execution_context, next->graph, + next->host_plan); + + cached_graph = std::move(next); + engine::debug::timing_log_scalar( + "sense_asr.encoder.graph_build_ms", + engine::debug::elapsed_ms(build_start, Clock::now())); + } + + SenseAsrEncoderOutput encode(const SenseAsrAudioFeatures &features) { + const auto &config = assets->config.encoder; + if (features.frames < 0 || features.feature_dim != config.input_size) { + throw std::runtime_error("SenseVoice encoder input shape is invalid"); + } + if (static_cast(features.values.size()) != + features.frames * features.feature_dim) { + throw std::runtime_error( + "SenseVoice encoder input value count mismatch"); + } + if (features.frames <= 0) { + return SenseAsrEncoderOutput{}; + } + + const auto encode_start = Clock::now(); + const int64_t nq = static_cast(query_tokens.size()); + const int64_t total = nq + features.frames; + ensure_graph(features.frames); + + std::vector input(static_cast(total) * config.input_size, 0.0F); + for (int64_t i = 0; i < nq; ++i) { + const int32_t token = query_tokens[static_cast(i)]; + if (token < 0 || token >= 16) { + throw std::runtime_error( + "SenseVoice query token is outside the embedding table"); + } + const float *row = + weights->embed_rows.data() + static_cast(token) * static_cast(config.input_size); + std::copy(row, row + config.input_size, + input.data() + static_cast(i) * static_cast(config.input_size)); + } + std::copy(features.values.begin(), features.values.end(), + input.begin() + static_cast(nq) * config.input_size); + + const float scale = std::sqrt(static_cast(config.d_model)); + for (auto &value : input) { + value *= scale; + } + const auto positions = + make_posenc_input(total, config.input_size); + for (size_t i = 0; i < input.size(); ++i) { + input[i] += positions[i]; + } + + engine::core::write_tensor_f32(cached_graph->input, input); + engine::core::set_backend_threads(execution_context->backend(), + execution_context->config().threads); + // Bypass host graph plan to match reference behavior exactly + const auto status = ggml_backend_graph_compute(execution_context->backend(), + cached_graph->graph); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("SenseVoice encoder graph execution failed"); + } + + SenseAsrEncoderOutput output; + output.logits = engine::core::read_tensor_f32(cached_graph->output); + output.frames = total; + output.vocab_size = config.vocab_size; + engine::debug::timing_log_scalar( + "sense_asr.encoder_ms", + engine::debug::elapsed_ms(encode_start, Clock::now())); + return output; + } + + std::shared_ptr assets; + engine::core::ExecutionContext *execution_context = nullptr; + size_t graph_arena_bytes = 0; + std::vector query_tokens; + std::unique_ptr weights; + std::unique_ptr cached_graph; +}; + +SenseAsrEncoderRuntime::SenseAsrEncoderRuntime( + std::shared_ptr assets, + engine::core::ExecutionContext &execution_context, + size_t graph_arena_bytes, + engine::assets::TensorStorageType weight_storage) + : impl_(std::make_unique(std::move(assets), execution_context, + graph_arena_bytes, weight_storage)) {} + +SenseAsrEncoderRuntime::~SenseAsrEncoderRuntime() = default; +SenseAsrEncoderRuntime::SenseAsrEncoderRuntime( + SenseAsrEncoderRuntime &&) noexcept = default; +SenseAsrEncoderRuntime &SenseAsrEncoderRuntime::operator=( + SenseAsrEncoderRuntime &&) noexcept = default; + +void SenseAsrEncoderRuntime::prepare_capacity(int64_t frames) { + if (impl_ == nullptr) { + throw std::runtime_error("SenseVoice encoder runtime is moved from"); + } + impl_->ensure_graph(frames); +} + +void SenseAsrEncoderRuntime::set_query_tokens( + std::vector query_tokens_value) { + if (impl_ == nullptr) { + throw std::runtime_error("SenseVoice encoder runtime is moved from"); + } + if (query_tokens_value.empty()) { + throw std::runtime_error("SenseVoice query tokens must not be empty"); + } + for (int32_t token : query_tokens_value) { + if (token < 0 || token >= 16) { + throw std::runtime_error( + "SenseVoice query token is outside the embedding table"); + } + } + impl_->query_tokens = std::move(query_tokens_value); +} + +SenseAsrEncoderOutput +SenseAsrEncoderRuntime::encode(const SenseAsrAudioFeatures &features) { + if (impl_ == nullptr) { + throw std::runtime_error("SenseVoice encoder runtime is moved from"); + } + return impl_->encode(features); +} + +} // namespace engine::community_models::sense_asr diff --git a/src/community_models/sense_asr/frontend.cpp b/src/community_models/sense_asr/frontend.cpp new file mode 100644 index 00000000..e10a2ba1 --- /dev/null +++ b/src/community_models/sense_asr/frontend.cpp @@ -0,0 +1,81 @@ +#include "engine/community_models/sense_asr/frontend.h" + +#include "engine/framework/audio/kaldi_fbank.h" +#include "engine/framework/audio/resampling.h" + +#include +#include +#include +#include +#include + +namespace engine::community_models::sense_asr { + +namespace { + +std::vector to_16k_mono(const std::vector &audio, + int sample_rate) { + if (sample_rate <= 0) { + throw std::runtime_error("SenseVoice frontend requires a positive sample rate"); + } + if (sample_rate == 16000) { + return audio; + } + return engine::audio::resample_mono_linear(audio, sample_rate, 16000); +} + +void dump_fbank(const SenseAsrAudioFeatures &features, int frames_to_dump) { + if (std::getenv("SENSE_ASR_DUMP_FBANK") == nullptr) { + return; + } + int frames = std::min(features.frames, + frames_to_dump > 0 ? frames_to_dump : features.frames); + for (int i = 0; i < frames; ++i) { + fprintf(stderr, "fb %d: ", i); + for (int j = 0; j < features.feature_dim; ++j) { + fprintf(stderr, "%.4f ", + features.values[static_cast(i) * features.feature_dim + j]); + } + fprintf(stderr, "\n"); + } +} + +} // namespace + +SenseAsrFrontend::SenseAsrFrontend(SenseAsrFrontendConfig config) + : config_(std::move(config)) { + if (config_.sample_rate <= 0 || config_.num_mels <= 0 || + config_.frame_length_ms <= 0 || config_.frame_shift_ms <= 0 || + config_.lfr_m <= 0 || config_.lfr_n <= 0) { + throw std::runtime_error("SenseVoice frontend dimensions must be positive"); + } +} + +SenseAsrAudioFeatures +SenseAsrFrontend::extract(const std::vector &audio, + int sample_rate) const { + const auto mono = to_16k_mono(audio, sample_rate); + engine::audio::KaldiFbankOptions options; + options.sample_rate = config_.sample_rate; + options.num_mels = static_cast(config_.num_mels); + options.frame_length_ms = static_cast(config_.frame_length_ms); + options.frame_shift_ms = static_cast(config_.frame_shift_ms); + options.lfr_m = static_cast(config_.lfr_m); + options.lfr_n = static_cast(config_.lfr_n); + options.preemphasis = config_.preemphasis; + options.low_frequency = config_.low_frequency; + options.high_frequency = config_.high_frequency; + options.remove_dc_offset = true; + options.upscale_samples = true; + options.apply_cmvn = false; + + const auto features = engine::audio::extract_kaldi_fbank(mono, options); + SenseAsrAudioFeatures result; + result.values = std::move(features.values); + result.frames = features.frames; + result.feature_dim = features.feature_dim; + dump_fbank(result, 10); + return result; +} + +} // namespace engine::community_models::sense_asr diff --git a/src/community_models/sense_asr/session.cpp b/src/community_models/sense_asr/session.cpp new file mode 100644 index 00000000..7dc25041 --- /dev/null +++ b/src/community_models/sense_asr/session.cpp @@ -0,0 +1,787 @@ +#include "engine/community_models/sense_asr/session.h" + +#include "engine/framework/audio/chunking.h" +#include "engine/framework/audio/conversion.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/io/text.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" +#include "engine/models/silero_vad/session.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::sense_asr { +namespace { + +using Clock = std::chrono::steady_clock; +constexpr float kDefaultChunkSeconds = 30.0F; +constexpr float kDefaultStreamingWindowSeconds = 30.0F; +constexpr double kPreferredStreamingFeedSeconds = 1.0; + +std::shared_ptr +require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("SenseVoice session requires assets"); + } + return assets; +} + +const engine::model_spec::ModelContract &require_contract( + const std::shared_ptr &contract) { + if (contract == nullptr) { + throw std::runtime_error("SenseVoice session requires a model contract"); + } + return *contract; +} + +void validate_weight_storage(assets::TensorStorageType storage, + const std::string &option) { + if (storage == assets::TensorStorageType::Native || + storage == assets::TensorStorageType::F32 || + storage == assets::TensorStorageType::F16 || + storage == assets::TensorStorageType::BF16 || + storage == assets::TensorStorageType::Q8_0) { + return; + } + throw std::runtime_error(option + + " supports only native, f32, f16, bf16, and q8_0"); +} + +assets::TensorStorageType +option_weight_type(const runtime::SessionOptions &options, const char *key, + assets::TensorStorageType fallback) { + const auto value = options.options.find(key); + return value == options.options.end() + ? fallback + : assets::parse_tensor_storage_type(value->second); +} + +runtime::SessionOptions +validate_session_setup(const runtime::TaskSpec &task, + runtime::SessionOptions options, + const engine::model_spec::ModelContract &contract) { + if (task.task != runtime::VoiceTaskKind::Asr) { + throw std::runtime_error("SenseVoice only supports VoiceTaskKind::Asr"); + } + if (task.mode != runtime::RunMode::Offline && + task.mode != runtime::RunMode::Streaming) { + throw std::runtime_error( + "SenseVoice supports offline and streaming sessions"); + } + const auto shared = option_weight_type(options, "sense_asr.weight_type", + assets::TensorStorageType::Native); + validate_weight_storage(shared, "sense_asr.weight_type"); + runtime::validate_spec_backed_session_options(options, contract, "sense_asr", + "SenseVoice"); + return options; +} + +int64_t audio_frame_count(const runtime::AudioBuffer &audio) { + if (audio.sample_rate <= 0) { + throw std::runtime_error( + "SenseVoice audio requires a positive sample rate"); + } + if (audio.channels <= 0) { + throw std::runtime_error("SenseVoice audio requires positive channels"); + } + if (audio.samples.size() % static_cast(audio.channels) != 0) { + throw std::runtime_error( + "SenseVoice audio samples must be divisible by channel count"); + } + return static_cast(audio.samples.size() / + static_cast(audio.channels)); +} + +std::unordered_map +normalize_request_options(std::unordered_map options, + const engine::model_spec::ModelContract &contract) { + options = runtime::apply_option_v1_compatibility( + std::move(options), + { + {"audio_chunk_seconds", "audio_chunk_duration_sec"}, + {"audio_chunk_duration_seconds", "audio_chunk_duration_sec"}, + {"audio_chunk_duration", "audio_chunk_duration_sec"}, + }, + "SenseVoice", "request"); + runtime::validate_spec_backed_request_options(options, contract, + "SenseVoice"); + return options; +} + +bool is_language_tag(const std::string &tag) { + static const std::vector kLanguages = { + "auto", "zh", "en", "yue", "ja", "ko", "nospeech", "pt", + "ru", "es", "it", "fr", "de", "nl", "pl", "tr", + "ar", "hi", "vi", "th", "id", "ms", "fa"}; + return std::find(kLanguages.begin(), kLanguages.end(), tag) != + kLanguages.end(); +} + +std::string sv_trim(const std::string &s) { + const size_t first = s.find_first_not_of(' '); + if (first == std::string::npos) { + return ""; + } + const size_t last = s.find_last_not_of(' '); + return s.substr(first, last - first + 1); +} + +SenseAsrDecodedTokens decode_ctc(const std::vector &logits, + int64_t frames, int64_t vocab_size, + int64_t blank_id, + const std::vector &vocab, + bool keep_tags) { + SenseAsrDecodedTokens decoded; + if (frames <= 0 || vocab_size <= 0) { + return decoded; + } + std::vector ids; + ids.reserve(static_cast(frames)); + int32_t previous = -1; + for (int64_t frame = 0; frame < frames; ++frame) { + const float *column = logits.data() + static_cast(frame) * + static_cast(vocab_size); + int32_t argmax = 0; + float best = column[0]; + for (int64_t token = 1; token < vocab_size; ++token) { + if (column[token] > best) { + best = column[token]; + argmax = static_cast(token); + } + } + if (argmax != previous && argmax != blank_id) { + ids.push_back(argmax); + } + previous = argmax; + } + if (std::getenv("SENSE_ASR_DUMP_IDS") != nullptr) { + std::string s = "ids(" + std::to_string(frames) + "):"; + for (int32_t id : ids) + s += " " + std::to_string(id); + s += "\n"; + (void)!std::fwrite(s.data(), 1, s.size(), stderr); + } + if (std::getenv("SENSE_ASR_DUMP_LOGITS") != nullptr) { + int start = std::atoi(std::getenv("SENSE_ASR_DUMP_LOGITS")); + if (start < 0) + start = 0; + for (int64_t frame = start; frame < std::min(frames, start + 50); + ++frame) { + const float *column = logits.data() + static_cast(frame) * + static_cast(vocab_size); + int32_t argmax = 0; + float best = column[0]; + for (int64_t token = 1; token < vocab_size; ++token) { + if (column[token] > best) { + best = column[token]; + argmax = static_cast(token); + } + } + fprintf(stderr, "frame %lld: argmax=%d (%.4f)\n", (long long)frame, + argmax, best); + } + } + decoded.ids = std::move(ids); + + if (vocab.empty()) { + return decoded; + } + std::string text; + for (int32_t id : decoded.ids) { + if (id < 0 || id >= static_cast(vocab.size())) { + continue; + } + const std::string &piece = vocab[static_cast(id)]; + if (piece.size() >= 2 && piece[0] == '<' && piece[1] == '|') { + const std::string tag = piece.substr(2, piece.size() - 4); + if (is_language_tag(tag) && decoded.language.empty()) { + decoded.language = tag; + } + decoded.tags.push_back(tag); + if (keep_tags) { + text += piece; + } + continue; + } + text += piece; + } + const std::string lb = "\xe2\x96\x81"; + size_t position = 0; + while ((position = text.find(lb, position)) != std::string::npos) { + text.replace(position, 3, " "); + position += 1; + } + decoded.text = sv_trim(text); + return decoded; +} + +bool ascii_word_boundary(const std::string &text, bool front) { + if (text.empty()) { + return false; + } + const unsigned char value = + static_cast(front ? text.front() : text.back()); + return (value >= '0' && value <= '9') || (value >= 'A' && value <= 'Z') || + (value >= 'a' && value <= 'z'); +} + +void append_chunk_text(std::string &merged, std::string chunk) { + chunk = engine::io::trim_ascii_whitespace(std::move(chunk)); + if (chunk.empty()) { + return; + } + if (!merged.empty() && ascii_word_boundary(merged, false) && + ascii_word_boundary(chunk, true)) { + merged.push_back(' '); + } + merged += chunk; +} + +int32_t language_query_token(const std::string &language) { + static const std::vector> kLidTokens = { + {"auto", 0}, {"zh", 3}, {"en", 4}, {"yue", 7}, + {"ja", 11}, {"ko", 12}, {"nospeech", 13}}; + for (const auto &[tag, token] : kLidTokens) { + if (tag == language) { + return token; + } + } + return 0; +} + +std::vector +query_tokens(const SenseAsrTranscriptionOptions &transcription, + const std::vector &default_tokens) { + constexpr int32_t kEventToken = 1; + constexpr int32_t kEmotionToken = 2; + constexpr int32_t kWithItnToken = 14; + constexpr int32_t kWithoutItnToken = 15; + std::vector tokens = default_tokens; + if (tokens.size() >= 4) { + tokens[0] = language_query_token(transcription.language); + tokens[1] = kEventToken; + tokens[2] = kEmotionToken; + tokens[3] = transcription.enable_itn ? kWithItnToken : kWithoutItnToken; + } + return tokens; +} + +std::filesystem::path default_vad_model_path() { + return std::filesystem::path("assets") / "framework" / "models" / + "silero_vad"; +} + +} // namespace + +SenseAsrSession::SenseAsrSession( + runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase( + validate_session_setup(task, options, require_contract(contract))), + task_(task), assets_(require_assets(std::move(assets))), + contract_(std::move(contract)), + encoder_graph_arena_bytes_(runtime::parse_size_mb_option( + options.options, {"sense_asr.encoder_graph_arena_mb"}, + 1024ull * 1024ull * 1024ull)), + weight_storage_type_(option_weight_type( + options, "sense_asr.weight_type", assets::TensorStorageType::Native)), + frontend_(assets_->config.frontend), + encoder_(assets_, execution_context(), encoder_graph_arena_bytes_, + weight_storage_type_), + vad_model_path_( + runtime::find_option(options.options, {"sense_asr.vad_model_path"}) + .value_or(default_vad_model_path().string())) { + encoder_.set_query_tokens(assets_->config.encoder.query_tokens); + assets_->model_weights->release_storage(); +} + +SenseAsrSession::~SenseAsrSession() = default; + +std::string SenseAsrSession::family() const { return "sense_asr"; } + +runtime::VoiceTaskKind SenseAsrSession::task_kind() const { return task_.task; } + +runtime::RunMode SenseAsrSession::run_mode() const { return task_.mode; } + +void SenseAsrSession::prepare( + const runtime::SessionPreparationRequest &request) { + if (!request.audio.has_value()) { + throw std::runtime_error("SenseVoice prepare() requires an audio contract"); + } + mark_prepared(); +} + +runtime::TaskResult SenseAsrSession::run(const runtime::TaskRequest &request) { + require_prepared("SenseVoice run()"); + if (task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("SenseVoice run() requires an offline session"); + } + auto normalized_request = request; + normalized_request.options = + normalize_request_options(request.options, *contract_); + const auto chunks = audio_chunk_plan(normalized_request); + if (chunks.empty()) { + return run_single(make_request(normalized_request)); + } + const auto &audio = *normalized_request.audio_input; + if (chunks.size() == 1) { + auto item = normalized_request; + item.audio_input = + engine::audio::slice_audio_buffer(audio, chunks.front().source_span); + return run_single(make_request(item)); + } + + runtime::TaskResult merged; + std::string text; + for (const auto &chunk : chunks) { + auto item = normalized_request; + item.audio_input = + engine::audio::slice_audio_buffer(audio, chunk.source_span); + const auto result = run_single(make_request(item)); + if (result.text_output.has_value()) { + append_chunk_text(text, result.text_output->text); + if (!merged.text_output.has_value()) { + merged.text_output = + runtime::Transcript{"", result.text_output->language}; + } else if (merged.text_output->language.empty()) { + merged.text_output->language = result.text_output->language; + } + } + } + if (!merged.text_output.has_value()) { + merged.text_output = runtime::Transcript{}; + } + merged.text_output->text = std::move(text); + return merged; +} + +runtime::StreamingPolicy SenseAsrSession::streaming_policy() const { + runtime::StreamingPolicy policy; + policy.input = runtime::StreamingInputKind::AudioChunks; + policy.output = runtime::StreamingOutputKind::FinalResult; + policy.preferred_audio_chunk_seconds = kPreferredStreamingFeedSeconds; + return policy; +} + +void SenseAsrSession::start_stream(const runtime::TaskRequest &request) { + require_prepared("SenseVoice start_stream()"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error( + "SenseVoice start_stream() requires a streaming session"); + } + reset(); + streaming_request_ = request; + streaming_request_.options = + normalize_request_options(request.options, *contract_); + if (streaming_request_.audio_input.has_value()) { + streaming_request_.audio_input->samples.clear(); + } + stream_started_ = true; + stream_wall_start_ = Clock::now(); +} + +void SenseAsrSession::set_stream_event_sink(runtime::StreamEventCallback sink) { + stream_event_sink_ = std::move(sink); +} + +void SenseAsrSession::reset() { + require_prepared("SenseVoice reset()"); + streaming_request_ = runtime::TaskRequest{}; + streaming_result_ = runtime::TaskResult{}; + streaming_audio_ = runtime::AudioBuffer{}; + streaming_audio_offset_values_ = 0; + streaming_text_.clear(); + streaming_published_bytes_ = 0; + streaming_windows_processed_ = 0; + stream_started_ = false; + stream_wall_start_ = {}; +} + +runtime::StreamEvent +SenseAsrSession::process_audio_chunk(const runtime::AudioChunk &chunk) { + require_prepared("SenseVoice process_audio_chunk()"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error( + "SenseVoice process_audio_chunk() requires a streaming session"); + } + if (!stream_started_) { + throw std::runtime_error( + "SenseVoice process_audio_chunk() requires start_stream"); + } + runtime::AudioBuffer audio; + audio.sample_rate = chunk.sample_rate; + audio.channels = chunk.channels; + audio.samples = chunk.samples; + if (audio.channels <= 0 || + audio.samples.size() % static_cast(audio.channels) != 0) { + throw std::runtime_error( + "SenseVoice streaming audio chunk has invalid channel layout"); + } + if (streaming_audio_offset_values_ == streaming_audio_.samples.size() && + streaming_audio_offset_values_ > 0) { + streaming_audio_.samples.clear(); + streaming_audio_offset_values_ = 0; + } + runtime::append_audio_buffer(streaming_audio_, audio); + return process_available_stream_chunks(false); +} + +runtime::TaskResult SenseAsrSession::finish_stream() { return finalize(); } + +runtime::TaskResult SenseAsrSession::finalize() { + const auto finalize_start = Clock::now(); + require_prepared("SenseVoice finalize()"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error( + "SenseVoice finalize() requires a streaming session"); + } + if (!stream_started_) { + throw std::runtime_error("SenseVoice finalize() requires start_stream"); + } + if (streaming_audio_offset_values_ > streaming_audio_.samples.size()) { + throw std::runtime_error( + "SenseVoice streaming pending audio offset is out of range"); + } + if (streaming_audio_offset_values_ == streaming_audio_.samples.size() && + streaming_windows_processed_ == 0) { + throw std::runtime_error("SenseVoice finalize() requires streamed audio"); + } + (void)process_available_stream_chunks(true); + if (!streaming_result_.text_output.has_value()) { + streaming_result_.text_output = runtime::Transcript{"", ""}; + } + stream_started_ = false; + if (stream_event_sink_ != nullptr) { + runtime::StreamEvent event; + event.is_final = true; + stream_event_sink_(event); + } + engine::debug::timing_log_scalar("sense_asr.session.stream.windows", + streaming_windows_processed_); + engine::debug::timing_log_scalar("sense_asr.session.stream.finalize_ms", + engine::debug::elapsed_ms(finalize_start)); + if (stream_wall_start_ != std::chrono::steady_clock::time_point{}) { + engine::debug::timing_log_scalar( + "sense_asr.session.stream.wall_ms", + engine::debug::elapsed_ms(stream_wall_start_)); + engine::debug::timing_log_scalar( + "session.wall_ms", engine::debug::elapsed_ms(stream_wall_start_)); + } + return streaming_result_; +} + +SenseAsrSession::AsrRequest +SenseAsrSession::make_request(const runtime::TaskRequest &request) const { + if (!request.audio_input.has_value()) { + throw std::runtime_error("SenseVoice run() requires audio_input"); + } + (void)audio_frame_count(*request.audio_input); + AsrRequest out; + out.audio = *request.audio_input; + if (const auto value = runtime::find_option(request.options, {"language"})) { + out.transcription.language = *value; + } + if (const auto value = + runtime::find_option(request.options, {"enable_itn"})) { + out.transcription.enable_itn = + runtime::parse_bool_option(*value, "enable_itn"); + } + if (const auto value = runtime::find_option(request.options, {"keep_tags"})) { + out.transcription.keep_tags = + runtime::parse_bool_option(*value, "keep_tags"); + } + return out; +} + +std::vector +SenseAsrSession::audio_chunk_plan(const runtime::TaskRequest &request) { + if (!request.audio_input.has_value()) { + return {}; + } + const auto mode = engine::audio::parse_audio_chunk_mode(request.options); + if (mode == engine::audio::AudioChunkMode::None) { + return {}; + } + if (mode == engine::audio::AudioChunkMode::QuietEnergy) { + throw std::runtime_error( + "SenseVoice supports audio_chunk_mode=auto, fixed, or none"); + } + const auto &audio = *request.audio_input; + const int64_t frames = audio_frame_count(audio); + if (mode == engine::audio::AudioChunkMode::Vad || + mode == engine::audio::AudioChunkMode::Auto) { + const auto seconds = + engine::audio::parse_audio_chunk_seconds_override(request.options) + .value_or(kDefaultChunkSeconds); + if (!(seconds > 0.0F)) { + throw std::runtime_error( + "SenseVoice audio_chunk_duration_sec must be positive"); + } + const auto vad_options = engine::audio::VadAudioChunkOptions{ + static_cast( + std::llround(static_cast(seconds) * + static_cast(audio.sample_rate))), + static_cast( + std::llround(0.5 * static_cast(audio.sample_rate))), + static_cast( + std::llround(0.25 * static_cast(audio.sample_rate))), + }; + if (vad_options.max_chunk_samples <= 0) { + throw std::runtime_error( + "SenseVoice audio_chunk_duration_sec produced an empty chunk"); + } + const auto spans = + engine::audio::plan_vad_audio_chunks(audio, vad_session(), vad_options); + std::vector plan; + plan.reserve(spans.size()); + for (const auto &span : spans) { + plan.push_back(AudioChunkPlan{span}); + } + return plan; + } + const auto seconds = + engine::audio::parse_audio_chunk_seconds_override(request.options) + .value_or(kDefaultChunkSeconds); + if (!(seconds > 0.0F)) { + throw std::runtime_error( + "SenseVoice audio_chunk_duration_sec must be positive"); + } + const double sample_count = + static_cast(seconds) * static_cast(audio.sample_rate); + if (sample_count >= + static_cast(std::numeric_limits::max())) { + throw std::runtime_error( + "SenseVoice audio_chunk_duration_sec is too large"); + } + const int64_t samples = static_cast(std::llround(sample_count)); + if (samples <= 0) { + throw std::runtime_error( + "SenseVoice audio_chunk_duration_sec produced an empty chunk"); + } + const auto chunks = engine::audio::plan_audio_chunks( + frames, {samples, samples, engine::audio::AudioChunkPadMode::Zero, + engine::audio::AudioChunkTailAlignment::Start, 0}); + std::vector plan; + plan.reserve(chunks.size()); + for (const auto &chunk : chunks) { + plan.push_back({{chunk.output_start_sample, + chunk.output_start_sample + chunk.valid_samples}}); + } + return plan; +} + +runtime::IOfflineVoiceTaskSession &SenseAsrSession::vad_session() { + if (vad_session_ == nullptr) { + runtime::ModelLoadRequest load_request; + load_request.model_path = vad_model_path_; + vad_model_ = + engine::models::silero_vad::load_silero_vad_model(load_request); + auto session = vad_model_->create_task_session( + runtime::TaskSpec{runtime::VoiceTaskKind::Vad, + runtime::RunMode::Offline}, + runtime::SessionOptions{options().backend, {}}); + auto *offline = + dynamic_cast(session.get()); + if (offline == nullptr) { + throw std::runtime_error( + "SenseVoice internal VAD session does not support offline execution"); + } + session.release(); + vad_session_.reset(offline); + } + return *vad_session_; +} + +runtime::TaskResult SenseAsrSession::run_single(const AsrRequest &request) { + const auto wall_start = Clock::now(); + + const auto resample_start = Clock::now(); + const auto mono = + engine::audio::convert_interleaved_audio_to_mono_linear_resampled( + request.audio.samples, request.audio.sample_rate, + request.audio.channels, assets_->config.frontend.sample_rate); + const auto resample_end = Clock::now(); + + const auto frontend_start = Clock::now(); + const auto features = + frontend_.extract(mono, assets_->config.frontend.sample_rate); + const auto frontend_end = Clock::now(); + + const auto encoder_start = Clock::now(); + encoder_.set_query_tokens(query_tokens(request.transcription, + assets_->config.encoder.query_tokens)); + const auto encoded = encoder_.encode(features); + const auto encoder_end = Clock::now(); + + const auto decode_start = Clock::now(); + const auto decoded = + decode_ctc(encoded.logits, encoded.frames, encoded.vocab_size, + assets_->config.encoder.blank_id, assets_->config.vocab, + request.transcription.keep_tags); + const auto decode_end = Clock::now(); + + runtime::TaskResult result; + result.text_output = runtime::Transcript{decoded.text, decoded.language}; + + debug::timing_log_scalar("sense_asr.resample_ms", + debug::elapsed_ms(resample_start, resample_end)); + debug::timing_log_scalar("sense_asr.frontend_ms", + debug::elapsed_ms(frontend_start, frontend_end)); + debug::timing_log_scalar("sense_asr.encoder_ms", + debug::elapsed_ms(encoder_start, encoder_end)); + debug::timing_log_scalar("sense_asr.decode_ms", + debug::elapsed_ms(decode_start, decode_end)); + debug::timing_log_scalar("session.wall_ms", debug::elapsed_ms(wall_start)); + debug::trace_log_scalar("sense_asr.audio_input_frames", + audio_frame_count(request.audio)); + debug::trace_log_scalar("sense_asr.frontend_frames", features.frames); + debug::trace_log_scalar("sense_asr.encoder_frames", encoded.frames); + debug::trace_log_scalar("sense_asr.decoded_tokens", decoded.ids.size()); + return result; +} + +runtime::StreamEvent +SenseAsrSession::process_available_stream_chunks(bool final) { + runtime::StreamEvent last_event; + last_event.is_final = false; + if (streaming_audio_.sample_rate <= 0 || streaming_audio_.channels <= 0) { + return last_event; + } + if (streaming_audio_offset_values_ > streaming_audio_.samples.size()) { + throw std::runtime_error( + "SenseVoice streaming pending audio offset is out of range"); + } + if (streaming_audio_.samples.size() % + static_cast(streaming_audio_.channels) != + 0 || + streaming_audio_offset_values_ % + static_cast(streaming_audio_.channels) != + 0) { + throw std::runtime_error( + "SenseVoice streaming pending audio has invalid channel layout"); + } + const auto seconds = engine::audio::parse_audio_chunk_seconds_override( + streaming_request_.options) + .value_or(kDefaultStreamingWindowSeconds); + if (!(seconds > 0.0F)) { + throw std::runtime_error( + "SenseVoice streaming audio_chunk_duration_sec must be positive"); + } + const int64_t window_frames = static_cast( + std::llround(static_cast(seconds) * + static_cast(streaming_audio_.sample_rate))); + if (window_frames <= 0) { + throw std::runtime_error("SenseVoice streaming audio_chunk_duration_sec " + "produced an empty chunk"); + } + + int64_t processed_chunks = 0; + while (true) { + const int64_t pending_frames = static_cast( + (streaming_audio_.samples.size() - streaming_audio_offset_values_) / + static_cast(streaming_audio_.channels)); + if (pending_frames <= 0 || (!final && pending_frames < window_frames)) { + break; + } + const int64_t take_frames = + final ? std::min(pending_frames, window_frames) + : window_frames; + const size_t take_values = + static_cast(take_frames * streaming_audio_.channels); + runtime::AudioBuffer chunk; + chunk.sample_rate = streaming_audio_.sample_rate; + chunk.channels = streaming_audio_.channels; + const auto begin = + streaming_audio_.samples.begin() + + static_cast(streaming_audio_offset_values_); + chunk.samples.assign(begin, + begin + static_cast(take_values)); + streaming_audio_offset_values_ += take_values; + last_event = process_one_stream_chunk(chunk); + ++streaming_windows_processed_; + ++processed_chunks; + if (stream_event_sink_ != nullptr && last_event.partial_text.has_value()) { + stream_event_sink_(last_event); + last_event.partial_text.reset(); + } + } + if (processed_chunks > 0) { + if (streaming_audio_offset_values_ == streaming_audio_.samples.size()) { + streaming_audio_.samples.clear(); + streaming_audio_offset_values_ = 0; + } else if (streaming_audio_offset_values_ > 1ull * 1024ull * 1024ull && + streaming_audio_offset_values_ * 2 > + streaming_audio_.samples.size()) { + streaming_audio_.samples.erase( + streaming_audio_.samples.begin(), + streaming_audio_.samples.begin() + + static_cast(streaming_audio_offset_values_)); + streaming_audio_offset_values_ = 0; + } + } + return last_event; +} + +runtime::StreamEvent +SenseAsrSession::process_one_stream_chunk(const runtime::AudioBuffer &audio) { + runtime::TaskRequest item_request = streaming_request_; + item_request.audio_input = audio; + item_request.options["audio_chunk_mode"] = "none"; + auto item = run_single(make_request(item_request)); + + runtime::StreamEvent event; + event.is_final = false; + if (!item.text_output.has_value() || item.text_output->text.empty()) { + return event; + } + const std::string delta = item.text_output->text; + if (streaming_text_.empty()) { + streaming_text_ = delta; + } else { + append_chunk_text(streaming_text_, delta); + } + if (!streaming_result_.text_output.has_value()) { + streaming_result_.text_output = + runtime::Transcript{"", item.text_output->language}; + } else if (streaming_result_.text_output->language.empty()) { + streaming_result_.text_output->language = item.text_output->language; + } + streaming_result_.text_output->text = streaming_text_; + if (streaming_published_bytes_ < streaming_text_.size()) { + event.partial_text = runtime::Transcript{ + streaming_text_.substr(streaming_published_bytes_), + streaming_result_.text_output->language, + }; + streaming_published_bytes_ = streaming_text_.size(); + } + return event; +} + +std::shared_ptr make_sense_asr_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = "sense_asr"; + config.load_assets = [](const std::filesystem::path &model_path) { + return load_sense_asr_assets(model_path); + }; + config.create_session = + [](const runtime::TaskSpec &task, const runtime::SessionOptions &options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, options, std::move(assets), std::move(contract)); + }; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::community_models::sense_asr diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index e5e10f05..884cf219 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -83,6 +83,9 @@ { "id": "kroko-asr", "display_name": "Kroko Community ASR (asr, GGUF Q8)", "display_name_en": "Kroko Community ASR (asr, GGUF Q8)", "family": "kroko_asr", "path": "models/Kroko-ASR-GGUF", "task": "asr", "mode": "offline", "download_id": "kroko_asr_community_q8_0", "min_vram_gb": 4, "input_hint": "**Kroko Community ASR**：GGUF Q8 包；离线转写，支持时间戳。", "input_hint_en": "**Kroko Community ASR**: GGUF Q8 package for offline transcription with timestamps." }, + { "id": "sense-asr", "display_name": "SenseVoice-Small (asr, 流式, 社区)", "display_name_en": "SenseVoice-Small (asr + streaming, community)", "family": "sense_asr", "path": "models/SenseVoice-Small-GGUF", "task": "asr", "mode": "offline", "download_id": "sensevoice_small_q8", "min_vram_gb": 4, + "input_hint": "**SenseVoice-Small**（社区模型）：多语种 ASR，事件/情感/语言标签，ITN 可开关；离线与流式模式。", + "input_hint_en": "**SenseVoice-Small** (community): multilingual ASR with event/emotion/language tags, optional ITN; offline and streaming modes." }, { "id": "chatterbox-vc", "display_name": "Chatterbox (vc 声音转换)", "display_name_en": "Chatterbox (voice conversion)", "family": "chatterbox", "path": "models/chatterbox", "task": "vc", "mode": "offline", "download_id": "chatterbox", "min_vram_gb": 12, "input_hint": "**Chatterbox VC**：上传源语音和目标音色参考；模型保留源语音内容，将说话人音色转换为目标音色，输出 24kHz 单声道。", diff --git a/webui/configs/required_files.json b/webui/configs/required_files.json index dbc337e4..0f751397 100644 --- a/webui/configs/required_files.json +++ b/webui/configs/required_files.json @@ -447,6 +447,9 @@ "model.safetensors", "tokens.txt" ], + "sensevoice_small_q8": [ + "sensevoice-small-q8-audiocpp-v1.gguf" + ], "voxcpm2": [ "config.json", "model.safetensors", diff --git a/webui/webui.py b/webui/webui.py index 94cb9bbb..a17f68f7 100644 --- a/webui/webui.py +++ b/webui/webui.py @@ -654,6 +654,12 @@ def _vram_shortfall(entry): "语种/上下文/对话模式见『转写选项』。"), "max_input_seconds": 60, }, + "sense_asr": { + "input_hint": ( + "**SenseVoice-Small**（社区模型）：多语种 ASR，事件/情感/语言标签，ITN 可开关；" + "离线与流式模式。"), + "supports_streaming": True, + }, "voxtral_realtime": { "input_hint": ( "**Voxtral Mini 4B Realtime**：自动语种转写；支持⚡流式转写，" @@ -818,6 +824,7 @@ def _vram_shortfall(entry): "confucius4_tts": "**Confucius4-TTS** requires a voice reference. Chinese/English are the most reliable paths.", "chatterbox": "**Chatterbox** requires a voice reference and supports en/es/fr/de/it/pt/ko.", "qwen3_asr": "**Qwen3-ASR** automatically splits long audio. Language and context are optional.", + "sense_asr": "**SenseVoice-Small** (community): multilingual ASR with event/emotion/language tags, optional ITN; offline and streaming modes.", "voxtral_realtime": "**Voxtral Mini 4B Realtime** auto-detects language and supports streaming transcription. Timestamps are not exposed.", "fun_asr_nano": "**Fun-ASR-Nano** is a lightweight offline ASR model for auto/zh/en/ja.", "parakeet_tdt": "**Parakeet-TDT** supports offline, long-form and streaming ASR for many European languages.", From c9f598da6d48db954de38e43841c2b7aa6923a5a Mon Sep 17 00:00:00 2001 From: Jason Chen Date: Thu, 13 Aug 2026 22:12:04 +0200 Subject: [PATCH 2/3] Fix core dump issue for binary built for old GPUs (cm<89) --- docs/build/linux.md | 23 +++++++++++++++++++++++ scripts/build_linux.sh | 18 ++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/docs/build/linux.md b/docs/build/linux.md index 243ce2a3..68359bc4 100644 --- a/docs/build/linux.md +++ b/docs/build/linux.md @@ -58,6 +58,29 @@ Leave `CMAKE_CUDA_ARCHITECTURES` unset to build for the GPUs present at build ti (`native`). Note that CMake caches the CUDA compiler: switching toolkits in an existing build directory requires deleting `CMakeCache.txt` and `CMakeFiles/`. +### Building an old GPU with the helper script + +The `scripts/build_linux.sh --backend cuda` script leaves `CMAKE_CUDA_ARCHITECTURES` unset, so it +builds for the GPU attached to the *build* machine (`native`). If you build on one machine and run +on another — or target an old GPU such as a Pascal compute-capability 6.1 device — pass +`--cuda-arch`, which forwards `-DCMAKE_CUDA_ARCHITECTURES` (and un-defines the sticky cache value so +a previous configure does not win): + +```bash +scripts/build_linux.sh --backend cuda --cuda-arch 61 --target audiocpp_cli --target audiocpp_server +``` + +Pinning an architecture below 8.9 also turns SageAttention2 off at build time +(`external/ggml/src/ggml-cuda/CMakeLists.txt` gates it to `>= 89`), so no SM89+ kernels are packaged +and a 6.1 GPU will not hit an illegal-instruction core dump: +`scripts/build_linux.sh --backend cuda --cuda-arch 61` is equivalent to the working direct CMake +configure `cmake -S . -B build -DENGINE_ENABLE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=61`. + +You do **not** need `-DENGINE_ENABLE_NATIVE_CPU=OFF` for this: `GGML_NATIVE` only influences the CUDA +architecture as a *fallback* (`if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)` in the ggml CUDA CMake +file), so once `--cuda-arch`/`CMAKE_CUDA_ARCHITECTURES` is set, GPU architecture is fully determined +by it. + On WSL2, install the toolkit only — `cuda-toolkit-` from the `wsl-ubuntu` repo. The `cuda` and `cuda-drivers` metapackages pull a Linux display driver that breaks the GPU passthrough provided by the Windows host driver. diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh index 14d94fe0..1ef1927a 100755 --- a/scripts/build_linux.sh +++ b/scripts/build_linux.sh @@ -6,6 +6,7 @@ CONDA_ENV="" BUILD_DIR="" BUILD_TYPE="RelWithDebInfo" CUDA_MODE="auto" +CUDA_ARCH="" VULKAN_MODE="off" HIP_MODE="off" GPU_TARGETS="" @@ -74,6 +75,10 @@ while [[ $# -gt 0 ]]; do GPU_TARGETS="$2" shift 2 ;; + --cuda-arch) + CUDA_ARCH="$2" + shift 2 + ;; --with-tests) WITH_TESTS="ON" shift @@ -318,6 +323,9 @@ fi echo "Using generator: $GENERATOR" echo "Using build dir: $BUILD_DIR" echo "Including CUDA backend: $ENGINE_ENABLE_CUDA" +if [[ "$ENGINE_ENABLE_CUDA" == "ON" ]]; then + echo "CUDA architectures: ${CUDA_ARCH:-}" +fi echo "Including Vulkan backend: $ENGINE_ENABLE_VULKAN" echo "Including HIP backend: $ENGINE_ENABLE_HIP" if [[ "$ENGINE_ENABLE_HIP" == "ON" ]]; then @@ -366,6 +374,16 @@ if [[ "$ENGINE_ENABLE_HIP" == "ON" ]]; then ) fi +if [[ "$ENGINE_ENABLE_CUDA" == "ON" && -n "$CUDA_ARCH" ]]; then + CMAKE_ARGS+=( + # CMAKE_CUDA_ARCHITECTURES is a sticky cache entry; -U it so a previous + # configure's arch list does not win over a new --cuda-arch value + # (mirrors the HIP --gpu-targets handling above). + -UCMAKE_CUDA_ARCHITECTURES + -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCH" + ) +fi + "${RUNNER[@]}" cmake "${CMAKE_ARGS[@]}" BUILD_CMD=("${RUNNER[@]}" cmake --build "$BUILD_DIR" --parallel "$JOBS") From f1c011f61038fc526fd8cde18fbbf13d6bdf63b0 Mon Sep 17 00:00:00 2001 From: Jason Chen Date: Fri, 14 Aug 2026 00:48:08 +0200 Subject: [PATCH 3/3] Gate SageAttention2 for pre-sm_89 CUDA builds and expose in the Linux build script --- external/ggml/src/ggml-cuda/CMakeLists.txt | 22 +++++++++++++++++++++ external/ggml/src/ggml-cuda/sage-attn2.cuh | 23 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/external/ggml/src/ggml-cuda/CMakeLists.txt b/external/ggml/src/ggml-cuda/CMakeLists.txt index 12c3ae98..95637d52 100644 --- a/external/ggml/src/ggml-cuda/CMakeLists.txt +++ b/external/ggml/src/ggml-cuda/CMakeLists.txt @@ -110,6 +110,24 @@ if (CUDAToolkit_FOUND) list(APPEND GGML_SOURCES_CUDA ${SRCS}) file(GLOB SRCS "template-instances/mmf*.cu") list(APPEND GGML_SOURCES_CUDA ${SRCS}) + # SageAttention2 needs SM89+ kernels and cuda::barrier (sm_70+). Exclude it + # unless every selected CUDA architecture is >= 89; sage-attn2.cuh provides + # stub definitions for the dispatcher when it is compiled out. + set(GGML_CUDA_SAGE_ATTN2_ENABLED ON) + foreach(ARCH IN LISTS CMAKE_CUDA_ARCHITECTURES) + string(REGEX MATCH "^[0-9]+" ARCH_NUMBER "${ARCH}") + if (ARCH_NUMBER STREQUAL "" OR ARCH_NUMBER LESS 89) + set(GGML_CUDA_SAGE_ATTN2_ENABLED OFF) + endif() + endforeach() + + if (NOT GGML_CUDA_SAGE_ATTN2_ENABLED) + list(REMOVE_ITEM GGML_SOURCES_CUDA + "${CMAKE_CURRENT_SOURCE_DIR}/sage-attn2.cu" + "sage-attn2.cu") + message(STATUS "CUDA SageAttention2 disabled for CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") + endif() + if (GGML_CUDA_FA_ALL_QUANTS) file(GLOB SRCS "template-instances/fattn-vec*.cu") @@ -127,6 +145,10 @@ if (CUDAToolkit_FOUND) ${GGML_HEADERS_CUDA} ${GGML_SOURCES_CUDA} ) + + if (GGML_CUDA_SAGE_ATTN2_ENABLED) + target_compile_definitions(ggml-cuda PRIVATE GGML_CUDA_SAGE_ATTN2_ENABLED) + endif() add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE}) diff --git a/external/ggml/src/ggml-cuda/sage-attn2.cuh b/external/ggml/src/ggml-cuda/sage-attn2.cuh index 87dfdb70..ff5c0973 100644 --- a/external/ggml/src/ggml-cuda/sage-attn2.cuh +++ b/external/ggml/src/ggml-cuda/sage-attn2.cuh @@ -1,5 +1,6 @@ #include "common.cuh" +#if defined(GGML_CUDA_SAGE_ATTN2_ENABLED) void ggml_cuda_sage_attn2(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_sage_attn2_i8(ggml_backend_cuda_context & ctx, ggml_tensor * dst); @@ -7,3 +8,25 @@ void ggml_cuda_sage_attn2_i8(ggml_backend_cuda_context & ctx, ggml_tensor * dst) bool ggml_cuda_sage_attn2_supported(int device, const ggml_tensor * dst); bool ggml_cuda_sage_attn2_i8_supported(int device, const ggml_tensor * dst); +#else +// SageAttention2 was not built for the selected CUDA architectures (their SM +// is below 89). Provide stub definitions so the ggml-cuda dispatcher still +// links; supported() returns false, so the op is never attached to a graph. +static inline void ggml_cuda_sage_attn2(ggml_backend_cuda_context &, ggml_tensor *) { + GGML_ABORT("CUDA SageAttention2 was not built for the selected CUDA architectures"); +} + +static inline void ggml_cuda_sage_attn2_i8(ggml_backend_cuda_context &, ggml_tensor *) { + GGML_ABORT("CUDA SageAttention2 was not built for the selected CUDA architectures"); +} + +static inline bool ggml_cuda_sage_attn2_supported(int device, const ggml_tensor * dst) { + (void) device; (void) dst; + return false; +} + +static inline bool ggml_cuda_sage_attn2_i8_supported(int device, const ggml_tensor * dst) { + (void) device; (void) dst; + return false; +} +#endif