The open-source AI voice studio. Clone, dictate, create.
Fix runaway MLX Qwen audio chunks (#964)
AirHuaed54347e8167
diff --git a/backend/backends/__init__.py b/backend/backends/__init__.pyindex 2437a87b..142c430c 100644--- a/backend/backends/__init__.py+++ b/backend/backends/__init__.py@@ -56,6 +56,7 @@ class ModelConfig:model_size: str = "default"size_mb: int = 0needs_trim: bool = False+ retries_runaway: bool = Falsesupports_instruct: bool = Falselanguages: list[str] = field(default_factory=lambda: ["en"])@@ -232,6 +233,10 @@ def _get_qwen_model_configs() -> list[ModelConfig]:repo_1_7b = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"repo_0_6b = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"+ # mlx-audio can continue after an EOS miss with silence followed by+ # codec noise. Retry only the affected text as smaller chunks.+ retries_runaway = backend_type == "mlx"+return [ModelConfig(model_name="qwen-tts-1.7B",@@ -240,6 +245,7 @@ def _get_qwen_model_configs() -> list[ModelConfig]:hf_repo_id=repo_1_7b,model_size="1.7B",size_mb=3500,+ retries_runaway=retries_runaway,supports_instruct=False, # Base model drops instruct silentlylanguages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],),@@ -250,6 +256,7 @@ def _get_qwen_model_configs() -> list[ModelConfig]:hf_repo_id=repo_0_6b,model_size="0.6B",size_mb=1200,+ retries_runaway=retries_runaway,supports_instruct=False,languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],),@@ -504,6 +511,14 @@ def engine_needs_trim(engine: str) -> bool:return False+def engine_retries_runaway(engine: str) -> bool:+ """Whether unstable output should be retried in smaller chunks."""+ for cfg in get_tts_model_configs():+ if cfg.engine == engine:+ return cfg.retries_runaway+ return False++def engine_has_model_sizes(engine: str) -> bool:"""Whether this engine supports multiple model sizes (only Qwen currently)."""configs = [c for c in get_tts_model_configs() if c.engine == engine]diff --git a/backend/routes/generations.py b/backend/routes/generations.pyindex 215c96cb..fbbeece6 100644--- a/backend/routes/generations.py+++ b/backend/routes/generations.py@@ -321,7 +321,13 @@ async def stream_speech(db: Session = Depends(get_db),):"""Generate speech and stream the WAV audio directly without saving to disk."""- from ..backends import get_tts_backend_for_engine, ensure_model_cached_or_raise, load_engine_model, engine_needs_trim+ from ..backends import (+ engine_needs_trim,+ engine_retries_runaway,+ ensure_model_cached_or_raise,+ get_tts_backend_for_engine,+ load_engine_model,+ )profile = await profiles.get_profile(data.profile_id, db)if not profile:@@ -347,10 +353,15 @@ async def stream_speech(from ..utils.chunked_tts import generate_chunkedtrim_fn = None+ runaway_detector = Noneif engine_needs_trim(engine):from ..utils.audio import trim_tts_outputtrim_fn = trim_tts_output+ if engine_retries_runaway(engine):+ from ..utils.audio import has_tts_runaway++ runaway_detector = has_tts_runawayaudio, sample_rate = await generate_chunked(tts_model,@@ -362,6 +373,7 @@ async def stream_speech(max_chunk_chars=data.max_chunk_chars,crossfade_ms=data.crossfade_ms,trim_fn=trim_fn,+ runaway_detector=runaway_detector,)effects_chain_config = Nonediff --git a/backend/services/generation.py b/backend/services/generation.pyindex ce8fe93c..a4b2e8a3 100644--- a/backend/services/generation.py+++ b/backend/services/generation.py@@ -48,9 +48,14 @@ async def run_generation(This is the single entry point for all background generation work.It is designed to be enqueued via ``services.task_queue.enqueue_generation``."""- from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim+ from ..backends import (+ engine_needs_trim,+ engine_retries_runaway,+ get_tts_backend_for_engine,+ load_engine_model,+ )from ..utils.chunked_tts import generate_chunked- from ..utils.audio import normalize_audio, save_audio, trim_tts_output+ from ..utils.audio import has_tts_runaway, normalize_audio, save_audio, trim_tts_outputtask_manager = get_task_manager()bg_db = next(get_db())@@ -72,12 +77,14 @@ async def run_generation(await history.update_generation_status(generation_id, "generating", bg_db)trim_fn = trim_tts_output if engine_needs_trim(engine) else None+ runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else Nonegen_kwargs: dict = dict(language=language,seed=seed if mode != "regenerate" else None,instruct=instruct,trim_fn=trim_fn,+ runaway_detector=runaway_detector,)if max_chunk_chars is not None:gen_kwargs["max_chunk_chars"] = max_chunk_chars@@ -267,9 +274,14 @@ async def generate_audio_sync(normalize, then encodes in-memory via :func:`tts.audio_to_wav_bytes`(same helper ``/generate/stream`` uses)."""- from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim+ from ..backends import (+ engine_needs_trim,+ engine_retries_runaway,+ get_tts_backend_for_engine,+ load_engine_model,+ )from ..utils.chunked_tts import generate_chunked- from ..utils.audio import normalize_audio, trim_tts_output+ from ..utils.audio import has_tts_runaway, normalize_audio, trim_tts_outputfrom . import ttsbg_db = next(get_db())@@ -287,12 +299,14 @@ async def generate_audio_sync(bg_db.close()trim_fn = trim_tts_output if engine_needs_trim(engine) else None+ runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else Nonegen_kwargs: dict = dict(language=language,seed=seed,instruct=instruct,trim_fn=trim_fn,+ runaway_detector=runaway_detector,)if max_chunk_chars is not None:gen_kwargs["max_chunk_chars"] = max_chunk_charsdiff --git a/backend/tests/test_qwen_runaway_retry.py b/backend/tests/test_qwen_runaway_retry.pynew file mode 100644index 00000000..650385f8--- /dev/null+++ b/backend/tests/test_qwen_runaway_retry.py@@ -0,0 +1,117 @@+"""Regression coverage for runaway MLX Qwen TTS output."""++from unittest.mock import patch++import numpy as np+import pytest++from backend.backends import engine_needs_trim, engine_retries_runaway+from backend.utils.audio import has_tts_runaway+from backend.utils.chunked_tts import generate_chunked++SAMPLE_RATE = 1000+++def test_mlx_qwen_enables_runaway_retry_without_aggressive_trim():+ with patch("backend.backends.get_backend_type", return_value="mlx"):+ assert engine_needs_trim("qwen") is False+ assert engine_retries_runaway("qwen") is True+++def test_pytorch_qwen_keeps_runaway_retry_disabled():+ with patch("backend.backends.get_backend_type", return_value="pytorch"):+ assert engine_needs_trim("qwen") is False+ assert engine_retries_runaway("qwen") is False+++def test_detector_flags_long_internal_silence():+ speech = np.full(2 * SAMPLE_RATE, 0.2, dtype=np.float32)+ runaway_gap = np.zeros(2500, dtype=np.float32)+ hallucinated_noise = np.full(2 * SAMPLE_RATE, 0.8, dtype=np.float32)+ audio = np.concatenate([speech, runaway_gap, hallucinated_noise])++ assert has_tts_runaway(audio, SAMPLE_RATE) is True+++def test_detector_ignores_normal_internal_pause():+ speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)+ normal_pause = np.zeros(1200, dtype=np.float32)+ audio = np.concatenate([speech, normal_pause, speech])++ assert has_tts_runaway(audio, SAMPLE_RATE) is False+++def test_trailing_silence_is_not_a_runaway():+ speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)+ trailing_silence = np.zeros(2 * SAMPLE_RATE, dtype=np.float32)++ assert (+ has_tts_runaway(+ np.concatenate([speech, trailing_silence]),+ SAMPLE_RATE,+ )+ is False+ )+++async def test_runaway_chunk_is_retried_as_smaller_chunks():+ class FakeBackend:+ def __init__(self):+ self.calls = []++ async def generate(self, text, *_args):+ self.calls.append(text)+ if len(text) > 200:+ speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)+ silence = np.zeros(2500, dtype=np.float32)+ noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)+ return np.concatenate([speech, silence, noise]), SAMPLE_RATE+ return np.full(SAMPLE_RATE, 0.2, dtype=np.float32), SAMPLE_RATE++ backend = FakeBackend()+ text = f"{'A' * 119}. {'B' * 119}."++ audio, sample_rate = await generate_chunked(+ backend,+ text,+ {},+ max_chunk_chars=800,+ crossfade_ms=50,+ runaway_detector=has_tts_runaway,+ )++ assert sample_rate == SAMPLE_RATE+ assert backend.calls == [text, f"{'A' * 119}.", f"{'B' * 119}."]+ assert len(audio) == 1950+++async def test_persistent_runaway_fails_instead_of_returning_corrupt_audio():+ class AlwaysRunawayBackend:+ def __init__(self):+ self.calls = []++ async def generate(self, text, *_args):+ self.calls.append(text)+ speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)+ silence = np.zeros(2500, dtype=np.float32)+ noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)+ return np.concatenate([speech, silence, noise]), SAMPLE_RATE++ backend = AlwaysRunawayBackend()+ text = f"{'A' * 119}. {'B' * 119}."++ with pytest.raises(+ RuntimeError,+ match="remained unstable after retrying smaller text chunks",+ ):+ await generate_chunked(+ backend,+ text,+ {},+ max_chunk_chars=800,+ runaway_detector=has_tts_runaway,+ )++ assert [len(call) for call in backend.calls] == [241, 120, 100]diff --git a/backend/utils/audio.py b/backend/utils/audio.pyindex 7e0fd6fd..bccab2c8 100644--- a/backend/utils/audio.py+++ b/backend/utils/audio.py@@ -110,6 +110,43 @@ def save_audio(raise OSError(f"Failed to save audio to {path}: {e}") from e+def has_tts_runaway(+ audio: np.ndarray,+ sample_rate: int = 24000,+ frame_ms: int = 20,+ silence_threshold_db: float = -40.0,+ max_internal_silence_ms: int = 2000,+) -> bool:+ """Detect speech followed by a long silence and then more output.++ This shape is a reliable signal that a TTS model missed EOS and resumed+ with hallucinated speech or codec noise. Leading and trailing silence do+ not count because they are not bounded by non-silent audio.+ """+ frame_len = int(sample_rate * frame_ms / 1000)+ if frame_len == 0 or len(audio) < frame_len:+ return False++ n_frames = len(audio) // frame_len+ threshold_linear = 10 ** (silence_threshold_db / 20)+ max_silence_frames = int(max_internal_silence_ms / frame_ms)+ seen_speech = False+ consecutive_silence = 0++ for i in range(n_frames):+ frame = audio[i * frame_len : (i + 1) * frame_len]+ is_speech = np.sqrt(np.mean(frame**2)) >= threshold_linear+ if is_speech:+ if seen_speech and consecutive_silence >= max_silence_frames:+ return True+ seen_speech = True+ consecutive_silence = 0+ elif seen_speech:+ consecutive_silence += 1++ return False++def trim_tts_output(audio: np.ndarray,sample_rate: int = 24000,diff --git a/backend/utils/chunked_tts.py b/backend/utils/chunked_tts.pyindex 1f43379e..7dbefdb7 100644--- a/backend/utils/chunked_tts.py+++ b/backend/utils/chunked_tts.py@@ -20,6 +20,8 @@ logger = logging.getLogger("voicebox.chunked-tts")# Default chunk size in characters. Can be overridden per-request via# the ``max_chunk_chars`` field on GenerationRequest.DEFAULT_MAX_CHUNK_CHARS = 800+MAX_RUNAWAY_RETRIES = 2+MIN_RUNAWAY_RETRY_CHARS = 100# Common abbreviations that should NOT be treated as sentence endings.# Lowercase for case-insensitive matching.@@ -211,6 +213,7 @@ async def generate_chunked(max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,crossfade_ms: int = 50,trim_fn=None,+ runaway_detector=None,) -> Tuple[np.ndarray, int]:"""Generate audio with automatic chunking for long text.@@ -239,25 +242,75 @@ async def generate_chunked(Optional ``(audio, sample_rate) -> audio`` post-processingfunction applied to each chunk before concatenation (e.g.``trim_tts_output`` for Chatterbox engines).+ runaway_detector : callable | None+ Optional ``(audio, sample_rate) -> bool`` detector. When it flags+ unstable output, the affected text is split in half and retried.Returns-------(audio, sample_rate) : Tuple[np.ndarray, int]"""- chunks = split_text_into_chunks(text, max_chunk_chars)-- if len(chunks) <= 1:- # Short text — single-shot fast path- audio, sample_rate = await backend.generate(- text,+ async def generate_one(+ chunk_text: str,+ chunk_seed: int | None,+ retry_depth: int = 0,+ ) -> tuple[np.ndarray, int]:+ chunk_audio, chunk_sr = await backend.generate(+ chunk_text,voice_prompt,language,- seed,+ chunk_seed,instruct,)++ if runaway_detector is not None and runaway_detector(chunk_audio, chunk_sr):+ if retry_depth >= MAX_RUNAWAY_RETRIES or len(chunk_text) <= MIN_RUNAWAY_RETRY_CHARS:+ raise RuntimeError(+ "TTS output remained unstable after retrying smaller text chunks"+ )++ retry_max_chars = max(MIN_RUNAWAY_RETRY_CHARS, len(chunk_text) // 2)+ retry_chunks = split_text_into_chunks(chunk_text, retry_max_chars)+ if len(retry_chunks) <= 1:+ raise RuntimeError("Unable to split unstable TTS output for retry")++ logger.warning(+ "Detected unstable TTS output for %d chars; retrying as %d smaller chunks",+ len(chunk_text),+ len(retry_chunks),+ )+ retry_audio: list[np.ndarray] = []+ for i, retry_text in enumerate(retry_chunks):+ retry_seed = (+ chunk_seed + ((retry_depth + 1) * 1000) + i+ if chunk_seed is not None+ else None+ )+ audio, sample_rate = await generate_one(+ retry_text,+ retry_seed,+ retry_depth + 1,+ )+ retry_audio.append(np.asarray(audio, dtype=np.float32))++ return (+ concatenate_audio_chunks(+ retry_audio,+ sample_rate,+ crossfade_ms=crossfade_ms,+ ),+ sample_rate,+ )+if trim_fn is not None:- audio = trim_fn(audio, sample_rate)- return audio, sample_rate+ chunk_audio = trim_fn(chunk_audio, chunk_sr)+ return np.asarray(chunk_audio, dtype=np.float32), chunk_sr++ chunks = split_text_into_chunks(text, max_chunk_chars)++ if len(chunks) <= 1:+ # Short text — single-shot fast path+ return await generate_one(text, seed)# Long text — chunked generationlogger.info(@@ -281,17 +334,12 @@ async def generate_chunked(# always produces the same output.chunk_seed = (seed + i) if seed is not None else None- chunk_audio, chunk_sr = await backend.generate(+ chunk_audio, chunk_sr = await generate_one(chunk_text,- voice_prompt,- language,chunk_seed,- instruct,)- if trim_fn is not None:- chunk_audio = trim_fn(chunk_audio, chunk_sr)- audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))+ audio_chunks.append(chunk_audio)if sample_rate is None:sample_rate = chunk_srdiff --git a/docs/content/docs/developer/tts-generation.mdx b/docs/content/docs/developer/tts-generation.mdxindex 5b902755..0f685622 100644--- a/docs/content/docs/developer/tts-generation.mdx+++ b/docs/content/docs/developer/tts-generation.mdx@@ -49,6 +49,7 @@ class ModelConfig:model_size: str = "default"size_mb: int = 0needs_trim: bool = False+ retries_runaway: bool = Falsesupports_instruct: bool = Falselanguages: list[str] = field(default_factory=lambda: ["en"])```@@ -59,6 +60,7 @@ Registry helpers in `backends/__init__.py` replace what used to be per-engine `i- `get_tts_model_configs()` — only TTS variants- `get_model_config(model_name)` — lookup by name- `engine_needs_trim(engine)` — whether output should run through `trim_tts_output()`+- `engine_retries_runaway(engine)` — whether unstable output should be retried as smaller chunks- `load_engine_model(engine, model_size)` — downloads + loads, handles engines with multiple sizes- `get_tts_backend_for_engine(engine)` — thread-safe backend factory with double-checked locking@@ -152,7 +154,7 @@ The request path from frontend to audio file:6. **Inference** — the engine's `generate()` returns `(audio_array, sample_rate)`.-7. **Post-process** — if `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.+7. **Validate and post-process** — engines with `retries_runaway=True` retry unstable output as smaller chunks. If `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.8. **Persist** — audio is written to the generations directory, a row is inserted into the `generations` table, and the response includes the generation metadata.