The open-source AI voice studio. Clone, dictate, create.
feat: chunked TTS generation for long text (engine-agnostic)
James Pine70ca7f66cb79
diff --git a/backend/main.py b/backend/main.pyindex 69fcc869..39e135e5 100644--- a/backend/main.py+++ b/backend/main.py@@ -824,18 +824,24 @@ async def generate_speech(engine=engine,)- audio, sample_rate = await tts_model.generate(- data.text,- voice_prompt,- data.language,- data.seed,- data.instruct,- )+ from .utils.chunked_tts import generate_chunked- # Trim trailing silence/hallucination for Chatterbox output+ # Resolve per-chunk trim function for engines that need it+ trim_fn = Noneif engine in ("chatterbox", "chatterbox_turbo"):from .utils.audio import trim_tts_output- audio = trim_tts_output(audio, sample_rate)+ trim_fn = trim_tts_output++ audio, sample_rate = await generate_chunked(+ tts_model,+ data.text,+ voice_prompt,+ language=data.language,+ seed=data.seed,+ instruct=data.instruct,+ max_chunk_chars=data.max_chunk_chars,+ trim_fn=trim_fn,+ )# Calculate durationduration = len(audio) / sample_rate@@ -949,18 +955,23 @@ async def stream_speech(data.profile_id, db, engine=engine,)- audio, sample_rate = await tts_model.generate(- data.text,- voice_prompt,- data.language,- data.seed,- data.instruct,- )+ from .utils.chunked_tts import generate_chunked- # Trim trailing silence/hallucination for Chatterbox output+ trim_fn = Noneif engine in ("chatterbox", "chatterbox_turbo"):from .utils.audio import trim_tts_output- audio = trim_tts_output(audio, sample_rate)+ trim_fn = trim_tts_output++ audio, sample_rate = await generate_chunked(+ tts_model,+ data.text,+ voice_prompt,+ language=data.language,+ seed=data.seed,+ instruct=data.instruct,+ max_chunk_chars=data.max_chunk_chars,+ trim_fn=trim_fn,+ )wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)diff --git a/backend/models.py b/backend/models.pyindex ebfe70e9..771dfa7b 100644--- a/backend/models.py+++ b/backend/models.py@@ -52,12 +52,13 @@ class ProfileSampleResponse(BaseModel):class GenerationRequest(BaseModel):"""Request model for voice generation."""profile_id: str- text: str = Field(..., min_length=1, max_length=5000)+ text: str = Field(..., min_length=1, max_length=50000)language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")seed: Optional[int] = Field(None, ge=0)model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")instruct: Optional[str] = Field(None, max_length=500)engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo)$")+ max_chunk_chars: int = Field(default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting")class GenerationResponse(BaseModel):diff --git a/backend/utils/chunked_tts.py b/backend/utils/chunked_tts.pynew file mode 100644index 00000000..b9b9f0cc--- /dev/null+++ b/backend/utils/chunked_tts.py@@ -0,0 +1,298 @@+"""+Chunked TTS generation utilities.++Splits long text into sentence-boundary chunks, generates audio per-chunk+via any TTSBackend, and concatenates with crossfade. All logic is+engine-agnostic — it wraps the standard ``TTSBackend.generate()`` interface.++Short text (≤ max_chunk_chars) uses the single-shot fast path with zero+overhead.+"""++import logging+import re+from typing import List, Tuple++import numpy as np++logger = logging.getLogger("voicebox.chunked-tts")++# Default chunk size in characters. Can be overridden per-request via+# the ``max_chunk_chars`` field on GenerationRequest.+DEFAULT_MAX_CHUNK_CHARS = 800++# Common abbreviations that should NOT be treated as sentence endings.+# Lowercase for case-insensitive matching.+_ABBREVIATIONS = frozenset(+ {+ "mr",+ "mrs",+ "ms",+ "dr",+ "prof",+ "sr",+ "jr",+ "st",+ "ave",+ "blvd",+ "inc",+ "ltd",+ "corp",+ "dept",+ "est",+ "approx",+ "vs",+ "etc",+ "e.g",+ "i.e",+ "a.m",+ "p.m",+ "u.s",+ "u.s.a",+ "u.k",+ }+)++# Paralinguistic tags used by Chatterbox Turbo. The splitter must never+# cut inside one of these.+_PARA_TAG_RE = re.compile(r"\[[^\]]*\]")+++# ---------------------------------------------------------------------------+# Text splitting+# ---------------------------------------------------------------------------+++def split_text_into_chunks(text: str, max_chars: int = DEFAULT_MAX_CHUNK_CHARS) -> List[str]:+ """Split *text* at natural boundaries into chunks of at most *max_chars*.++ Priority: sentence-end (``.!?`` not preceded by an abbreviation and not+ inside brackets) → clause boundary (``;:,—``) → whitespace → hard cut.++ Paralinguistic tags like ``[laugh]`` are treated as atomic and will not+ be split across chunks.+ """+ text = text.strip()+ if not text:+ return []+ if len(text) <= max_chars:+ return [text]++ chunks: List[str] = []+ remaining = text++ while remaining:+ remaining = remaining.lstrip()+ if not remaining:+ break+ if len(remaining) <= max_chars:+ chunks.append(remaining)+ break++ segment = remaining[:max_chars]++ # Try to split at the last real sentence ending+ split_pos = _find_last_sentence_end(segment)+ if split_pos == -1:+ split_pos = _find_last_clause_boundary(segment)+ if split_pos == -1:+ split_pos = segment.rfind(" ")+ if split_pos == -1:+ # Absolute fallback: hard cut but avoid splitting inside a tag+ split_pos = _safe_hard_cut(segment, max_chars)++ chunk = remaining[: split_pos + 1].strip()+ if chunk:+ chunks.append(chunk)+ remaining = remaining[split_pos + 1 :]++ return chunks+++def _find_last_sentence_end(text: str) -> int:+ """Return the index of the last sentence-ending punctuation in *text*.++ Skips periods that follow common abbreviations (``Dr.``, ``Mr.``, etc.)+ and periods inside bracket tags (``[laugh]``). Also handles CJK+ sentence-ending punctuation (``。!?``).+ """+ best = -1+ # ASCII sentence ends+ for m in re.finditer(r"[.!?](?:\s|$)", text):+ pos = m.start()+ char = text[pos]+ # Skip periods after abbreviations+ if char == ".":+ # Walk backwards to find the preceding word+ word_start = pos - 1+ while word_start >= 0 and text[word_start].isalpha():+ word_start -= 1+ word = text[word_start + 1 : pos].lower()+ if word in _ABBREVIATIONS:+ continue+ # Skip decimal numbers (digit immediately before the period)+ if word_start >= 0 and text[word_start].isdigit():+ continue+ # Skip if we're inside a bracket tag+ if _inside_bracket_tag(text, pos):+ continue+ best = pos+ # CJK sentence-ending punctuation+ for m in re.finditer(r"[\u3002\uff01\uff1f]", text):+ if m.start() > best:+ best = m.start()+ return best+++def _find_last_clause_boundary(text: str) -> int:+ """Return the index of the last clause-boundary punctuation."""+ best = -1+ for m in re.finditer(r"[;:,\u2014](?:\s|$)", text):+ pos = m.start()+ # Skip if inside a bracket tag+ if _inside_bracket_tag(text, pos):+ continue+ best = pos+ return best+++def _inside_bracket_tag(text: str, pos: int) -> bool:+ """Return True if *pos* falls inside a ``[...]`` tag."""+ for m in _PARA_TAG_RE.finditer(text):+ if m.start() < pos < m.end():+ return True+ return False+++def _safe_hard_cut(segment: str, max_chars: int) -> int:+ """Find a hard-cut position that doesn't split a ``[tag]``."""+ cut = max_chars - 1+ # Check if the cut falls inside a bracket tag; if so, move before it+ for m in _PARA_TAG_RE.finditer(segment):+ if m.start() < cut < m.end():+ return m.start() - 1 if m.start() > 0 else cut+ return cut+++# ---------------------------------------------------------------------------+# Audio concatenation+# ---------------------------------------------------------------------------+++def concatenate_audio_chunks(+ chunks: List[np.ndarray],+ sample_rate: int,+ crossfade_ms: int = 50,+) -> np.ndarray:+ """Concatenate audio arrays with a short crossfade to eliminate clicks.++ Each chunk is expected to be a 1-D float32 ndarray at *sample_rate* Hz.+ """+ if not chunks:+ return np.array([], dtype=np.float32)+ if len(chunks) == 1:+ return chunks[0]++ crossfade_samples = int(sample_rate * crossfade_ms / 1000)+ result = np.array(chunks[0], dtype=np.float32, copy=True)++ for chunk in chunks[1:]:+ if len(chunk) == 0:+ continue+ overlap = min(crossfade_samples, len(result), len(chunk))+ if overlap > 0:+ fade_out = np.linspace(1.0, 0.0, overlap, dtype=np.float32)+ fade_in = np.linspace(0.0, 1.0, overlap, dtype=np.float32)+ result[-overlap:] = result[-overlap:] * fade_out + chunk[:overlap] * fade_in+ result = np.concatenate([result, chunk[overlap:]])+ else:+ result = np.concatenate([result, chunk])++ return result+++# ---------------------------------------------------------------------------+# Engine-agnostic chunked generation+# ---------------------------------------------------------------------------+++async def generate_chunked(+ backend,+ text: str,+ voice_prompt: dict,+ language: str = "en",+ seed: int | None = None,+ instruct: str | None = None,+ max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,+ trim_fn=None,+) -> Tuple[np.ndarray, int]:+ """Generate audio with automatic chunking for long text.++ For text shorter than *max_chunk_chars* this is a thin wrapper around+ ``backend.generate()`` with zero overhead.++ For longer text the input is split at natural sentence boundaries,+ each chunk is generated independently, optionally trimmed (useful for+ Chatterbox engines that hallucinate trailing noise), and the results+ are concatenated with a short crossfade.++ Parameters+ ----------+ backend : TTSBackend+ Any backend implementing the ``generate()`` protocol.+ text : str+ Input text (may be arbitrarily long).+ voice_prompt, language, seed, instruct+ Forwarded to ``backend.generate()`` verbatim.+ max_chunk_chars : int+ Maximum characters per chunk (default 800).+ trim_fn : callable | None+ Optional ``(audio, sample_rate) -> audio`` post-processing+ function applied to each chunk before concatenation (e.g.+ ``trim_tts_output`` for Chatterbox engines).++ Returns+ -------+ (audio, sample_rate) : Tuple[np.ndarray, int]+ """+ chunks = split_text_into_chunks(text, max_chunk_chars)++ if len(chunks) <= 1:+ # Short text — single-shot fast path+ audio, sample_rate = await backend.generate(+ text, voice_prompt, language, seed, instruct,+ )+ if trim_fn is not None:+ audio = trim_fn(audio, sample_rate)+ return audio, sample_rate++ # Long text — chunked generation+ logger.info(+ "Splitting %d chars into %d chunks (max %d chars each)",+ len(text), len(chunks), max_chunk_chars,+ )+ audio_chunks: List[np.ndarray] = []+ sample_rate: int | None = None++ for i, chunk_text in enumerate(chunks):+ logger.info(+ "Generating chunk %d/%d (%d chars)",+ i + 1, len(chunks), len(chunk_text),+ )+ # Vary the seed per chunk to avoid correlated RNG artefacts,+ # but keep it deterministic so the same (text, seed) pair+ # always produces the same output.+ chunk_seed = (seed + i) if seed is not None else None++ chunk_audio, chunk_sr = await backend.generate(+ chunk_text, voice_prompt, language, chunk_seed, instruct,+ )+ if trim_fn is not None:+ chunk_audio = trim_fn(chunk_audio, chunk_sr)++ audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))+ if sample_rate is None:+ sample_rate = chunk_sr++ audio = concatenate_audio_chunks(audio_chunks, sample_rate)+ return audio, sample_rate