Skip to content

The open-source AI voice studio. Clone, dictate, create.

feat: chunked TTS generation for long text (engine-agnostic)

James Pine70ca7f66cb79
diff --git a/backend/main.py b/backend/main.py
index 69fcc869..39e135e5 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -824,18 +824,24 @@ async def generate_speech(
engine=engine,
)
- audio, sample_rate = await tts_model.generate(
- data.text,
- voice_prompt,
- data.language,
- data.seed,
- data.instruct,
- )
+ from .utils.chunked_tts import generate_chunked
- # Trim trailing silence/hallucination for Chatterbox output
+ # Resolve per-chunk trim function for engines that need it
+ trim_fn = None
if engine in ("chatterbox", "chatterbox_turbo"):
from .utils.audio import trim_tts_output
- audio = trim_tts_output(audio, sample_rate)
+ trim_fn = trim_tts_output
+
+ audio, sample_rate = await generate_chunked(
+ tts_model,
+ data.text,
+ voice_prompt,
+ language=data.language,
+ seed=data.seed,
+ instruct=data.instruct,
+ max_chunk_chars=data.max_chunk_chars,
+ trim_fn=trim_fn,
+ )
# Calculate duration
duration = len(audio) / sample_rate
@@ -949,18 +955,23 @@ async def stream_speech(
data.profile_id, db, engine=engine,
)
- audio, sample_rate = await tts_model.generate(
- data.text,
- voice_prompt,
- data.language,
- data.seed,
- data.instruct,
- )
+ from .utils.chunked_tts import generate_chunked
- # Trim trailing silence/hallucination for Chatterbox output
+ trim_fn = None
if engine in ("chatterbox", "chatterbox_turbo"):
from .utils.audio import trim_tts_output
- audio = trim_tts_output(audio, sample_rate)
+ trim_fn = trim_tts_output
+
+ audio, sample_rate = await generate_chunked(
+ tts_model,
+ data.text,
+ voice_prompt,
+ language=data.language,
+ seed=data.seed,
+ instruct=data.instruct,
+ max_chunk_chars=data.max_chunk_chars,
+ trim_fn=trim_fn,
+ )
wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)
diff --git a/backend/models.py b/backend/models.py
index ebfe70e9..771dfa7b 100644
--- a/backend/models.py
+++ b/backend/models.py
@@ -52,12 +52,13 @@ class ProfileSampleResponse(BaseModel):
class GenerationRequest(BaseModel):
"""Request model for voice generation."""
profile_id: str
- text: str = Field(..., min_length=1, max_length=5000)
+ text: str = Field(..., min_length=1, max_length=50000)
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")
seed: Optional[int] = Field(None, ge=0)
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
instruct: Optional[str] = Field(None, max_length=500)
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo)$")
+ max_chunk_chars: int = Field(default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting")
class GenerationResponse(BaseModel):
diff --git a/backend/utils/chunked_tts.py b/backend/utils/chunked_tts.py
new file mode 100644
index 00000000..b9b9f0cc
--- /dev/null
+++ b/backend/utils/chunked_tts.py
@@ -0,0 +1,298 @@
+"""
+Chunked TTS generation utilities.
+
+Splits long text into sentence-boundary chunks, generates audio per-chunk
+via any TTSBackend, and concatenates with crossfade. All logic is
+engine-agnostic — it wraps the standard ``TTSBackend.generate()`` interface.
+
+Short text (≤ max_chunk_chars) uses the single-shot fast path with zero
+overhead.
+"""
+
+import logging
+import re
+from typing import List, Tuple
+
+import numpy as np
+
+logger = logging.getLogger("voicebox.chunked-tts")
+
+# Default chunk size in characters. Can be overridden per-request via
+# the ``max_chunk_chars`` field on GenerationRequest.
+DEFAULT_MAX_CHUNK_CHARS = 800
+
+# Common abbreviations that should NOT be treated as sentence endings.
+# Lowercase for case-insensitive matching.
+_ABBREVIATIONS = frozenset(
+ {
+ "mr",
+ "mrs",
+ "ms",
+ "dr",
+ "prof",
+ "sr",
+ "jr",
+ "st",
+ "ave",
+ "blvd",
+ "inc",
+ "ltd",
+ "corp",
+ "dept",
+ "est",
+ "approx",
+ "vs",
+ "etc",
+ "e.g",
+ "i.e",
+ "a.m",
+ "p.m",
+ "u.s",
+ "u.s.a",
+ "u.k",
+ }
+)
+
+# Paralinguistic tags used by Chatterbox Turbo. The splitter must never
+# cut inside one of these.
+_PARA_TAG_RE = re.compile(r"\[[^\]]*\]")
+
+
+# ---------------------------------------------------------------------------
+# Text splitting
+# ---------------------------------------------------------------------------
+
+
+def split_text_into_chunks(text: str, max_chars: int = DEFAULT_MAX_CHUNK_CHARS) -> List[str]:
+ """Split *text* at natural boundaries into chunks of at most *max_chars*.
+
+ Priority: sentence-end (``.!?`` not preceded by an abbreviation and not
+ inside brackets) → clause boundary (``;:,—``) → whitespace → hard cut.
+
+ Paralinguistic tags like ``[laugh]`` are treated as atomic and will not
+ be split across chunks.
+ """
+ text = text.strip()
+ if not text:
+ return []
+ if len(text) <= max_chars:
+ return [text]
+
+ chunks: List[str] = []
+ remaining = text
+
+ while remaining:
+ remaining = remaining.lstrip()
+ if not remaining:
+ break
+ if len(remaining) <= max_chars:
+ chunks.append(remaining)
+ break
+
+ segment = remaining[:max_chars]
+
+ # Try to split at the last real sentence ending
+ split_pos = _find_last_sentence_end(segment)
+ if split_pos == -1:
+ split_pos = _find_last_clause_boundary(segment)
+ if split_pos == -1:
+ split_pos = segment.rfind(" ")
+ if split_pos == -1:
+ # Absolute fallback: hard cut but avoid splitting inside a tag
+ split_pos = _safe_hard_cut(segment, max_chars)
+
+ chunk = remaining[: split_pos + 1].strip()
+ if chunk:
+ chunks.append(chunk)
+ remaining = remaining[split_pos + 1 :]
+
+ return chunks
+
+
+def _find_last_sentence_end(text: str) -> int:
+ """Return the index of the last sentence-ending punctuation in *text*.
+
+ Skips periods that follow common abbreviations (``Dr.``, ``Mr.``, etc.)
+ and periods inside bracket tags (``[laugh]``). Also handles CJK
+ sentence-ending punctuation (``。!?``).
+ """
+ best = -1
+ # ASCII sentence ends
+ for m in re.finditer(r"[.!?](?:\s|$)", text):
+ pos = m.start()
+ char = text[pos]
+ # Skip periods after abbreviations
+ if char == ".":
+ # Walk backwards to find the preceding word
+ word_start = pos - 1
+ while word_start >= 0 and text[word_start].isalpha():
+ word_start -= 1
+ word = text[word_start + 1 : pos].lower()
+ if word in _ABBREVIATIONS:
+ continue
+ # Skip decimal numbers (digit immediately before the period)
+ if word_start >= 0 and text[word_start].isdigit():
+ continue
+ # Skip if we're inside a bracket tag
+ if _inside_bracket_tag(text, pos):
+ continue
+ best = pos
+ # CJK sentence-ending punctuation
+ for m in re.finditer(r"[\u3002\uff01\uff1f]", text):
+ if m.start() > best:
+ best = m.start()
+ return best
+
+
+def _find_last_clause_boundary(text: str) -> int:
+ """Return the index of the last clause-boundary punctuation."""
+ best = -1
+ for m in re.finditer(r"[;:,\u2014](?:\s|$)", text):
+ pos = m.start()
+ # Skip if inside a bracket tag
+ if _inside_bracket_tag(text, pos):
+ continue
+ best = pos
+ return best
+
+
+def _inside_bracket_tag(text: str, pos: int) -> bool:
+ """Return True if *pos* falls inside a ``[...]`` tag."""
+ for m in _PARA_TAG_RE.finditer(text):
+ if m.start() < pos < m.end():
+ return True
+ return False
+
+
+def _safe_hard_cut(segment: str, max_chars: int) -> int:
+ """Find a hard-cut position that doesn't split a ``[tag]``."""
+ cut = max_chars - 1
+ # Check if the cut falls inside a bracket tag; if so, move before it
+ for m in _PARA_TAG_RE.finditer(segment):
+ if m.start() < cut < m.end():
+ return m.start() - 1 if m.start() > 0 else cut
+ return cut
+
+
+# ---------------------------------------------------------------------------
+# Audio concatenation
+# ---------------------------------------------------------------------------
+
+
+def concatenate_audio_chunks(
+ chunks: List[np.ndarray],
+ sample_rate: int,
+ crossfade_ms: int = 50,
+) -> np.ndarray:
+ """Concatenate audio arrays with a short crossfade to eliminate clicks.
+
+ Each chunk is expected to be a 1-D float32 ndarray at *sample_rate* Hz.
+ """
+ if not chunks:
+ return np.array([], dtype=np.float32)
+ if len(chunks) == 1:
+ return chunks[0]
+
+ crossfade_samples = int(sample_rate * crossfade_ms / 1000)
+ result = np.array(chunks[0], dtype=np.float32, copy=True)
+
+ for chunk in chunks[1:]:
+ if len(chunk) == 0:
+ continue
+ overlap = min(crossfade_samples, len(result), len(chunk))
+ if overlap > 0:
+ fade_out = np.linspace(1.0, 0.0, overlap, dtype=np.float32)
+ fade_in = np.linspace(0.0, 1.0, overlap, dtype=np.float32)
+ result[-overlap:] = result[-overlap:] * fade_out + chunk[:overlap] * fade_in
+ result = np.concatenate([result, chunk[overlap:]])
+ else:
+ result = np.concatenate([result, chunk])
+
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Engine-agnostic chunked generation
+# ---------------------------------------------------------------------------
+
+
+async def generate_chunked(
+ backend,
+ text: str,
+ voice_prompt: dict,
+ language: str = "en",
+ seed: int | None = None,
+ instruct: str | None = None,
+ max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,
+ trim_fn=None,
+) -> Tuple[np.ndarray, int]:
+ """Generate audio with automatic chunking for long text.
+
+ For text shorter than *max_chunk_chars* this is a thin wrapper around
+ ``backend.generate()`` with zero overhead.
+
+ For longer text the input is split at natural sentence boundaries,
+ each chunk is generated independently, optionally trimmed (useful for
+ Chatterbox engines that hallucinate trailing noise), and the results
+ are concatenated with a short crossfade.
+
+ Parameters
+ ----------
+ backend : TTSBackend
+ Any backend implementing the ``generate()`` protocol.
+ text : str
+ Input text (may be arbitrarily long).
+ voice_prompt, language, seed, instruct
+ Forwarded to ``backend.generate()`` verbatim.
+ max_chunk_chars : int
+ Maximum characters per chunk (default 800).
+ trim_fn : callable | None
+ Optional ``(audio, sample_rate) -> audio`` post-processing
+ function applied to each chunk before concatenation (e.g.
+ ``trim_tts_output`` for Chatterbox engines).
+
+ Returns
+ -------
+ (audio, sample_rate) : Tuple[np.ndarray, int]
+ """
+ chunks = split_text_into_chunks(text, max_chunk_chars)
+
+ if len(chunks) <= 1:
+ # Short text — single-shot fast path
+ audio, sample_rate = await backend.generate(
+ text, voice_prompt, language, seed, instruct,
+ )
+ if trim_fn is not None:
+ audio = trim_fn(audio, sample_rate)
+ return audio, sample_rate
+
+ # Long text — chunked generation
+ logger.info(
+ "Splitting %d chars into %d chunks (max %d chars each)",
+ len(text), len(chunks), max_chunk_chars,
+ )
+ audio_chunks: List[np.ndarray] = []
+ sample_rate: int | None = None
+
+ for i, chunk_text in enumerate(chunks):
+ logger.info(
+ "Generating chunk %d/%d (%d chars)",
+ i + 1, len(chunks), len(chunk_text),
+ )
+ # Vary the seed per chunk to avoid correlated RNG artefacts,
+ # but keep it deterministic so the same (text, seed) pair
+ # always produces the same output.
+ chunk_seed = (seed + i) if seed is not None else None
+
+ chunk_audio, chunk_sr = await backend.generate(
+ chunk_text, voice_prompt, language, chunk_seed, instruct,
+ )
+ if trim_fn is not None:
+ chunk_audio = trim_fn(chunk_audio, chunk_sr)
+
+ audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))
+ if sample_rate is None:
+ sample_rate = chunk_sr
+
+ audio = concatenate_audio_chunks(audio_chunks, sample_rate)
+ return audio, sample_rate