VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(generate): trim engine edge silence from generated speech
velixiodbafbac3c8f8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f17da38cd..bd86e345e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -19,6 +19,7 @@ metadata and the backend fallback mirror it.### Fixed+- Generated speech no longer starts and ends with silence, and `[pause]` markers give exactly the pause you asked for (Raw preset and explicit durations untouched)- Engine sidecars no longer load PyTorch just to answer the Test Engine ping, and a failed test logs its reason locally (#3026) — thanks @tekay-de!### Fixeddiff --git a/backend/api/routers/generation.py b/backend/api/routers/generation.pyindex a17ab87bc..5d2d3564d 100644--- a/backend/api/routers/generation.py+++ b/backend/api/routers/generation.py@@ -511,10 +511,35 @@ def _note_generate_progress() -> None:pass-def _render_with_pauses(gen_span, segments, sample_rate):+def _trim_output_edges(audio_out, sample_rate):+ """Strip the engine's own lead-in/tail silence from a finished take.++ Engines pad every render with near-silence at both ends, so a plain+ Generate used to start and end with an audible blank. Same helper and+ floor as the dub path (50 ms of context kept; internal pauses and dead+ renders untouched). Non-tensor output passes through unchanged.+ """+ import torch+ from services.audio_dsp import trim_speech_padding++ if not torch.is_tensor(audio_out):+ return audio_out+ return trim_speech_padding(audio_out, sample_rate)+++def _wants_edge_trim(effect_preset, duration=None):+ """Edge trim applies unless the user asked for raw model output or an+ explicit target duration (trimming would shorten what they asked for)."""+ return (effect_preset or "broadcast") != "raw" and not duration+++def _render_with_pauses(gen_span, segments, sample_rate, *, trim_spans=False):"""Synthesize ``[(text, pause_ms), ...]`` spans and stitch silence betweenthem (issue #276).+ ``trim_spans`` strips each span's engine padding first, so a requested+ pause is the pause the user hears, not pause + the engine's own silence.+``gen_span(text) -> torch.Tensor`` synthesizes one text span (raw modeloutput). A silence buffer of ``pause_ms`` is inserted after a span whenrequested, matching the audio tensor's channel dims / dtype / device.@@ -526,7 +551,10 @@ def _render_with_pauses(gen_span, segments, sample_rate):items = [] # ('a', tensor) for audio, ('s', n_samples) for silencefor span_text, pause_ms in segments:if span_text and span_text.strip():- items.append(("a", gen_span(span_text)))+ audio = gen_span(span_text)+ if trim_spans:+ audio = _trim_output_edges(audio, sample_rate)+ items.append(("a", audio))_note_generate_progress()if pause_ms > 0:n = int(round(sample_rate * pause_ms / 1000.0))@@ -572,9 +600,13 @@ def _sanitize_audio(audio_out):@_render_timed('effects')-def _apply_effect_chain(audio_out, sample_rate, effect_preset, *, skip_mastering=False):- """Shared post-DSP for /generate: preset validation → mastering →- effect chain → loudness normalization.+def _apply_effect_chain(audio_out, sample_rate, effect_preset, *, skip_mastering=False,+ trim_edges=False):+ """Shared post-DSP for /generate: preset validation → edge trim →+ mastering → effect chain → loudness normalization.++ ``trim_edges`` is for a finished whole take only: streamed preview chunks+ leave it off so they keep the same timing as the assembled take.``skip_mastering`` honors a backend's ``applies_own_mastering`` flag(issue #312): studio engines (e.g. VoxCPM2's native 48 kHz output)@@ -606,6 +638,9 @@ def _apply_effect_chain(audio_out, sample_rate, effect_preset, *, skip_mastering# Raw: skip all DSP — return raw model outputreturn audio_out+ if trim_edges:+ audio_out = _trim_output_edges(audio_out, sample_rate)+if not skip_mastering:audio_out = apply_mastering(audio_out, sample_rate=sample_rate)chain = get_effect_chain(preset)@@ -1173,7 +1208,8 @@ def _run_inference(# Per-span duration is left to the model; an explicit overall# `duration` can't be meaningfully split across spans.return _gen(span_text, None)[0]- audio_out = _render_with_pauses(_gen_span, segments, sr)+ audio_out = _render_with_pauses(_gen_span, segments, sr,+ trim_spans=_wants_edge_trim(effect_preset))else:# Wave 1.2: long text is split at sentence boundaries and the# per-chunk audio crossfaded — removes the length ceiling. Short@@ -1203,7 +1239,12 @@ def _run_inference(# Apply DSP effect preset. The VoiceStudio model never masters its own# output, so mastering always runs here (unchanged behavior).- return _apply_effect_chain(audio_out, sr, effect_preset)+ # Pause renders were trimmed per span above; trimming the whole take+ # would also eat a deliberate leading/trailing [pause].+ return _apply_effect_chain(+ audio_out, sr, effect_preset,+ trim_edges=not has_pause and _wants_edge_trim(effect_preset, duration),+ )except ValueError as e:# Don't wrap validation errors in OOM message@@ -1276,7 +1317,8 @@ def _run_backend_inference(span_kwargs["seed"] = used_seedfirst_span = Falsereturn trace_call("synthesis", backend.generate, span_text, duration=None, **span_kwargs)- audio_out = _render_with_pauses(_gen_span, segments, sr)+ audio_out = _render_with_pauses(_gen_span, segments, sr,+ trim_spans=_wants_edge_trim(effect_preset))else:# Wave 1.2: sentence-boundary chunking for long text (see# _run_inference for the rationale; behavior is identical here).@@ -1310,6 +1352,7 @@ def _run_backend_inference(return _apply_effect_chain(audio_out, sr, effect_preset,skip_mastering=getattr(backend, "applies_own_mastering", False),+ trim_edges=not has_pause and _wants_edge_trim(effect_preset, duration),)except ValueError as e:@@ -2213,7 +2256,8 @@ async def generate_speech(sink=_dropped_sink)skip = (getattr(_backend, "applies_own_mastering", False)if _backend is not None else False)- return _apply_effect_chain(audio_out, sr, effect_preset, skip_mastering=skip)+ return _apply_effect_chain(audio_out, sr, effect_preset, skip_mastering=skip,+ trim_edges=_wants_edge_trim(effect_preset))except ValueError:raiseexcept Exception as e:diff --git a/backend/api/routers/openai_compat.py b/backend/api/routers/openai_compat.pyindex eb805f5ee..a419f2477 100644--- a/backend/api/routers/openai_compat.py+++ b/backend/api/routers/openai_compat.py@@ -583,10 +583,15 @@ def _typed_speech_http_error(e: Exception) -> Optional[HTTPException]:def _run_tts(backend, text: str, kw: dict):"""Run TTS inference in the GPU thread pool."""- from services.audio_dsp import apply_mastering, normalize_audio+ import torch+ from services.audio_dsp import apply_mastering, normalize_audio, trim_speech_paddingfrom services.watermark import mark_syntheticwav = backend.generate(text, **kw)sr = backend.sample_rate+ # Strip the engine's lead-in/tail silence, as /generate does — unless the+ # caller asked for an exact `duration`, which a trim would shorten.+ if not kw.get("duration") and torch.is_tensor(wav):+ wav = trim_speech_padding(wav, sr)# Engines that already emit mastered, studio-grade audio (e.g. VoxCPM2's# native 48 kHz) opt out of apply_mastering via `applies_own_mastering`.# That chain's highpass + Compressor is tuned for VoiceStudio's 24 kHz clonediff --git a/tests/test_generate_edge_trim.py b/tests/test_generate_edge_trim.pynew file mode 100644index 000000000..d17c19a53--- /dev/null+++ b/tests/test_generate_edge_trim.py@@ -0,0 +1,90 @@+"""Generated takes must not start or end with the engine's own silence padding.++Engines pad every render with near-silence at both ends; only the dub path+trimmed it, so every Generate / OpenAI-API take opened and closed on a blank.+"""+from types import SimpleNamespace++import pytest+import torch++import services.audio_dsp as dsp+from api.routers import generation as gen+from api.routers import openai_compat as oc++SR = 1000 # 1 sample per ms keeps the arithmetic readable+MARGIN = 50 # trim_speech_padding keeps 50 ms of context+++def _padded(lead=400, speech=1000, tail=600):+ """[1, n] take: `lead` ms of engine silence, speech, `tail` ms of silence."""+ return torch.cat([torch.zeros(1, lead), torch.full((1, speech), 0.5),+ torch.zeros(1, tail)], dim=-1)+++def no_dsp(monkeypatch):+ # Isolate the trim from mastering / pedalboard: both become identity.+ monkeypatch.setattr(dsp, "apply_mastering", lambda a, sample_rate=None: a)+ monkeypatch.setattr(dsp, "apply_effects_chain", lambda a, sample_rate, chain: a)+++def test_whole_take_trim_strips_engine_padding(no_dsp):+ out = gen._apply_effect_chain(_padded(), SR, "broadcast", trim_edges=True)+ assert out.shape[-1] == 1000 + 2 * MARGIN+++def test_preview_chunks_and_raw_keep_their_edges(no_dsp):+ assert gen._apply_effect_chain(_padded(), SR, "broadcast").shape[-1] == 2000+ assert gen._apply_effect_chain(_padded(), SR, "raw", trim_edges=True).shape[-1] == 2000+++def test_dead_render_is_not_shrunk(no_dsp):+ silent = torch.zeros(1, 2000)+ assert gen._apply_effect_chain(silent, SR, "broadcast", trim_edges=True).shape[-1] == 2000++[email protected]("preset,duration,expected", [+ ("broadcast", None, True), (None, None, True), ("raw", None, False),+ ("broadcast", 5.0, False),+])+def test_wants_edge_trim(preset, duration, expected):+ assert gen._wants_edge_trim(preset, duration) is expected+++def test_pause_spans_trimmed_so_requested_pause_is_exact():+ segs = [("", 300), ("hello", 500), ("world", 200)]+ out = gen._render_with_pauses(lambda t: _padded(), segs, SR, trim_spans=True)+ span = 1000 + 2 * MARGIN+ # Deliberate leading and trailing pauses survive; engine padding does not.+ assert out.shape[-1] == 300 + span + 500 + span + 200+ assert torch.all(out[:, :300] == 0)+ assert torch.all(out[:, -200:] == 0)+++def test_pause_spans_untrimmed_by_default():+ out = gen._render_with_pauses(lambda t: _padded(), [("hi", 0)], SR)+ assert out.shape[-1] == 2000+++class _Backend:+ sample_rate = SR+ applies_own_mastering = True++ def generate(self, text, **kw):+ return _padded()+++def openai_tts(monkeypatch):+ import services.watermark as wm+ monkeypatch.setattr(wm, "mark_synthetic", lambda w, sr, **k: w)+ return lambda kw: oc._run_tts(_Backend(), "hi", kw)[0]+++def test_openai_speech_trims_edges(openai_tts):+ assert openai_tts({}).shape[-1] == 1000 + 2 * MARGIN+++def test_openai_speech_keeps_explicit_duration(openai_tts):+ assert openai_tts({"duration": 2.0}).shape[-1] == 2000