Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(generate): trim engine edge silence from generated speech

velixiodbafbac3c8f8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f17da38cd..bd86e345e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ metadata and the backend fallback mirror it.
### Fixed
+- Generated speech no longer starts and ends with silence, and `[pause]` markers give exactly the pause you asked for (Raw preset and explicit durations untouched)
- Engine sidecars no longer load PyTorch just to answer the Test Engine ping, and a failed test logs its reason locally (#3026) — thanks @tekay-de!
### Fixed
diff --git a/backend/api/routers/generation.py b/backend/api/routers/generation.py
index a17ab87bc..5d2d3564d 100644
--- a/backend/api/routers/generation.py
+++ b/backend/api/routers/generation.py
@@ -511,10 +511,35 @@ def _note_generate_progress() -> None:
pass
-def _render_with_pauses(gen_span, segments, sample_rate):
+def _trim_output_edges(audio_out, sample_rate):
+ """Strip the engine's own lead-in/tail silence from a finished take.
+
+ Engines pad every render with near-silence at both ends, so a plain
+ Generate used to start and end with an audible blank. Same helper and
+ floor as the dub path (50 ms of context kept; internal pauses and dead
+ renders untouched). Non-tensor output passes through unchanged.
+ """
+ import torch
+ from services.audio_dsp import trim_speech_padding
+
+ if not torch.is_tensor(audio_out):
+ return audio_out
+ return trim_speech_padding(audio_out, sample_rate)
+
+
+def _wants_edge_trim(effect_preset, duration=None):
+ """Edge trim applies unless the user asked for raw model output or an
+ explicit target duration (trimming would shorten what they asked for)."""
+ return (effect_preset or "broadcast") != "raw" and not duration
+
+
+def _render_with_pauses(gen_span, segments, sample_rate, *, trim_spans=False):
"""Synthesize ``[(text, pause_ms), ...]`` spans and stitch silence between
them (issue #276).
+ ``trim_spans`` strips each span's engine padding first, so a requested
+ pause is the pause the user hears, not pause + the engine's own silence.
+
``gen_span(text) -> torch.Tensor`` synthesizes one text span (raw model
output). A silence buffer of ``pause_ms`` is inserted after a span when
requested, matching the audio tensor's channel dims / dtype / device.
@@ -526,7 +551,10 @@ def _render_with_pauses(gen_span, segments, sample_rate):
items = [] # ('a', tensor) for audio, ('s', n_samples) for silence
for span_text, pause_ms in segments:
if span_text and span_text.strip():
- items.append(("a", gen_span(span_text)))
+ audio = gen_span(span_text)
+ if trim_spans:
+ audio = _trim_output_edges(audio, sample_rate)
+ items.append(("a", audio))
_note_generate_progress()
if pause_ms > 0:
n = int(round(sample_rate * pause_ms / 1000.0))
@@ -572,9 +600,13 @@ def _sanitize_audio(audio_out):
@_render_timed('effects')
-def _apply_effect_chain(audio_out, sample_rate, effect_preset, *, skip_mastering=False):
- """Shared post-DSP for /generate: preset validation → mastering →
- effect chain → loudness normalization.
+def _apply_effect_chain(audio_out, sample_rate, effect_preset, *, skip_mastering=False,
+ trim_edges=False):
+ """Shared post-DSP for /generate: preset validation → edge trim →
+ mastering → effect chain → loudness normalization.
+
+ ``trim_edges`` is for a finished whole take only: streamed preview chunks
+ leave it off so they keep the same timing as the assembled take.
``skip_mastering`` honors a backend's ``applies_own_mastering`` flag
(issue #312): studio engines (e.g. VoxCPM2's native 48 kHz output)
@@ -606,6 +638,9 @@ def _apply_effect_chain(audio_out, sample_rate, effect_preset, *, skip_mastering
# Raw: skip all DSP — return raw model output
return audio_out
+ if trim_edges:
+ audio_out = _trim_output_edges(audio_out, sample_rate)
+
if not skip_mastering:
audio_out = apply_mastering(audio_out, sample_rate=sample_rate)
chain = get_effect_chain(preset)
@@ -1173,7 +1208,8 @@ def _run_inference(
# Per-span duration is left to the model; an explicit overall
# `duration` can't be meaningfully split across spans.
return _gen(span_text, None)[0]
- audio_out = _render_with_pauses(_gen_span, segments, sr)
+ audio_out = _render_with_pauses(_gen_span, segments, sr,
+ trim_spans=_wants_edge_trim(effect_preset))
else:
# Wave 1.2: long text is split at sentence boundaries and the
# per-chunk audio crossfaded — removes the length ceiling. Short
@@ -1203,7 +1239,12 @@ def _run_inference(
# Apply DSP effect preset. The VoiceStudio model never masters its own
# output, so mastering always runs here (unchanged behavior).
- return _apply_effect_chain(audio_out, sr, effect_preset)
+ # Pause renders were trimmed per span above; trimming the whole take
+ # would also eat a deliberate leading/trailing [pause].
+ return _apply_effect_chain(
+ audio_out, sr, effect_preset,
+ trim_edges=not has_pause and _wants_edge_trim(effect_preset, duration),
+ )
except ValueError as e:
# Don't wrap validation errors in OOM message
@@ -1276,7 +1317,8 @@ def _run_backend_inference(
span_kwargs["seed"] = used_seed
first_span = False
return trace_call("synthesis", backend.generate, span_text, duration=None, **span_kwargs)
- audio_out = _render_with_pauses(_gen_span, segments, sr)
+ audio_out = _render_with_pauses(_gen_span, segments, sr,
+ trim_spans=_wants_edge_trim(effect_preset))
else:
# Wave 1.2: sentence-boundary chunking for long text (see
# _run_inference for the rationale; behavior is identical here).
@@ -1310,6 +1352,7 @@ def _run_backend_inference(
return _apply_effect_chain(
audio_out, sr, effect_preset,
skip_mastering=getattr(backend, "applies_own_mastering", False),
+ trim_edges=not has_pause and _wants_edge_trim(effect_preset, duration),
)
except ValueError as e:
@@ -2213,7 +2256,8 @@ async def generate_speech(
sink=_dropped_sink)
skip = (getattr(_backend, "applies_own_mastering", False)
if _backend is not None else False)
- return _apply_effect_chain(audio_out, sr, effect_preset, skip_mastering=skip)
+ return _apply_effect_chain(audio_out, sr, effect_preset, skip_mastering=skip,
+ trim_edges=_wants_edge_trim(effect_preset))
except ValueError:
raise
except Exception as e:
diff --git a/backend/api/routers/openai_compat.py b/backend/api/routers/openai_compat.py
index eb805f5ee..a419f2477 100644
--- a/backend/api/routers/openai_compat.py
+++ b/backend/api/routers/openai_compat.py
@@ -583,10 +583,15 @@ def _typed_speech_http_error(e: Exception) -> Optional[HTTPException]:
def _run_tts(backend, text: str, kw: dict):
"""Run TTS inference in the GPU thread pool."""
- from services.audio_dsp import apply_mastering, normalize_audio
+ import torch
+ from services.audio_dsp import apply_mastering, normalize_audio, trim_speech_padding
from services.watermark import mark_synthetic
wav = backend.generate(text, **kw)
sr = backend.sample_rate
+ # Strip the engine's lead-in/tail silence, as /generate does — unless the
+ # caller asked for an exact `duration`, which a trim would shorten.
+ if not kw.get("duration") and torch.is_tensor(wav):
+ wav = trim_speech_padding(wav, sr)
# Engines that already emit mastered, studio-grade audio (e.g. VoxCPM2's
# native 48 kHz) opt out of apply_mastering via `applies_own_mastering`.
# That chain's highpass + Compressor is tuned for VoiceStudio's 24 kHz clone
diff --git a/tests/test_generate_edge_trim.py b/tests/test_generate_edge_trim.py
new file mode 100644
index 000000000..d17c19a53
--- /dev/null
+++ b/tests/test_generate_edge_trim.py
@@ -0,0 +1,90 @@
+"""Generated takes must not start or end with the engine's own silence padding.
+
+Engines pad every render with near-silence at both ends; only the dub path
+trimmed it, so every Generate / OpenAI-API take opened and closed on a blank.
+"""
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+import services.audio_dsp as dsp
+from api.routers import generation as gen
+from api.routers import openai_compat as oc
+
+SR = 1000 # 1 sample per ms keeps the arithmetic readable
+MARGIN = 50 # trim_speech_padding keeps 50 ms of context
+
+
+def _padded(lead=400, speech=1000, tail=600):
+ """[1, n] take: `lead` ms of engine silence, speech, `tail` ms of silence."""
+ return torch.cat([torch.zeros(1, lead), torch.full((1, speech), 0.5),
+ torch.zeros(1, tail)], dim=-1)
+
+
+def no_dsp(monkeypatch):
+ # Isolate the trim from mastering / pedalboard: both become identity.
+ monkeypatch.setattr(dsp, "apply_mastering", lambda a, sample_rate=None: a)
+ monkeypatch.setattr(dsp, "apply_effects_chain", lambda a, sample_rate, chain: a)
+
+
+def test_whole_take_trim_strips_engine_padding(no_dsp):
+ out = gen._apply_effect_chain(_padded(), SR, "broadcast", trim_edges=True)
+ assert out.shape[-1] == 1000 + 2 * MARGIN
+
+
+def test_preview_chunks_and_raw_keep_their_edges(no_dsp):
+ assert gen._apply_effect_chain(_padded(), SR, "broadcast").shape[-1] == 2000
+ assert gen._apply_effect_chain(_padded(), SR, "raw", trim_edges=True).shape[-1] == 2000
+
+
+def test_dead_render_is_not_shrunk(no_dsp):
+ silent = torch.zeros(1, 2000)
+ assert gen._apply_effect_chain(silent, SR, "broadcast", trim_edges=True).shape[-1] == 2000
+
+
[email protected]("preset,duration,expected", [
+ ("broadcast", None, True), (None, None, True), ("raw", None, False),
+ ("broadcast", 5.0, False),
+])
+def test_wants_edge_trim(preset, duration, expected):
+ assert gen._wants_edge_trim(preset, duration) is expected
+
+
+def test_pause_spans_trimmed_so_requested_pause_is_exact():
+ segs = [("", 300), ("hello", 500), ("world", 200)]
+ out = gen._render_with_pauses(lambda t: _padded(), segs, SR, trim_spans=True)
+ span = 1000 + 2 * MARGIN
+ # Deliberate leading and trailing pauses survive; engine padding does not.
+ assert out.shape[-1] == 300 + span + 500 + span + 200
+ assert torch.all(out[:, :300] == 0)
+ assert torch.all(out[:, -200:] == 0)
+
+
+def test_pause_spans_untrimmed_by_default():
+ out = gen._render_with_pauses(lambda t: _padded(), [("hi", 0)], SR)
+ assert out.shape[-1] == 2000
+
+
+class _Backend:
+ sample_rate = SR
+ applies_own_mastering = True
+
+ def generate(self, text, **kw):
+ return _padded()
+
+
+def openai_tts(monkeypatch):
+ import services.watermark as wm
+ monkeypatch.setattr(wm, "mark_synthetic", lambda w, sr, **k: w)
+ return lambda kw: oc._run_tts(_Backend(), "hi", kw)[0]
+
+
+def test_openai_speech_trims_edges(openai_tts):
+ assert openai_tts({}).shape[-1] == 1000 + 2 * MARGIN
+
+
+def test_openai_speech_keeps_explicit_duration(openai_tts):
+ assert openai_tts({"duration": 2.0}).shape[-1] == 2000