Skip to content

The open-source AI voice studio. Clone, dictate, create.

fix(audio): preprocess reference samples instead of rejecting them (#502)

Jamie Pine115de231d08d
diff --git a/backend/tests/test_audio_preprocess.py b/backend/tests/test_audio_preprocess.py
new file mode 100644
index 00000000..e52c44af
--- /dev/null
+++ b/backend/tests/test_audio_preprocess.py
@@ -0,0 +1,112 @@
+"""
+Unit tests for reference-audio preprocessing.
+
+Covers :func:`backend.utils.audio.preprocess_reference_audio` and
+:func:`backend.utils.audio.validate_and_load_reference_audio`.
+"""
+
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+import soundfile as sf
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from utils.audio import ( # noqa: E402
+ preprocess_reference_audio,
+ validate_and_load_reference_audio,
+)
+
+
+SR = 24000
+
+
+def _tone(duration_s: float, amp: float = 0.3, freq: float = 220.0) -> np.ndarray:
+ n = int(duration_s * SR)
+ t = np.arange(n, dtype=np.float32) / SR
+ return (amp * np.sin(2 * np.pi * freq * t)).astype(np.float32)
+
+
+def test_peak_cap_scales_hot_input():
+ audio = _tone(3.0, amp=0.99)
+ out = preprocess_reference_audio(audio, SR)
+ assert np.abs(out).max() <= 0.951
+
+
+def test_peak_cap_leaves_moderate_input_untouched():
+ audio = _tone(3.0, amp=0.5)
+ out = preprocess_reference_audio(audio, SR)
+ assert np.isclose(np.abs(out).max(), 0.5, atol=1e-3)
+
+
+def test_dc_offset_removed():
+ audio = _tone(3.0, amp=0.3) + 0.1
+ out = preprocess_reference_audio(audio, SR)
+ assert abs(float(np.mean(out))) < 1e-3
+
+
+def test_silence_is_trimmed_with_padding_kept():
+ silence = np.zeros(int(SR * 1.0), dtype=np.float32)
+ speech = _tone(3.0, amp=0.3)
+ audio = np.concatenate([silence, speech, silence])
+ out = preprocess_reference_audio(audio, SR)
+ # Most of the 2s of leading/trailing silence should be gone, but the
+ # 3s of speech plus ~200ms of padding should remain.
+ assert len(audio) - len(out) >= SR, "expected >=1s of silence trimmed"
+ assert len(out) >= int(3.0 * SR), "speech body should be preserved"
+
+
+def test_clean_audio_is_not_padded_past_original_length():
+ # Well-recorded audio with no edge silence shouldn't get longer after
+ # preprocessing — otherwise a 29.9 s upload could be pushed past the
+ # 30 s max_duration ceiling downstream.
+ audio = _tone(3.0, amp=0.3)
+ out = preprocess_reference_audio(audio, SR)
+ assert len(out) <= len(audio)
+
+
+def test_empty_input_returns_empty():
+ out = preprocess_reference_audio(np.zeros(0, dtype=np.float32), SR)
+ assert out.size == 0
+
+
+def test_validate_accepts_previously_rejected_hot_file(tmp_path):
+ audio = _tone(3.0, amp=0.995)
+ path = tmp_path / "hot.wav"
+ sf.write(str(path), audio, SR)
+
+ ok, err, out_audio, out_sr = validate_and_load_reference_audio(str(path))
+
+ assert ok, f"expected pass, got error: {err}"
+ assert out_audio is not None
+ assert out_sr == SR
+ assert np.abs(out_audio).max() <= 0.951
+
+
+def test_validate_still_rejects_silent_input(tmp_path):
+ audio = np.zeros(int(SR * 3.0), dtype=np.float32)
+ path = tmp_path / "silent.wav"
+ sf.write(str(path), audio, SR)
+
+ ok, err, _, _ = validate_and_load_reference_audio(str(path))
+
+ assert not ok
+ assert err is not None
+ assert "too short" in err.lower() or "quiet" in err.lower()
+
+
+def test_validate_rejects_too_short(tmp_path):
+ audio = _tone(0.5, amp=0.3)
+ path = tmp_path / "short.wav"
+ sf.write(str(path), audio, SR)
+
+ ok, err, _, _ = validate_and_load_reference_audio(str(path))
+
+ assert not ok
+ assert "too short" in (err or "").lower()
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/backend/utils/audio.py b/backend/utils/audio.py
index 2016992c..7e0fd6fd 100644
--- a/backend/utils/audio.py
+++ b/backend/utils/audio.py
@@ -199,6 +199,66 @@ def trim_tts_output(
return trimmed
+def preprocess_reference_audio(
+ audio: np.ndarray,
+ sample_rate: int,
+ peak_target: float = 0.95,
+ trim_top_db: float = 40.0,
+ edge_padding_ms: int = 100,
+) -> np.ndarray:
+ """
+ Clean up a reference-audio sample before validation/storage.
+
+ Removes DC offset, trims leading/trailing silence, and caps the peak so a
+ slightly-hot recording doesn't get rejected downstream as "clipping". The
+ goal is to accept reasonable real-world recordings — not to repair badly
+ distorted ones. True clipping artifacts inside the waveform can't be
+ recovered by peak scaling and will still sound bad.
+
+ Args:
+ audio: Mono audio array.
+ sample_rate: Sample rate of ``audio`` in Hz.
+ peak_target: Peak amplitude cap in [0, 1]. Applied only if the input
+ peak exceeds this value.
+ trim_top_db: Silence threshold for edge trimming, in dB below peak.
+ 40 dB sits below normal speech dynamic range (≈30 dB) so soft
+ trailing syllables are preserved, while still catching obvious
+ leading/trailing silence. Lower values are more aggressive;
+ librosa's own default is 60.
+ edge_padding_ms: Milliseconds of padding to add back at each edge
+ *only if* trimming shortened the waveform, so TTS engines have a
+ brief silence to anchor on without ever making the output longer
+ than the input.
+
+ Returns:
+ Preprocessed audio array (float32).
+ """
+ audio = audio.astype(np.float32, copy=False)
+
+ if audio.size == 0:
+ return audio
+
+ audio = audio - float(np.mean(audio))
+
+ trimmed, _ = librosa.effects.trim(audio, top_db=trim_top_db)
+ if 0 < trimmed.size < audio.size:
+ pad_each = int(sample_rate * edge_padding_ms / 1000)
+ # Never pad past the original length — for near-max-duration uploads
+ # an unconditional pad would push them over the 30 s ceiling and
+ # trigger a spurious "too long" rejection.
+ headroom = (audio.size - trimmed.size) // 2
+ pad = min(pad_each, max(headroom, 0))
+ if pad > 0:
+ trimmed = np.pad(trimmed, (pad, pad), mode="constant")
+ audio = trimmed
+
+ peak = float(np.abs(audio).max())
+ if peak > peak_target and peak > 0:
+ audio = audio * (peak_target / peak)
+
+ return audio
+
+
def validate_reference_audio(
audio_path: str,
min_duration: float = 2.0,
@@ -207,13 +267,13 @@ def validate_reference_audio(
) -> Tuple[bool, Optional[str]]:
"""
Validate reference audio for voice cloning.
-
+
Args:
audio_path: Path to audio file
min_duration: Minimum duration in seconds
max_duration: Maximum duration in seconds
min_rms: Minimum RMS level
-
+
Returns:
Tuple of (is_valid, error_message)
"""
@@ -231,26 +291,28 @@ def validate_and_load_reference_audio(
) -> Tuple[bool, Optional[str], Optional[np.ndarray], Optional[int]]:
"""
Validate and load reference audio in a single pass.
-
+
+ Applies :func:`preprocess_reference_audio` before checks so that
+ slightly-hot recordings aren't rejected as clipping. Duration and RMS
+ checks run on the preprocessed waveform.
+
Returns:
Tuple of (is_valid, error_message, audio_array, sample_rate)
"""
try:
audio, sr = load_audio(audio_path)
+ audio = preprocess_reference_audio(audio, sr)
duration = len(audio) / sr
-
+
if duration < min_duration:
return False, f"Audio too short (minimum {min_duration} seconds)", None, None
if duration > max_duration:
return False, f"Audio too long (maximum {max_duration} seconds)", None, None
-
+
rms = np.sqrt(np.mean(audio**2))
if rms < min_rms:
return False, "Audio is too quiet or silent", None, None
-
- if np.abs(audio).max() > 0.99:
- return False, "Audio is clipping (reduce input gain)", None, None
-
+
return True, None, audio, sr
except Exception as e:
return False, f"Error validating audio: {str(e)}", None, None