The open-source AI voice studio. Clone, dictate, create.
fix(audio): preprocess reference samples instead of rejecting them (#502)
Jamie Pine115de231d08d
diff --git a/backend/tests/test_audio_preprocess.py b/backend/tests/test_audio_preprocess.pynew file mode 100644index 00000000..e52c44af--- /dev/null+++ b/backend/tests/test_audio_preprocess.py@@ -0,0 +1,112 @@+"""+Unit tests for reference-audio preprocessing.++Covers :func:`backend.utils.audio.preprocess_reference_audio` and+:func:`backend.utils.audio.validate_and_load_reference_audio`.+"""++import sys+from pathlib import Path++import numpy as np+import pytest+import soundfile as sf++sys.path.insert(0, str(Path(__file__).parent.parent))++from utils.audio import ( # noqa: E402+ preprocess_reference_audio,+ validate_and_load_reference_audio,+)+++SR = 24000+++def _tone(duration_s: float, amp: float = 0.3, freq: float = 220.0) -> np.ndarray:+ n = int(duration_s * SR)+ t = np.arange(n, dtype=np.float32) / SR+ return (amp * np.sin(2 * np.pi * freq * t)).astype(np.float32)+++def test_peak_cap_scales_hot_input():+ audio = _tone(3.0, amp=0.99)+ out = preprocess_reference_audio(audio, SR)+ assert np.abs(out).max() <= 0.951+++def test_peak_cap_leaves_moderate_input_untouched():+ audio = _tone(3.0, amp=0.5)+ out = preprocess_reference_audio(audio, SR)+ assert np.isclose(np.abs(out).max(), 0.5, atol=1e-3)+++def test_dc_offset_removed():+ audio = _tone(3.0, amp=0.3) + 0.1+ out = preprocess_reference_audio(audio, SR)+ assert abs(float(np.mean(out))) < 1e-3+++def test_silence_is_trimmed_with_padding_kept():+ silence = np.zeros(int(SR * 1.0), dtype=np.float32)+ speech = _tone(3.0, amp=0.3)+ audio = np.concatenate([silence, speech, silence])+ out = preprocess_reference_audio(audio, SR)+ # Most of the 2s of leading/trailing silence should be gone, but the+ # 3s of speech plus ~200ms of padding should remain.+ assert len(audio) - len(out) >= SR, "expected >=1s of silence trimmed"+ assert len(out) >= int(3.0 * SR), "speech body should be preserved"+++def test_clean_audio_is_not_padded_past_original_length():+ # Well-recorded audio with no edge silence shouldn't get longer after+ # preprocessing — otherwise a 29.9 s upload could be pushed past the+ # 30 s max_duration ceiling downstream.+ audio = _tone(3.0, amp=0.3)+ out = preprocess_reference_audio(audio, SR)+ assert len(out) <= len(audio)+++def test_empty_input_returns_empty():+ out = preprocess_reference_audio(np.zeros(0, dtype=np.float32), SR)+ assert out.size == 0+++def test_validate_accepts_previously_rejected_hot_file(tmp_path):+ audio = _tone(3.0, amp=0.995)+ path = tmp_path / "hot.wav"+ sf.write(str(path), audio, SR)++ ok, err, out_audio, out_sr = validate_and_load_reference_audio(str(path))++ assert ok, f"expected pass, got error: {err}"+ assert out_audio is not None+ assert out_sr == SR+ assert np.abs(out_audio).max() <= 0.951+++def test_validate_still_rejects_silent_input(tmp_path):+ audio = np.zeros(int(SR * 3.0), dtype=np.float32)+ path = tmp_path / "silent.wav"+ sf.write(str(path), audio, SR)++ ok, err, _, _ = validate_and_load_reference_audio(str(path))++ assert not ok+ assert err is not None+ assert "too short" in err.lower() or "quiet" in err.lower()+++def test_validate_rejects_too_short(tmp_path):+ audio = _tone(0.5, amp=0.3)+ path = tmp_path / "short.wav"+ sf.write(str(path), audio, SR)++ ok, err, _, _ = validate_and_load_reference_audio(str(path))++ assert not ok+ assert "too short" in (err or "").lower()+++if __name__ == "__main__":+ pytest.main([__file__, "-v"])diff --git a/backend/utils/audio.py b/backend/utils/audio.pyindex 2016992c..7e0fd6fd 100644--- a/backend/utils/audio.py+++ b/backend/utils/audio.py@@ -199,6 +199,66 @@ def trim_tts_output(return trimmed+def preprocess_reference_audio(+ audio: np.ndarray,+ sample_rate: int,+ peak_target: float = 0.95,+ trim_top_db: float = 40.0,+ edge_padding_ms: int = 100,+) -> np.ndarray:+ """+ Clean up a reference-audio sample before validation/storage.++ Removes DC offset, trims leading/trailing silence, and caps the peak so a+ slightly-hot recording doesn't get rejected downstream as "clipping". The+ goal is to accept reasonable real-world recordings — not to repair badly+ distorted ones. True clipping artifacts inside the waveform can't be+ recovered by peak scaling and will still sound bad.++ Args:+ audio: Mono audio array.+ sample_rate: Sample rate of ``audio`` in Hz.+ peak_target: Peak amplitude cap in [0, 1]. Applied only if the input+ peak exceeds this value.+ trim_top_db: Silence threshold for edge trimming, in dB below peak.+ 40 dB sits below normal speech dynamic range (≈30 dB) so soft+ trailing syllables are preserved, while still catching obvious+ leading/trailing silence. Lower values are more aggressive;+ librosa's own default is 60.+ edge_padding_ms: Milliseconds of padding to add back at each edge+ *only if* trimming shortened the waveform, so TTS engines have a+ brief silence to anchor on without ever making the output longer+ than the input.++ Returns:+ Preprocessed audio array (float32).+ """+ audio = audio.astype(np.float32, copy=False)++ if audio.size == 0:+ return audio++ audio = audio - float(np.mean(audio))++ trimmed, _ = librosa.effects.trim(audio, top_db=trim_top_db)+ if 0 < trimmed.size < audio.size:+ pad_each = int(sample_rate * edge_padding_ms / 1000)+ # Never pad past the original length — for near-max-duration uploads+ # an unconditional pad would push them over the 30 s ceiling and+ # trigger a spurious "too long" rejection.+ headroom = (audio.size - trimmed.size) // 2+ pad = min(pad_each, max(headroom, 0))+ if pad > 0:+ trimmed = np.pad(trimmed, (pad, pad), mode="constant")+ audio = trimmed++ peak = float(np.abs(audio).max())+ if peak > peak_target and peak > 0:+ audio = audio * (peak_target / peak)++ return audio++def validate_reference_audio(audio_path: str,min_duration: float = 2.0,@@ -207,13 +267,13 @@ def validate_reference_audio() -> Tuple[bool, Optional[str]]:"""Validate reference audio for voice cloning.-+Args:audio_path: Path to audio filemin_duration: Minimum duration in secondsmax_duration: Maximum duration in secondsmin_rms: Minimum RMS level-+Returns:Tuple of (is_valid, error_message)"""@@ -231,26 +291,28 @@ def validate_and_load_reference_audio() -> Tuple[bool, Optional[str], Optional[np.ndarray], Optional[int]]:"""Validate and load reference audio in a single pass.-++ Applies :func:`preprocess_reference_audio` before checks so that+ slightly-hot recordings aren't rejected as clipping. Duration and RMS+ checks run on the preprocessed waveform.+Returns:Tuple of (is_valid, error_message, audio_array, sample_rate)"""try:audio, sr = load_audio(audio_path)+ audio = preprocess_reference_audio(audio, sr)duration = len(audio) / sr-+if duration < min_duration:return False, f"Audio too short (minimum {min_duration} seconds)", None, Noneif duration > max_duration:return False, f"Audio too long (maximum {max_duration} seconds)", None, None-+rms = np.sqrt(np.mean(audio**2))if rms < min_rms:return False, "Audio is too quiet or silent", None, None-- if np.abs(audio).max() > 0.99:- return False, "Audio is clipping (reduce input gain)", None, None-+return True, None, audio, srexcept Exception as e:return False, f"Error validating audio: {str(e)}", None, None