Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep transcribing past a damaged audio frame (#13757)

Nilaybe35c8d7cc6c
diff --git a/studio/backend/core/inference/audio_inputs.py b/studio/backend/core/inference/audio_inputs.py
index f5a1e5b4b8e..565307a4b8c 100644
--- a/studio/backend/core/inference/audio_inputs.py
+++ b/studio/backend/core/inference/audio_inputs.py
@@ -132,6 +132,15 @@ def _av_open(av, source: str):
return av.open(source, mode = "r", format = None)
+def _decoded_frames(av, container):
+ for packet in container.demux(container.streams.audio[0]):
+ try:
+ frames = packet.decode()
+ except av.error.InvalidDataError:
+ continue
+ yield from frames
+
+
def transcode(
src: Path,
dst: Path,
@@ -177,7 +186,7 @@ def transcode(
if not container.streams.audio:
raise AudioInputError(400, "This file has no audio stream.")
try:
- for frame in container.decode(container.streams.audio[0]):
+ for frame in _decoded_frames(av, container):
if resampler is None:
rate = rate or min(int(frame.sample_rate or 0), MAX_RATE)
if rate <= 0:
diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.py
index aa5c15d0281..55b72ebb324 100644
--- a/studio/backend/core/inference/stt_sidecar.py
+++ b/studio/backend/core/inference/stt_sidecar.py
@@ -1198,21 +1198,26 @@ def _decode_audio_bounded(audio: bytes, cancel_event = None):
with _av_open(av, io.BytesIO(audio)) as container:
if not container.streams.audio:
raise SttAudioDecodeError("Could not decode the audio.")
- frames = iter(container.decode(audio = 0))
- while True:
- try:
- frame = next(frames)
- except StopIteration:
- break
- except InvalidDataError:
- continue
- if cancel_event is not None and cancel_event.is_set():
- raise SttTranscriptionCancelledError("Transcription cancelled.")
- frame.pts = None
- fifo.write(frame)
- if fifo.samples >= 500000:
- for resampled in resampler.resample(fifo.read()):
- write_frame(resampled)
+ try:
+ for packet in container.demux(audio = 0):
+ if cancel_event is not None and cancel_event.is_set():
+ raise SttTranscriptionCancelledError("Transcription cancelled.")
+ try:
+ frames = packet.decode()
+ except InvalidDataError:
+ continue
+ for frame in frames:
+ frame.pts = None
+ fifo.write(frame)
+ if fifo.samples >= 500000:
+ for resampled in resampler.resample(fifo.read()):
+ write_frame(resampled)
+ except (SttAudioTooLongError, SttTranscriptionCancelledError):
+ raise
+ except (FFmpegError, ValueError, RuntimeError) as exc:
+ if not (sample_count or fifo.samples):
+ raise
+ logger.info("STT decode stopped early: %s", exc)
if fifo.samples > 0:
for resampled in resampler.resample(fifo.read()):
write_frame(resampled)
diff --git a/studio/backend/tests/test_audio_inputs.py b/studio/backend/tests/test_audio_inputs.py
index 47cbaa36d43..5afe2d0b6bd 100644
--- a/studio/backend/tests/test_audio_inputs.py
+++ b/studio/backend/tests/test_audio_inputs.py
@@ -154,6 +154,15 @@ def test_an_upload_decodes_dedupes_and_deletes(client, fmt, codec, rate, layout,
assert not list(directory.iterdir())
+def test_an_upload_with_a_damaged_frame_keeps_the_audio_after_it(client):
+ data = bytearray(encode("mp3", "libmp3lame", 44100, "mono", 20.0))
+ middle = len(data) // 2
+ data[middle : middle + 2048] = bytes(range(256)) * 8
+ response = _upload(client, bytes(data), "damaged.mp3")
+ assert response.status_code == 201, response.text
+ assert response.json()["duration_s"] > 19
+
+
def test_an_oversize_body_is_refused_while_streaming():
seen: list[int] = []
with pytest.raises(audio_inputs.AudioInputError) as refused:
diff --git a/studio/backend/tests/test_stt_sidecar.py b/studio/backend/tests/test_stt_sidecar.py
index b808c989e00..454ce8f4580 100644
--- a/studio/backend/tests/test_stt_sidecar.py
+++ b/studio/backend/tests/test_stt_sidecar.py
@@ -1622,6 +1622,46 @@ def test_bounded_decoder_rejects_malformed_audio(audio):
_REAL_DECODE_AUDIO_BOUNDED(audio)
+def _mp3(av, rate, seconds):
+ tone = (
+ (0.3 * np.sin(np.arange(int(rate * seconds)) * 0.06) * 32767)
+ .astype(np.int16)
+ .reshape(1, -1)
+ )
+ buf = io.BytesIO()
+ with av.open(buf, mode = "w", format = "mp3") as out:
+ stream = out.add_stream("libmp3lame", rate = rate, layout = "mono")
+ for start in range(0, tone.shape[1], 882):
+ frame = av.AudioFrame.from_ndarray(
+ np.ascontiguousarray(tone[:, start : start + 882]), format = "s16", layout = "mono"
+ )
+ frame.sample_rate, frame.pts = rate, start
+ for packet in stream.encode(frame):
+ out.mux(packet)
+ for packet in stream.encode(None):
+ out.mux(packet)
+ return buf.getvalue()
+
+
+def test_a_damaged_frame_does_not_drop_the_rest_of_the_audio():
+ av = pytest.importorskip("av")
+ audio = bytearray(_mp3(av, 44100, 20))
+ middle = len(audio) // 2
+ audio[middle : middle + 2048] = bytes(range(256)) * 8
+
+ decoded = _REAL_DECODE_AUDIO_BOUNDED(bytes(audio))
+
+ assert len(decoded) / 16000 > 19
+
+
+def test_a_stream_that_cannot_continue_keeps_the_audio_before_it():
+ av = pytest.importorskip("av")
+
+ decoded = _REAL_DECODE_AUDIO_BOUNDED(_mp3(av, 44100, 10) + _mp3(av, 22050, 10))
+
+ assert len(decoded) / 16000 > 9.5
+
+
def test_bounded_decoder_rejects_container_without_audio_stream(monkeypatch):
class FakeFFmpegError(Exception):
pass