Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep transcribing past a damaged audio frame (#13757)
Nilaybe35c8d7cc6c
diff --git a/studio/backend/core/inference/audio_inputs.py b/studio/backend/core/inference/audio_inputs.pyindex f5a1e5b4b8e..565307a4b8c 100644--- a/studio/backend/core/inference/audio_inputs.py+++ b/studio/backend/core/inference/audio_inputs.py@@ -132,6 +132,15 @@ def _av_open(av, source: str):return av.open(source, mode = "r", format = None)+def _decoded_frames(av, container):+ for packet in container.demux(container.streams.audio[0]):+ try:+ frames = packet.decode()+ except av.error.InvalidDataError:+ continue+ yield from frames++def transcode(src: Path,dst: Path,@@ -177,7 +186,7 @@ def transcode(if not container.streams.audio:raise AudioInputError(400, "This file has no audio stream.")try:- for frame in container.decode(container.streams.audio[0]):+ for frame in _decoded_frames(av, container):if resampler is None:rate = rate or min(int(frame.sample_rate or 0), MAX_RATE)if rate <= 0:diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.pyindex aa5c15d0281..55b72ebb324 100644--- a/studio/backend/core/inference/stt_sidecar.py+++ b/studio/backend/core/inference/stt_sidecar.py@@ -1198,21 +1198,26 @@ def _decode_audio_bounded(audio: bytes, cancel_event = None):with _av_open(av, io.BytesIO(audio)) as container:if not container.streams.audio:raise SttAudioDecodeError("Could not decode the audio.")- frames = iter(container.decode(audio = 0))- while True:- try:- frame = next(frames)- except StopIteration:- break- except InvalidDataError:- continue- if cancel_event is not None and cancel_event.is_set():- raise SttTranscriptionCancelledError("Transcription cancelled.")- frame.pts = None- fifo.write(frame)- if fifo.samples >= 500000:- for resampled in resampler.resample(fifo.read()):- write_frame(resampled)+ try:+ for packet in container.demux(audio = 0):+ if cancel_event is not None and cancel_event.is_set():+ raise SttTranscriptionCancelledError("Transcription cancelled.")+ try:+ frames = packet.decode()+ except InvalidDataError:+ continue+ for frame in frames:+ frame.pts = None+ fifo.write(frame)+ if fifo.samples >= 500000:+ for resampled in resampler.resample(fifo.read()):+ write_frame(resampled)+ except (SttAudioTooLongError, SttTranscriptionCancelledError):+ raise+ except (FFmpegError, ValueError, RuntimeError) as exc:+ if not (sample_count or fifo.samples):+ raise+ logger.info("STT decode stopped early: %s", exc)if fifo.samples > 0:for resampled in resampler.resample(fifo.read()):write_frame(resampled)diff --git a/studio/backend/tests/test_audio_inputs.py b/studio/backend/tests/test_audio_inputs.pyindex 47cbaa36d43..5afe2d0b6bd 100644--- a/studio/backend/tests/test_audio_inputs.py+++ b/studio/backend/tests/test_audio_inputs.py@@ -154,6 +154,15 @@ def test_an_upload_decodes_dedupes_and_deletes(client, fmt, codec, rate, layout,assert not list(directory.iterdir())+def test_an_upload_with_a_damaged_frame_keeps_the_audio_after_it(client):+ data = bytearray(encode("mp3", "libmp3lame", 44100, "mono", 20.0))+ middle = len(data) // 2+ data[middle : middle + 2048] = bytes(range(256)) * 8+ response = _upload(client, bytes(data), "damaged.mp3")+ assert response.status_code == 201, response.text+ assert response.json()["duration_s"] > 19++def test_an_oversize_body_is_refused_while_streaming():seen: list[int] = []with pytest.raises(audio_inputs.AudioInputError) as refused:diff --git a/studio/backend/tests/test_stt_sidecar.py b/studio/backend/tests/test_stt_sidecar.pyindex b808c989e00..454ce8f4580 100644--- a/studio/backend/tests/test_stt_sidecar.py+++ b/studio/backend/tests/test_stt_sidecar.py@@ -1622,6 +1622,46 @@ def test_bounded_decoder_rejects_malformed_audio(audio):_REAL_DECODE_AUDIO_BOUNDED(audio)+def _mp3(av, rate, seconds):+ tone = (+ (0.3 * np.sin(np.arange(int(rate * seconds)) * 0.06) * 32767)+ .astype(np.int16)+ .reshape(1, -1)+ )+ buf = io.BytesIO()+ with av.open(buf, mode = "w", format = "mp3") as out:+ stream = out.add_stream("libmp3lame", rate = rate, layout = "mono")+ for start in range(0, tone.shape[1], 882):+ frame = av.AudioFrame.from_ndarray(+ np.ascontiguousarray(tone[:, start : start + 882]), format = "s16", layout = "mono"+ )+ frame.sample_rate, frame.pts = rate, start+ for packet in stream.encode(frame):+ out.mux(packet)+ for packet in stream.encode(None):+ out.mux(packet)+ return buf.getvalue()+++def test_a_damaged_frame_does_not_drop_the_rest_of_the_audio():+ av = pytest.importorskip("av")+ audio = bytearray(_mp3(av, 44100, 20))+ middle = len(audio) // 2+ audio[middle : middle + 2048] = bytes(range(256)) * 8++ decoded = _REAL_DECODE_AUDIO_BOUNDED(bytes(audio))++ assert len(decoded) / 16000 > 19+++def test_a_stream_that_cannot_continue_keeps_the_audio_before_it():+ av = pytest.importorskip("av")++ decoded = _REAL_DECODE_AUDIO_BOUNDED(_mp3(av, 44100, 10) + _mp3(av, 22050, 10))++ assert len(decoded) / 16000 > 9.5++def test_bounded_decoder_rejects_container_without_audio_stream(monkeypatch):class FakeFFmpegError(Exception):pass