Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""POST /audio/transcribe/source with stub engines; the account boundary is real."""
from __future__ import annotations
import asyncio
import io
import json
import sys
import wave
from pathlib import Path
import pytest
from core.inference import audio_gallery, audio_inputs, transcript_gallery
from core.inference.audio_cpp_models import AUDIO_CPP_REPO as REPO
from routes import inference
from utils.account_context import run_as
sys.path.insert(0, str(Path(__file__).resolve().parent))
from test_account_media_isolation import ALICE, _client, isolated # noqa: E402, F401
from test_audio_cpp_models import hub # noqa: E402, F401
from test_audio_inputs import _chunks, encode, wav_bytes # noqa: E402
MOSS = f"{REPO}/MOSS-Transcribe-Diarize-GGUF"
QWEN3 = f"{REPO}/Qwen3-ASR-0.6B-GGUF"
SEGMENTS = [
{"start": 0.12, "end": 1.0, "text": "Hello there.", "speaker": "S01"},
{"start": 1.1, "end": 2.0, "text": "General Kenobi.", "speaker": "S02"},
]
class _Sidecar:
loaded_model = None
def __init__(self):
self.result = {"text": "Hello there. General Kenobi.", "language": None, "duration": 2.0}
self.result.update(segments = SEGMENTS, speakers = ["S01", "S02"])
self.paths, self.bytes, self.events, self.loads = [], [], [], []
def ensure_aligner(
self,
model,
on_phase,
cancel_event = None,
):
self.events.append(("aligner", model))
self.aligner_cancel = cancel_event
def transcribe_path(self, path, model, language, *, timestamps, cancel_event, on_phase):
on_phase("transcribing")
with wave.open(str(path)) as w:
self.paths.append((Path(path), w.getframerate(), timestamps, language))
return {**self.result, "model": model}
def transcribe(self, raw, model, language, *_args, **_kwargs):
self.bytes.append(raw)
return {"text": "plain words", "language": language, "duration": 1.0, "model": model}
def cancel_transcription(self, cancel_event):
cancel_event.set()
return True
@pytest.fixture
def stub(monkeypatch, hub):
sidecar = _Sidecar()
def load(
model,
engine,
*_args,
timestamps = False,
on_phase = None,
**_kwargs,
):
sidecar.loads.append((model, engine))
sidecar.events.append(("load", timestamps))
if on_phase is not None:
on_phase("loading")
monkeypatch.setattr(inference, "_stt_lifecycle", lambda: (load, lambda *a, **k: []))
monkeypatch.setattr(inference, "_stt_sidecar_for", lambda engine: sidecar)
monkeypatch.setattr(inference.account_access, "require_model_access", lambda *a, **k: None)
return sidecar
def _input(account) -> str:
data = encode("webm", "libopus", 48000, "stereo", 1.0)
save = audio_inputs.save_stream(_chunks([data]), "meeting.webm")
return run_as(account, lambda: asyncio.run(save)[0]["id"])
def _post(client, **body):
body = {"model": MOSS, "engine": "audiocpp", **body}
response = client.post("/api/inference/audio/transcribe/source", json = body)
return response, [json.loads(line) for line in response.text.splitlines() if line.strip()]
@pytest.mark.parametrize(
"body",
[
{"source": {"input_id": "a" * 32}, "path": "/etc/passwd"},
{"source": {"path": "/etc/passwd"}},
{"source": {"input_id": "a" * 32, "clip_id": "b" * 32}},
{"source": {}},
{"source": {"input_id": "../../etc/passwd"}},
{"source": {"input_id": "a" * 32}, "title": "x" * 256},
{"source": {"input_id": "a" * 32}, "device": "tpu"},
],
)
def test_client_paths_and_unknown_fields_are_422(stub, body):
with _client(ALICE) as client:
assert _post(client, **body)[0].status_code == 422
assert not stub.paths and not stub.bytes
def test_audio_cpp_reads_a_16k_copy_in_the_account_and_the_record_names_the_source(stub, tmp_path):
input_id = _input(ALICE)
with _client(ALICE) as client:
response, events = _post(
client, source = {"input_id": input_id}, speakers = True, language = "en"
)
((path, rate, timestamps, language),) = stub.paths
assert path.parent == tmp_path / "accounts" / ALICE.account_id / "audio" / "inputs"
assert path.name.startswith(f"{input_id}.16000.mono") and rate == 16000
assert (timestamps, language) == (False, "en") and not stub.bytes
assert [e["phase"] for e in events if "phase" in e] == ["loading", "transcribing"]
complete = events[-1]
labels = [{"id": "S01", "label": "Speaker 1"}, {"id": "S02", "label": "Speaker 2"}]
assert (complete["speakers"], complete["segments"]) == (labels, SEGMENTS)
assert complete["source"] == {"kind": "input", "id": input_id, "name": "meeting.webm"}
assert complete["timestamps"] is True
record = complete["record"]
assert record["title"] == "meeting.webm" and record["source"] == complete["source"]
saved = client.get(f"/api/inference/audio/transcripts/{record['id']}").json()
assert (saved["segments"], saved["speakers"]) == (SEGMENTS, labels)
stored = run_as(
ALICE, lambda: (transcript_gallery.gallery_dir() / f"{record['id']}.json").read_text()
)
for text in (response.text, stored):
assert str(tmp_path) not in text and "/inputs/" not in text
def test_vibevoice_sources_are_prepared_at_24k_and_speakers_off_strips_them(stub):
meta = {"prompt": "p", "model": "m", "audio_type": "audiocpp_tts", "sample_rate": 24000}
meta.update(duration_s = 1.0, created_at = "2026-10-02T00:00:00Z")
clip_id = run_as(ALICE, audio_gallery.save, wav_bytes(1.0, 24000), meta)["id"]
with _client(ALICE) as client:
_, events = _post(client, model = f"{REPO}/VibeVoice-ASR-GGUF", source = {"clip_id": clip_id})
((path, rate, _, _),) = stub.paths
assert rate == 24000 and path.name.startswith(f"c-{clip_id}.24000.mono")
complete = events[-1]
assert "speakers" not in complete and "speakers" not in complete["record"]
assert all("speaker" not in s for s in complete["segments"])
def test_qwen3_timestamps_are_asked_of_the_sidecar(stub):
stub.result = {"text": "Concord returned.", "language": "English", "duration": 3.5}
source = {"input_id": _input(ALICE)}
with _client(ALICE) as client:
complete = _post(client, model = QWEN3, source = source, timestamps = True)[1][-1]
assert stub.paths[0][2] is True
# The aligner is fetched first and the server starts with it: one load, not two.
assert stub.events == [("aligner", QWEN3), ("load", True)]
assert stub.aligner_cancel is not None
assert complete["timestamps"] is False and "segments" not in complete
def test_another_engine_gets_the_prepared_bytes_past_the_encoded_upload_cap(stub, monkeypatch):
# 30 minutes of 16 kHz PCM is ~58 MB, past the 25 MB cap for encoded uploads.
monkeypatch.setattr(inference, "_MAX_AUDIO_RAW_BYTES", 1024)
source = {"input_id": _input(ALICE)}
with _client(ALICE) as client:
response, events = _post(client, model = "small", engine = "transformers", source = source)
assert response.status_code == 200, response.text
(raw,) = stub.bytes
assert len(raw) > 1024 and not stub.paths and stub.loads == [("small", "transformers")]
with wave.open(io.BytesIO(raw)) as w:
assert (w.getframerate(), w.getnchannels()) == (16000, 1)
assert (events[-1]["text"], events[-1]["timestamps"]) == ("plain words", False)
@pytest.mark.parametrize(
"model,engine,flag",
[
("small", "transformers", "timestamps"),
("qwen3-asr-0.6b", "mtmd", "timestamps"),
(f"{REPO}/Nemotron-3.5-ASR-Streaming-0.6B-GGUF", "audiocpp", "timestamps"),
(QWEN3, "audiocpp", "speakers"),
("small", "transformers", "speakers"),
],
)
def test_asking_an_unsupported_model_for_timestamps_or_speakers_is_422(stub, model, engine, flag):
body = {"model": model, "engine": engine, "source": {"input_id": _input(ALICE)}, flag: True}
with _client(ALICE) as client:
response, _ = _post(client, **body)
assert response.status_code == 422 and not stub.paths and not stub.bytes
expected = "cannot add timestamps" if flag == "timestamps" else "cannot tell speakers apart"
assert expected in response.json()["detail"]
def test_capabilities_route_answers_for_any_model(stub):
url = "/api/inference/audio/stt/capabilities"
with _client(ALICE) as client:
assert client.get(url, params = {"model": MOSS}).json()["speakers"] is True
unknown = client.get(url, params = {"model": "who/knows", "engine": "bogus"})
assert unknown.status_code == 200 and unknown.json()["timestamps"] == "unsupported"
def test_transcript_routes_get_rename_and_archive(stub):
with _client(ALICE) as client:
record = _post(client, source = {"input_id": _input(ALICE)}, speakers = True)[1][-1]["record"]
url = f"/api/inference/audio/transcripts/{record['id']}"
renamed = client.patch(url, json = {"speaker_names": {"S01": "Alice"}}).json()
assert renamed["speaker_names"] == {"S01": "Alice"} and "segments" not in renamed
assert client.get(url).json()["speaker_names"] == {"S01": "Alice"}
for bad in ({"S09": "Ghost"}, {"S01": "x" * 41}):
assert client.patch(url, json = {"speaker_names": bad}).status_code == 422
for bad in ({"pinned": True}, {}):
assert client.patch(url, json = bad).status_code == 422
cleared = client.patch(url, json = {"speaker_names": {"S01": None}, "archived": True}).json()
assert "speaker_names" not in cleared and cleared["archived"] is True
assert client.get("/api/inference/audio/transcripts/" + "e" * 32).status_code == 404
@pytest.fixture
def connection(monkeypatch):
import httpx
from core.inference import external_provider, providers
config = {
"provider_type": "custom",
"display_name": "Whisper Box",
"base_url": "http://stt.local:8000/v1",
"is_enabled": True,
}
monkeypatch.setattr(
inference.providers_db,
"get_provider",
lambda pid: dict(config) if pid == "conn-1" else None,
)
for module in (inference, providers):
monkeypatch.setattr(module, "validate_provider_base_url", lambda url: url)
monkeypatch.setattr(inference, "resolve_provider_api_key_or_400", lambda *a, **k: "sk-test")
calls = []
reply = {
"task": "transcribe",
"language": "english",
"duration": 1.0,
"text": " Hello there.",
"segments": [{"id": 0, "seek": 0, "start": 0.0, "end": 1.0, "text": " Hello there."}],
"words": [{"word": " Hello", "start": 0.0, "end": 0.4}],
}
class _Http:
async def post(self, url, **kwargs):
calls.append({"url": url, **kwargs})
return httpx.Response(200, json = reply, request = httpx.Request("POST", url))
monkeypatch.setattr(external_provider, "_client", lambda: _Http())
return calls
def test_a_connection_transcribes_the_source_and_saves_it(stub, connection):
input_id = _input(ALICE)
with _client(ALICE) as client:
response, events = _post(
client,
provider_id = "conn-1",
model = "whisper-1",
source = {"input_id": input_id},
timestamps = True,
)
assert response.status_code == 200, response.text
assert not (stub.paths or stub.bytes or stub.loads)
(call,) = connection
assert call["url"] == "http://stt.local:8000/v1/audio/transcriptions"
name, audio, content_type = call["files"]["file"]
assert (name, content_type, audio[:4]) == ("audio.flac", "audio/flac", b"fLaC")
assert call["data"]["model"] == "whisper-1"
assert call["data"]["response_format"] == "verbose_json"
assert call["data"]["timestamp_granularities[]"] == ["segment", "word"]
complete = events[-1]
assert complete["type"] == "complete" and complete["text"] == "Hello there."
assert complete["segments"] == [{"start": 0.0, "end": 1.0, "text": "Hello there."}]
assert complete["words"] == [{"start": 0.0, "end": 0.4, "word": "Hello"}]
assert complete["timestamps"] is True and complete["duration"] == pytest.approx(1.0, abs = 0.05)
record = complete["record"]
assert (record["model"], record["language"]) == ("whisper-1", "english")
assert record["source"] == {"kind": "input", "id": input_id, "name": "meeting.webm"}
def test_a_source_past_the_connection_limit_is_refused_before_upload(stub, connection, monkeypatch):
monkeypatch.setattr(inference, "_MAX_AUDIO_RAW_BYTES", 1024)
with _client(ALICE) as client:
_, events = _post(
client, provider_id = "conn-1", model = "whisper-1", source = {"input_id": _input(ALICE)}
)
assert events[-1]["type"] == "error"
assert "a connection takes" in events[-1]["message"]
assert not connection and not stub.bytes
def test_mtmd_source_keeps_its_context_preference_through_streaming(stub, monkeypatch):
calls = []
original = stub.transcribe
def transcribe(*args, **kwargs):
calls.append(kwargs)
return original(*args, **kwargs)
monkeypatch.setattr(stub, "transcribe", transcribe)
with _client(ALICE) as client:
response, events = _post(
client,
model = "qwen3-asr-0.6b",
engine = "mtmd",
source = {"input_id": _input(ALICE)},
context_length = 8192,
)
assert response.status_code == 200, response.text
assert events[-1]["type"] == "complete", events
assert calls[0]["context_length"] == 8192
def _long_input(account, seconds = 3.0) -> str:
save = audio_inputs.save_stream(_chunks([wav_bytes(seconds)]), "long.wav")
return run_as(account, lambda: asyncio.run(save)[0]["id"])
@pytest.fixture
def short_caps(monkeypatch):
from core.inference import stt_sidecar
monkeypatch.delenv(audio_inputs.TRANSCRIBE_MAX_MINUTES_ENV, raising = False)
monkeypatch.setattr(audio_inputs, "MAX_SECONDS", 1)
monkeypatch.setattr(stt_sidecar, "_MAX_AUDIO_SECONDS", 1)
def test_audio_cpp_reads_the_whole_source_past_the_other_engines_cap(stub, short_caps):
input_id = _long_input(ALICE)
with _client(ALICE) as client:
response, events = _post(client, source = {"input_id": input_id})
assert response.status_code == 200, response.text
((path, rate, _, _),) = stub.paths
assert path.name == f"{input_id}.16000.mono.m14400.wav"
with wave.open(str(path)) as w:
assert w.getnframes() / w.getframerate() == pytest.approx(3.0, abs = 0.05)
assert events[-1]["type"] == "complete"
def test_another_engine_refuses_a_source_past_its_cap_before_loading(stub, short_caps):
source = {"input_id": _long_input(ALICE)}
with _client(ALICE) as client:
response, _ = _post(client, model = "small", engine = "transformers", source = source)
assert response.status_code == 413
assert response.json()["detail"] == (
"This model transcribes up to 0 minutes of audio. Trim it, or pick an audio.cpp model, "
"which takes up to 4 hours."
)
assert not (stub.loads or stub.bytes or stub.paths)
def test_a_source_just_past_the_cap_is_refused_not_cut(stub, short_caps):
source = {"input_id": _long_input(ALICE, seconds = 1.5)}
with _client(ALICE) as client:
response, _ = _post(client, model = "small", engine = "transformers", source = source)
assert response.status_code == 413
assert not (stub.loads or stub.bytes or stub.paths)
def test_a_connection_refuses_a_source_past_its_cap_before_upload(stub, connection, short_caps):
source = {"input_id": _long_input(ALICE)}
with _client(ALICE) as client:
response, _ = _post(client, provider_id = "conn-1", model = "whisper-1", source = source)
assert response.status_code == 413
assert response.json()["detail"].startswith("A connection takes up to 0 minutes of audio.")
assert connection == [] and not (stub.loads or stub.bytes)