Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Regression coverage for audio context allocation and incomplete transcriptions."""
import json
import threading
from types import SimpleNamespace
import pytest
from core.inference import stt_mtmd_sidecar as mtmd
from core.inference.stt_sidecar import SttContextSizeError, SttIncompleteTranscriptionError
MODEL = "qwen3-asr-0.6b"
class _Process:
_next_pid = 81000
def __init__(self, command):
type(self)._next_pid += 1
self.pid = type(self)._next_pid
self.command = list(command)
self.returncode = None
def poll(self):
return self.returncode
def terminate(self):
self.returncode = -15
def kill(self):
self.returncode = -9
def wait(self, timeout = None):
return self.returncode
class _Wire:
"""A local server protocol double, including the separate audio token-count endpoint."""
def __init__(self):
self.requests = []
self.input_tokens = 200
self.count_status = 200
self.finish_reason = "stop"
self.before_count = None
self.before_generate = None
self.context_by_port = {}
def connection(self, *args, **kwargs):
wire = self
port = args[1]
class Connection:
sock = None
def request(
self,
method,
path,
body = None,
headers = None,
):
self.path = path
self.payload = json.loads(body) if body is not None else None
wire.requests.append((path, self.payload, headers))
def getresponse(self):
if self.path == "/props":
return Response(
200, {"default_generation_settings": {"n_ctx": wire.context_by_port[port]}}
)
is_count = self.path.endswith("/input_tokens")
callback = wire.before_count if is_count else wire.before_generate
if callback is not None:
callback()
if is_count:
body = (
{"input_tokens": wire.input_tokens, "object": "response.input_tokens"}
if wire.count_status == 200
else {"error": {"message": "not found"}}
)
return Response(wire.count_status, body)
return Response(
200,
{
"choices": [
{
"message": {
"content": "language English<asr_text>Complete sentence."
},
"finish_reason": wire.finish_reason,
}
]
},
)
def close(self):
pass
class Response:
def __init__(self, status, body):
self.status, self.body = status, body
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self):
return json.dumps(self.body).encode()
def __iter__(self):
yield b'data: {"choices":[{"delta":{"content":"language English<asr_text>Complete sentence."},"finish_reason":null}]}\n'
event = {"choices": [{"delta": {}, "finish_reason": wire.finish_reason}]}
yield b"data: " + json.dumps(event).encode() + b"\n"
yield b"data: [DONE]\n"
return Connection()
def count_requests(self):
return [request for request in self.requests if request[0].endswith("/input_tokens")]
def generation_requests(self):
return [request for request in self.requests if request[0] == "/v1/chat/completions"]
@pytest.fixture
def runtime(monkeypatch):
from core.inference import llama_cpp
wire, processes = _Wire(), []
def spawn(command, **kwargs):
process = _Process(command)
processes.append(process)
wire.context_by_port[int(command[command.index("--port") + 1])] = _context(process)
return process
monkeypatch.setattr(mtmd.subprocess, "Popen", spawn)
monkeypatch.setattr(mtmd, "adopt_pid", lambda _pid: None)
monkeypatch.setattr(mtmd, "forget_pid", lambda _pid: None)
monkeypatch.setattr(mtmd, "ensure_engine_available", lambda: "/fake/llama-server")
monkeypatch.setattr(mtmd, "_llama_server_child_env", lambda _binary: {})
monkeypatch.setattr(mtmd, "_training_active", lambda: False)
monkeypatch.setattr(llama_cpp, "_llama_server_api_key_enabled", lambda: False)
monkeypatch.setattr(
mtmd.MtmdSttSidecar, "_ensure_model_downloaded", lambda *_args: ("m.gguf", "p.gguf")
)
monkeypatch.setattr(
mtmd.MtmdSttSidecar, "_wait_for_server", staticmethod(lambda *_args, **_kwargs: True)
)
monkeypatch.setattr(mtmd, "_decode_audio_bounded", lambda *_args: b"x" * 16000)
monkeypatch.setattr(mtmd, "_pcm_to_wav_bytes", lambda _audio: b"RIFFwav")
monkeypatch.setattr(mtmd.http.client, "HTTPConnection", wire.connection)
sidecar = mtmd.MtmdSttSidecar(keep_alive_seconds = 0)
yield SimpleNamespace(sidecar = sidecar, wire = wire, processes = processes)
sidecar.unload(wait = False)
def _context(process):
return int(process.command[process.command.index("--ctx-size") + 1])
@pytest.mark.parametrize("stream", [False, True])
def test_length_finish_never_reports_a_successful_transcript(monkeypatch, stream):
"""A server can return HTTP 200 and useful partial text despite exhausting the token budget."""
wire = _Wire()
wire.finish_reason = "length"
monkeypatch.setattr(mtmd.http.client, "HTTPConnection", wire.connection)
sidecar = mtmd.MtmdSttSidecar(keep_alive_seconds = 0)
progress = []
with pytest.raises(
SttIncompleteTranscriptionError, match = "(?i)context|limit|incomplete|truncat"
):
sidecar._post_transcribe(
12345,
MODEL,
b"RIFFwav",
1.0,
on_progress = progress.append if stream else None,
)
@pytest.mark.parametrize("stream", [False, True])
def test_normal_stop_still_returns_complete_transcript(monkeypatch, stream):
wire = _Wire()
monkeypatch.setattr(mtmd.http.client, "HTTPConnection", wire.connection)
progress = []
assert (
mtmd.MtmdSttSidecar(keep_alive_seconds = 0)._post_transcribe(
12345,
MODEL,
b"RIFFwav",
1.0,
on_progress = progress.append if stream else None,
)
== "Complete sentence."
)
@pytest.mark.parametrize(
"input_tokens, expected_context", [(200, 4096), (3520, 4096), (3521, 8192)]
)
def test_auto_reserves_audio_output_and_margin(runtime, input_tokens, expected_context):
runtime.wire.input_tokens = input_tokens
result = runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert result["text"] == "Complete sentence."
assert runtime.wire.count_requests(), "Auto must count the actual prepared audio prompt"
assert _context(runtime.processes[-1]) == expected_context
assert runtime.wire.generation_requests()[-1][1]["max_tokens"] == 512
def test_device_less_warm_load_preserves_selected_context(runtime):
runtime.sidecar.load(MODEL, context_length = 8192)
process = runtime.processes[-1]
runtime.sidecar.load(MODEL, context_length = None)
assert runtime.processes == [process]
assert _context(process) == 8192
def test_manual_context_bounds_output_without_reloading_larger(runtime):
runtime.wire.input_tokens = 900
result = runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 1024)
assert result["text"] == "Complete sentence."
assert all(_context(process) == 1024 for process in runtime.processes)
assert runtime.wire.generation_requests()[-1][1]["max_tokens"] == 60
def test_count_and_generation_share_language_prefill_and_authentication(runtime):
runtime.sidecar.load(MODEL, context_length = 4096)
runtime.sidecar._api_key = "test-launch-key"
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 4096, language = "en-GB")
count = runtime.wire.count_requests()[-1]
generation = runtime.wire.generation_requests()[-1]
assert count[1]["messages"] == generation[1]["messages"]
assert count[1]["messages"][-1] == {
"role": "assistant",
"content": "language English<asr_text>",
}
for request in (count, generation):
assert request[1]["continue_final_message"] is True
assert request[1]["add_generation_prompt"] is False
assert request[2]["Authorization"] == "Bearer test-launch-key"
def test_manual_context_can_use_an_older_server_without_token_counting(runtime):
runtime.wire.count_status = 404
result = runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 4096)
assert result["text"] == "Complete sentence."
assert runtime.wire.generation_requests()
assert all(_context(process) == 4096 for process in runtime.processes)
@pytest.mark.parametrize("status", [404, 405, 501])
def test_auto_keeps_legacy_capacity_when_runtime_cannot_count_audio(runtime, status):
runtime.wire.count_status = status
result = runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert result["text"] == "Complete sentence."
assert runtime.wire.generation_requests()
assert runtime.sidecar.context_length == 65536
assert runtime.sidecar.context_length_preference == 0
assert _context(runtime.processes[-1]) == 65536
assert runtime.sidecar._active_requests == 0
def test_manual_context_rejects_input_that_leaves_no_output_room(runtime):
runtime.wire.input_tokens = 960
with pytest.raises(SttContextSizeError, match = "(?i)does not fit"):
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 1024)
assert not runtime.wire.generation_requests()
assert [_context(process) for process in runtime.processes] == [1024]
assert runtime.sidecar._active_requests == 0
def test_auto_rejects_recordings_beyond_model_capacity(runtime):
runtime.wire.input_tokens = 64961 # Input + 512 output + 64 margin exceeds 64K by one.
with pytest.raises(SttContextSizeError, match = "(?i)64K|65536"):
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert not runtime.wire.generation_requests()
assert [_context(process) for process in runtime.processes] == [4096]
def test_auto_accepts_exact_model_capacity(runtime):
runtime.wire.input_tokens = 64960
assert (
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)["text"]
== "Complete sentence."
)
assert _context(runtime.processes[-1]) == 65536
assert runtime.wire.generation_requests()[-1][1]["max_tokens"] == 512
def test_auto_resize_uses_the_new_servers_authentication_key(runtime, monkeypatch):
from core.inference import llama_cpp
monkeypatch.setattr(llama_cpp, "_llama_server_api_key_enabled", lambda: True)
monkeypatch.setattr(
llama_cpp.LlamaCppBackend,
"probe_server_capabilities",
lambda _binary: {"supports_api_key_file": True},
)
monkeypatch.setattr(llama_cpp, "_llama_server_key_launch", lambda _key: ([], {}, None))
keys = iter(("initial-launch-key", "resized-launch-key"))
monkeypatch.setattr(mtmd.secrets, "token_urlsafe", lambda _length: next(keys))
runtime.wire.input_tokens = 3521
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert runtime.wire.count_requests()[0][2]["Authorization"] == "Bearer initial-launch-key"
assert runtime.wire.generation_requests()[0][2]["Authorization"] == "Bearer resized-launch-key"
assert [_context(process) for process in runtime.processes] == [4096, 8192]
assert runtime.processes[0].poll() is not None
def test_manual_setting_cannot_replace_auto_preference_during_count(runtime):
runtime.wire.input_tokens = 3521
runtime.wire.before_count = lambda: runtime.sidecar.load(MODEL, context_length = 4096)
with pytest.raises(mtmd.SttModelBusyError):
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert not runtime.wire.generation_requests()
assert [_context(process) for process in runtime.processes] == [4096]
assert runtime.sidecar.context_length_preference == 0
def test_an_idle_timer_cannot_unload_a_request_while_it_counts(runtime):
runtime.wire.before_count = lambda: runtime.sidecar._idle_unload(runtime.sidecar._generation)
result = runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert result["text"] == "Complete sentence."
assert len(runtime.processes) == 1
assert runtime.processes[0].poll() is None
def test_forced_unload_during_count_does_not_send_audio_to_a_stale_port(runtime, monkeypatch):
monkeypatch.setattr(mtmd, "_ACTIVE_REQUEST_DRAIN_TIMEOUT", 0)
runtime.wire.before_count = lambda: runtime.sidecar.unload(wait = True)
with pytest.raises(mtmd.SttModelBusyError):
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert not runtime.wire.generation_requests()
assert runtime.processes[0].poll() is not None
assert runtime.sidecar._active_requests == 0
def test_cancelled_resize_wait_releases_its_request_without_killing_the_server(
runtime, monkeypatch
):
runtime.wire.input_tokens = 3521
counting, return_count, resizing = threading.Event(), threading.Event(), threading.Event()
cancelled = threading.Event()
errors = []
original_resize = runtime.sidecar._resize_for_request
def count():
counting.set()
assert return_count.wait(3)
def resize(*args, **kwargs):
resizing.set()
return original_resize(*args, **kwargs)
runtime.wire.before_count = count
monkeypatch.setattr(runtime.sidecar, "_resize_for_request", resize)
def transcribe():
try:
runtime.sidecar.transcribe(
b"encoded", model = MODEL, context_length = 0, cancel_event = cancelled
)
except Exception as exc:
errors.append(exc)
worker = threading.Thread(target = transcribe, daemon = True)
worker.start()
acquired = False
try:
assert counting.wait(3)
acquired = runtime.sidecar._start_lock.acquire(timeout = 2)
assert acquired
return_count.set()
assert resizing.wait(3)
runtime.sidecar.cancel_transcription(cancelled)
worker.join(timeout = 2)
assert not worker.is_alive(), "cancellation remained blocked behind another startup"
finally:
return_count.set()
if acquired:
runtime.sidecar._start_lock.release()
worker.join(timeout = 3)
assert len(errors) == 1 and isinstance(errors[0], mtmd.SttTranscriptionCancelledError)
assert runtime.sidecar._active_requests == 0
assert not runtime.wire.generation_requests()
assert len(runtime.processes) == 1 and runtime.processes[0].poll() is None
@pytest.mark.parametrize("second_input_tokens, second_succeeds", [(8000, False), (200, True)])
def test_context_resize_does_not_interrupt_a_sibling_transcription(
runtime, second_input_tokens, second_succeeds
):
first_started, release_first = threading.Event(), threading.Event()
first_results, first_errors = [], []
def count():
runtime.wire.input_tokens = (
3521 if threading.current_thread().name == "first-recording" else second_input_tokens
)
def generate():
if threading.current_thread().name == "first-recording":
first_started.set()
assert release_first.wait(5)
runtime.wire.before_count = count
runtime.wire.before_generate = generate
def first_request():
try:
first_results.append(
runtime.sidecar.transcribe(b"first", model = MODEL, context_length = 0)
)
except Exception as exc:
first_errors.append(exc)
worker = threading.Thread(target = first_request, name = "first-recording", daemon = True)
worker.start()
try:
assert first_started.wait(3)
resident = runtime.processes[-1]
assert _context(resident) == 8192
if second_succeeds:
assert (
runtime.sidecar.transcribe(b"second", model = MODEL, context_length = 0)["text"]
== "Complete sentence."
)
else:
# This case needs a genuinely larger window than the first recording's 8K.
with pytest.raises(mtmd.SttModelBusyError):
runtime.sidecar.transcribe(b"second", model = MODEL, context_length = 0)
assert runtime.processes[-1] is resident and resident.poll() is None
finally:
release_first.set()
worker.join(timeout = 5)
assert not worker.is_alive()
assert not first_errors
assert first_results[0]["text"] == "Complete sentence."
assert [_context(process) for process in runtime.processes] == [4096, 8192]
assert runtime.sidecar._active_requests == 0
def test_setting_changed_between_load_and_request_claim_is_not_overwritten(runtime, monkeypatch):
"""A same-model setting change need not replace the process or bump its generation."""
original_load = runtime.sidecar.load
runtime.wire.input_tokens = 3521
def load_then_select_manual(*args, **kwargs):
original_load(*args, **kwargs)
original_load(MODEL, context_length = 4096)
monkeypatch.setattr(runtime.sidecar, "load", load_then_select_manual)
with pytest.raises(mtmd.SttModelBusyError):
runtime.sidecar.transcribe(b"encoded", model = MODEL, context_length = 0)
assert not runtime.wire.generation_requests()
assert [_context(process) for process in runtime.processes] == [4096]
assert runtime.sidecar.context_length_preference == 4096
@pytest.mark.parametrize("context_length", [1000, 5000, 65537])
def test_a_size_llama_server_would_pad_is_refused_before_launch(runtime, context_length):
with pytest.raises(SttContextSizeError, match = "multiple of 256"):
runtime.sidecar.load(MODEL, context_length = context_length)
assert runtime.processes == []
def test_server_cannot_silently_exceed_the_requested_manual_context(runtime, monkeypatch):
monkeypatch.setattr(runtime.sidecar, "_read_server_context", lambda *_args, **_kwargs: 8192)
with pytest.raises(SttContextSizeError, match = "8192.*4096"):
runtime.sidecar.load(MODEL, context_length = 4096)
assert runtime.sidecar.loaded_model is None
assert len(runtime.processes) == 1 and runtime.processes[0].poll() is not None
assert runtime.sidecar.is_loading() is False
@pytest.mark.parametrize(
"failure",
[ConnectionResetError("connection reset"), mtmd.SttUnavailableError("props unavailable")],
)
def test_preemption_during_context_verification_stays_retryable(runtime, monkeypatch, failure):
"""Training can terminate a healthy child while the new /props probe is in flight."""
def preempt_during_props(*_args, **_kwargs):
assert runtime.sidecar.cancel_pending_load()
raise failure
monkeypatch.setattr(runtime.sidecar, "_read_server_context", preempt_during_props)
with pytest.raises(mtmd.SttLoadCancelledError):
runtime.sidecar.load(MODEL, context_length = 4096)
assert runtime.sidecar.loaded_model is None
assert runtime.sidecar.is_loading() is False
assert runtime.processes[0].poll() is not None
def test_cancelling_context_verification_interrupts_a_stalled_http_read(runtime, monkeypatch):
"""An unresponsive child must not turn cancellation into a sixty-second /props timeout."""
import subprocess
reading, unblock = threading.Event(), threading.Event()
cancelled = threading.Event()
errors = []
class Socket:
def shutdown(self, _how):
unblock.set()
class Response:
status = 200
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self):
reading.set()
assert unblock.wait(5), "the test did not clean up its blocked HTTP read"
raise OSError("connection interrupted")
class Connection:
def __init__(self, *_args, **_kwargs):
self.sock = Socket()
def request(self, method, path, **_kwargs):
assert method == "GET" and path == "/props"
def getresponse(self):
return Response()
def close(self):
pass
monkeypatch.setattr(mtmd.http.client, "HTTPConnection", Connection)
def load():
try:
runtime.sidecar.load(MODEL, context_length = 4096, request_cancel_event = cancelled)
except Exception as exc:
errors.append(exc)
loader = threading.Thread(target = load, daemon = True)
loader.start()
try:
assert reading.wait(3)
process = runtime.processes[0]
# Match an actual native child that ignores graceful shutdown until it is killed.
monkeypatch.setattr(process, "terminate", lambda: None)
def wait(timeout = None):
if process.poll() is None:
raise subprocess.TimeoutExpired("llama-server", timeout)
return process.returncode
monkeypatch.setattr(process, "wait", wait)
runtime.sidecar.cancel_transcription(cancelled)
loader.join(timeout = 1)
assert not loader.is_alive(), "cancelled load is still waiting on the /props HTTP timeout"
finally:
unblock.set()
loader.join(timeout = 3)
assert len(errors) == 1 and isinstance(errors[0], mtmd.SttLoadCancelledError)
assert process.poll() is not None
assert runtime.sidecar.is_loading() is False