Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop one audio chat from blocking every other chat (#11483)
Nilayaabbadb42e62
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 457622dae1f..4753de0eaac 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1871,6 +1871,11 @@ _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS = (_OPENAI_LLAMA_ADMISSION_IMAGE_EMBEDDING_CAP + _OPENAI_LLAMA_ADMISSION_IMAGE_WRAPPER_TOKENS)+# Also an upper bound: no mtmd audio projector emits over 25 embeddings a second, and+# Whisper-style preprocessing appends 30 s of silence before cutting 30 s windows.+_OPENAI_LLAMA_ADMISSION_AUDIO_TOKENS_PER_SECOND = 25+_OPENAI_LLAMA_ADMISSION_AUDIO_WINDOW_SECONDS = 30+# An ESTIMATE where the rest of the sizing is a bound: a run that generates more is# undercharged until something re-costs it, which a tool loop does every round boundary and# a plain chat cannot yet, so on a full cache a long uncapped generation can still overrun.@@ -2096,6 +2101,11 @@ def _openai_llama_admission_messages_for_estimate(estimate_content.append({"type": part_type, part_type: {"url": "[video]"}})continue+ # The recording is charged by duration from ``audio_base64``.+ if part_type == "input_audio":+ estimate_content.append({"type": part_type, part_type: {"data": "[audio]"}})+ continue+if part_type not in _ADMISSION_IMAGE_PART_TYPES:estimate_content.append(part)continue@@ -2133,6 +2143,27 @@ def _conversation_video_clips(messages) -> list[str]:return clips+def _openai_llama_admission_audio_tokens(b64: str) -> int:+ fallback = max(1, len(b64) // 4)+ if b64.startswith("data:"):+ b64 = b64.split(",", 1)[1] if "," in b64 else ""+ try:+ raw = base64.b64decode(b64)+ except Exception:+ return fallback+ container = _sniff_audio_container(raw)+ seconds = _passthrough_audio_seconds(raw, container, _MAX_AUDIO_SECONDS) if container else None+ if seconds is None:+ return fallback+ return (+ math.ceil(+ _OPENAI_LLAMA_ADMISSION_AUDIO_TOKENS_PER_SECOND+ * (seconds + _OPENAI_LLAMA_ADMISSION_AUDIO_WINDOW_SECONDS)+ )+ + _OPENAI_LLAMA_ADMISSION_IMAGE_WRAPPER_TOKENS+ )++def _openai_llama_admission_media_tokens(payload,*,@@ -2148,21 +2179,21 @@ def _openai_llama_admission_media_tokens(already carry is a second image really sent. The allowance is per-image rather thanper-byte because the real mtmd count follows the loaded projector, not base64 length.- Audio and video keep the old top-level accounting until they have a model-specific- estimate; this is scoped to the image path that regressed vision concurrency.+ Audio is charged by its header's duration, or by its bytes where none is stated. Video+ keeps the old top-level accounting until it has a model-specific estimate."""extra = 0extra += max(0, message_image_parts) * image_tokensif _legacy_image_is_distinct(payload):extra += image_tokens+ audio = getattr(payload, "audio_base64", None)+ if isinstance(audio, str) and audio:+ extra += _openai_llama_admission_audio_tokens(audio)# _inject_video_part splices the legacy clip into the conversation as input_video before the# loop starts, so during a recost the clips below already include it and charging the field# too priced it exactly twice.- fields = (- ("audio_base64",) if message_video_clips is not None else ("audio_base64", "video_base64")- )- for attribute in fields:- value = getattr(payload, attribute, None)+ if message_video_clips is None:+ value = getattr(payload, "video_base64", None)if isinstance(value, str) and value:extra += max(1, len(value) // 4)# Recosting passes the CURRENT conversation: a clip on a turn that truncate_oldest has since@@ -2381,10 +2412,13 @@ async def _openai_llama_admission_reserve_async(messages_override = None,) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:"""The same reservation, with the estimate off the loop when a replayed envelope- would make it parse one."""+ would make it parse one, or a recording would make it decode one."""tokens = _TOKENS_UNSET- if payload is not None and _messages_mention_mcp_images(- messages_override if messages_override is not None else payload.messages+ if payload is not None and (+ getattr(payload, "audio_base64", None)+ or _messages_mention_mcp_images(+ messages_override if messages_override is not None else payload.messages+ )):tokens = await asyncio.to_thread(_openai_llama_admission_estimate,@@ -21150,13 +21184,12 @@ def _wav_seconds(raw: bytes) -> Optional[float]:# here can check a codec's own header, so say so and let the# bounded decoder have it.return None- # Among the PCM fields nAvgBytesPerSec is the redundant one, so it- # is the one that can be moved alone. Multiplied by ten thousand it- # made half an hour read as a quarter of a second, and a quarter of- # a second is forwarded untouched. Recompute it, and take the- # declaration only where it agrees.- computed = sample_rate * (block_align or channels * (bits // 8))- byte_rate = computed if computed > 0 and declared_rate != computed else declared_rate+ # Measure a frame as llama.cpp's decoder (miniaudio) does: by sample width for+ # byte-aligned PCM, else by nBlockAlign. Trusting either rate field as written let an+ # inflated one make half an hour read as a quarter of a second, forwarded untouched.+ frame_bytes = channels * bits // 8 if bits % 8 == 0 else block_align+ computed = sample_rate * frame_bytes+ byte_rate = computed or declared_rateelif chunk == b"data":if byte_rate <= 0:return None@@ -25240,6 +25273,9 @@ async def produce_openai_chat_completions(except Exception as e:logger.warning("Audio decode failed: %s", e, exc_info = True)raise _reject(400, "Could not decode the provided audio file.")+ # Admission reads the duration from this field's header, which only the forwarded+ # wav/mp3 is sure to state; an m4a, ogg or flac upload would be charged by its bytes.+ payload.audio_base64 = audio_b64# llama-server samples frames but encodes each at the clip's resolution.video_b64 = Nonediff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.pyindex 338303fd3ab..6901df1954d 100644--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py@@ -16,12 +16,21 @@ estimate for a lease that runs up to 25 growing rounds.import base64import copy+import io+import wave+import numpy as npimport pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient++import routes.inference as inference_route+from auth.authentication import get_current_subjectfrom models.inference import AnthropicMessagesRequestfrom routes.inference import (_OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,+ _openai_llama_admission_media_tokens,_openai_llama_admission_messages_for_estimate,_openai_llama_admission_tokens,)@@ -368,6 +377,230 @@ class TestMediaIsCharged:assert cost > 2000, f"{field} was charged {cost}, i.e. nothing for the media"+def _wav_b64(seconds: float, rate: int = 16000) -> str:+ buf = io.BytesIO()+ with wave.open(buf, "wb") as w:+ w.setnchannels(1)+ w.setsampwidth(2)+ w.setframerate(rate)+ w.writeframes(b"\x00\x00" * int(seconds * rate))+ return base64.b64encode(buf.getvalue()).decode()+++def _mp3_b64(frames: int) -> str:+ # MPEG-1 Layer III, 128 kbps, 44.1 kHz: 417-byte frames of 1152 samples.+ frame = b"\xff\xfb\x90\x00" + b"\x00" * 413+ return base64.b64encode(frame * frames).decode()+++class TestAnAudioTurnIsChargedByItsDuration:+ """A recording was charged len(base64) // 4, so four seconds of 16 kHz WAV filled a 32768+ cache and every other chat queued behind it. mtmd charges audio by duration, not bytes."""++ class _Backend:+ base_url = "http://llama-audio"+ effective_parallel_slots = 4+ _kv_cache_context_total = 32768+ context_length = 32768+ _mmproj_projector_type = None+ _extra_args = None++ def test_an_audio_chat_does_not_reserve_the_whole_cache(self):+ audio = _wav_b64(20)+ assert len(audio) > 800_000+ budget = 32768+ payload = _Payload(+ messages = [{"role": "user", "content": "transcribe"}],+ audio_base64 = audio,+ max_tokens = 256,+ )+ cost = _openai_llama_admission_tokens(payload, budget = budget, capacity = 4)+ assert cost < budget // 4, f"20 s of audio reserved {cost} of {budget}"++ def test_a_text_chat_is_admitted_beside_an_audio_chat(self):+ async def scenario():+ queue = LlamaAdmissionQueue("media")+ config = LlamaAdmissionConfig()+ leases = []+ for fields in (+ {"audio_base64": _wav_b64(5), "content": "transcribe"},+ {"content": "hi"},+ ):+ payload = _Payload(+ messages = [{"role": "user", "content": fields.pop("content")}],+ max_tokens = 256,+ **fields,+ )+ reservation = queue.reserve(+ capacity = 4,+ config = config,+ budget = 32768,+ tokens = _openai_llama_admission_tokens(payload, budget = 32768, capacity = 4),+ )+ leases.append(reservation.lease_nowait())+ return leases++ audio, text = asyncio.run(scenario())+ assert audio is not None+ assert text is not None, "a text chat queued behind a 5 s recording"++ @pytest.mark.parametrize(+ "audio, seconds",+ [+ (_wav_b64(1), 1.0),+ ("data:audio/wav;base64," + _wav_b64(45), 45.0),+ (_mp3_b64(383), 383 * 1152 / 44100),+ ],+ ids = ["wav-1s", "wav-data-uri-45s", "mp3-10s"],+ )+ def test_the_charge_bounds_every_audio_projector(self, audio, seconds):+ cost = _openai_llama_admission_media_tokens(_Payload(audio_base64 = audio))+ # 25 embeddings a second, and a Whisper encoder pads each clip to a whole 30 s window.+ whisper_windows = int(seconds // 30) + 1+ assert cost >= max(25 * seconds, 750 * whisper_windows)+ assert cost <= 25 * (seconds + 30) + 256++ @pytest.mark.parametrize(+ "audio",+ [+ _image_b64(8),+ base64.b64encode(b"RIFF\x00\x00\x00\x00WAVEjunk" + b"\x00" * 4096).decode(),+ "not base64 at all!",+ ],+ ids = ["not-audio", "wav-without-data-chunk", "not-base64"],+ )+ def test_audio_without_a_stated_duration_keeps_the_byte_charge(self, audio):+ cost = _openai_llama_admission_media_tokens(_Payload(audio_base64 = audio))+ assert cost == max(1, len(audio) // 4)++ def test_video_keeps_the_byte_charge(self):+ clip = _wav_b64(20)+ cost = _openai_llama_admission_media_tokens(_Payload(video_base64 = clip, messages = []))+ assert cost == len(clip) // 4++ def test_a_tool_round_does_not_price_the_injected_recording_as_text(self):+ from routes.inference import _openai_llama_admission_recost++ class _Reservation:+ def __init__(self, lease):+ self._lease = lease++ def lease_nowait(self):+ return self._lease++ audio = _wav_b64(3)+ payload = _Payload(+ messages = [{"role": "user", "content": "transcribe"}],+ audio_base64 = audio,+ enable_tools = True,+ max_tokens = 256,+ )+ # What _inject_audio_part hands the tool loop.+ conversation = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "transcribe"},+ {"type": "input_audio", "input_audio": {"data": audio, "format": "wav"}},+ ],+ }+ ]++ async def run():+ queue = LlamaAdmissionQueue("test")+ reservation = queue.reserve(+ capacity = 4,+ config = LlamaAdmissionConfig(),+ budget = 32768,+ tokens = _openai_llama_admission_tokens(+ payload, budget = 32768, capacity = 4, tool_loop = True+ ),+ )+ lease = reservation.lease_nowait()+ assert lease is not None+ _openai_llama_admission_recost(+ _Reservation(lease),+ conversation,+ request = None,+ llama_backend = self._Backend(),+ payload = payload,+ output_tokens = 256,+ )+ return queue.snapshot().committed++ committed = asyncio.run(run())+ assert committed <= 32768 // 4, f"round zero re-costed a 3 s recording to {committed}"++ def test_the_recording_is_decoded_off_the_event_loop(self, monkeypatch):+ import threading++ import routes.inference as inference_route++ on_loop = []+ estimate = inference_route._openai_llama_admission_estimate++ def spy(**kwargs):+ on_loop.append(threading.current_thread() is threading.main_thread())+ return estimate(**kwargs)++ monkeypatch.setattr(inference_route, "_openai_llama_admission_estimate", spy)+ payload = _Payload(+ messages = [{"role": "user", "content": "transcribe"}],+ audio_base64 = _wav_b64(1),+ max_tokens = 64,+ )++ async def run():+ reservation, _ = await inference_route._openai_llama_admission_reserve_async(+ request = None, llama_backend = self._Backend(), payload = payload+ )+ reservation.cancel()++ asyncio.run(run())+ assert on_loop == [False]++ # An ogg header states no length, so only the WAV it is transcoded to can be measured.+ @pytest.mark.parametrize(+ "raw",+ [inference_route._mono_f32_to_wav_bytes(np.zeros(320_000), 16000), b"OggS" * 4**8],+ ids = ["wav", "ogg"],+ )+ def test_an_audio_chat_reserves_a_bound_on_its_duration(self, monkeypatch, raw):+ from .llama_backend_double import FakeLlamaCppBackend++ class _AudioGguf(FakeLlamaCppBackend):+ is_vision = _has_audio_input = True+ _kv_cache_context_total = context_length = 32768++ def generate_chat_completion(self, **_kwargs):+ yield "ok"++ charged, estimate = [], inference_route._openai_llama_admission_estimate+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", _AudioGguf)+ monkeypatch.setattr(+ inference_route,+ "_openai_llama_admission_estimate",+ lambda **kw: charged.append(estimate(**kw)) or charged[-1],+ )+ monkeypatch.setattr(+ inference_route, "_decode_audio_mono", lambda _raw: (np.zeros(20 * 16000), 16000)+ )+ app = FastAPI()+ app.include_router(inference_route.router, prefix = "/v1")+ app.dependency_overrides[get_current_subject] = lambda: "tester"+ body = {+ "max_tokens": 256,+ "messages": [{"role": "user", "content": "hi"}],+ "audio_base64": base64.b64encode(raw).decode(),+ }++ response = TestClient(app).post("/v1/chat/completions", json = body)++ assert response.status_code == 200, response.text+ # 25/s over the clip and a trailing 30 s window, the wrapper, output, and the prompt.+ assert 0 <= charged[0] - (25 * (20 + 30) + 128 + 256) < 32, charged++class TestAnAnthropicImageIsChargedLikeAnyOtherImage:"""/v1/messages reserves from the RAW Anthropic request, so its own image block has tobe compacted too. #9842 fixed this for /v1/chat/completions and left this surfacediff --git a/studio/backend/tests/test_llama_audio_attachment_container.py b/studio/backend/tests/test_llama_audio_attachment_container.pyindex a6ab9e3ce34..a65a051f412 100644--- a/studio/backend/tests/test_llama_audio_attachment_container.py+++ b/studio/backend/tests/test_llama_audio_attachment_container.py@@ -691,6 +691,15 @@ def test_a_forwarded_wav_is_bounded_by_its_own_header(monkeypatch):assert base64.b64decode(encoded) == short_wav+def test_an_inflated_block_align_cannot_shorten_a_forwarded_wav():+ wav = bytearray(_wav_header(16_000, 1, 16, 32_000) + bytes(32_000))+ wav[32:34] = (2_000).to_bytes(2, "little") # nBlockAlign, which the decoder ignores+ assert inference_route._wav_seconds(bytes(wav)) == 1.0+ # Unless the samples are not byte-aligned: 12-bit samples pad to nBlockAlign's two bytes.+ wav[32:36] = (2).to_bytes(2, "little") + (12).to_bytes(2, "little")+ assert inference_route._wav_seconds(bytes(wav)) == 1.0++def test_a_forwarded_mp3_is_bounded_by_its_frame_headers(monkeypatch):monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 60)try: