Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop one audio chat from blocking every other chat (#11483)

Nilayaabbadb42e62
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 457622dae1f..4753de0eaac 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1871,6 +1871,11 @@ _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS = (
_OPENAI_LLAMA_ADMISSION_IMAGE_EMBEDDING_CAP + _OPENAI_LLAMA_ADMISSION_IMAGE_WRAPPER_TOKENS
)
+# Also an upper bound: no mtmd audio projector emits over 25 embeddings a second, and
+# Whisper-style preprocessing appends 30 s of silence before cutting 30 s windows.
+_OPENAI_LLAMA_ADMISSION_AUDIO_TOKENS_PER_SECOND = 25
+_OPENAI_LLAMA_ADMISSION_AUDIO_WINDOW_SECONDS = 30
+
# An ESTIMATE where the rest of the sizing is a bound: a run that generates more is
# undercharged until something re-costs it, which a tool loop does every round boundary and
# a plain chat cannot yet, so on a full cache a long uncapped generation can still overrun.
@@ -2096,6 +2101,11 @@ def _openai_llama_admission_messages_for_estimate(
estimate_content.append({"type": part_type, part_type: {"url": "[video]"}})
continue
+ # The recording is charged by duration from ``audio_base64``.
+ if part_type == "input_audio":
+ estimate_content.append({"type": part_type, part_type: {"data": "[audio]"}})
+ continue
+
if part_type not in _ADMISSION_IMAGE_PART_TYPES:
estimate_content.append(part)
continue
@@ -2133,6 +2143,27 @@ def _conversation_video_clips(messages) -> list[str]:
return clips
+def _openai_llama_admission_audio_tokens(b64: str) -> int:
+ fallback = max(1, len(b64) // 4)
+ if b64.startswith("data:"):
+ b64 = b64.split(",", 1)[1] if "," in b64 else ""
+ try:
+ raw = base64.b64decode(b64)
+ except Exception:
+ return fallback
+ container = _sniff_audio_container(raw)
+ seconds = _passthrough_audio_seconds(raw, container, _MAX_AUDIO_SECONDS) if container else None
+ if seconds is None:
+ return fallback
+ return (
+ math.ceil(
+ _OPENAI_LLAMA_ADMISSION_AUDIO_TOKENS_PER_SECOND
+ * (seconds + _OPENAI_LLAMA_ADMISSION_AUDIO_WINDOW_SECONDS)
+ )
+ + _OPENAI_LLAMA_ADMISSION_IMAGE_WRAPPER_TOKENS
+ )
+
+
def _openai_llama_admission_media_tokens(
payload,
*,
@@ -2148,21 +2179,21 @@ def _openai_llama_admission_media_tokens(
already carry is a second image really sent. The allowance is per-image rather than
per-byte because the real mtmd count follows the loaded projector, not base64 length.
- Audio and video keep the old top-level accounting until they have a model-specific
- estimate; this is scoped to the image path that regressed vision concurrency.
+ Audio is charged by its header's duration, or by its bytes where none is stated. Video
+ keeps the old top-level accounting until it has a model-specific estimate.
"""
extra = 0
extra += max(0, message_image_parts) * image_tokens
if _legacy_image_is_distinct(payload):
extra += image_tokens
+ audio = getattr(payload, "audio_base64", None)
+ if isinstance(audio, str) and audio:
+ extra += _openai_llama_admission_audio_tokens(audio)
# _inject_video_part splices the legacy clip into the conversation as input_video before the
# loop starts, so during a recost the clips below already include it and charging the field
# too priced it exactly twice.
- fields = (
- ("audio_base64",) if message_video_clips is not None else ("audio_base64", "video_base64")
- )
- for attribute in fields:
- value = getattr(payload, attribute, None)
+ if message_video_clips is None:
+ value = getattr(payload, "video_base64", None)
if isinstance(value, str) and value:
extra += max(1, len(value) // 4)
# Recosting passes the CURRENT conversation: a clip on a turn that truncate_oldest has since
@@ -2381,10 +2412,13 @@ async def _openai_llama_admission_reserve_async(
messages_override = None,
) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:
"""The same reservation, with the estimate off the loop when a replayed envelope
- would make it parse one."""
+ would make it parse one, or a recording would make it decode one."""
tokens = _TOKENS_UNSET
- if payload is not None and _messages_mention_mcp_images(
- messages_override if messages_override is not None else payload.messages
+ if payload is not None and (
+ getattr(payload, "audio_base64", None)
+ or _messages_mention_mcp_images(
+ messages_override if messages_override is not None else payload.messages
+ )
):
tokens = await asyncio.to_thread(
_openai_llama_admission_estimate,
@@ -21150,13 +21184,12 @@ def _wav_seconds(raw: bytes) -> Optional[float]:
# here can check a codec's own header, so say so and let the
# bounded decoder have it.
return None
- # Among the PCM fields nAvgBytesPerSec is the redundant one, so it
- # is the one that can be moved alone. Multiplied by ten thousand it
- # made half an hour read as a quarter of a second, and a quarter of
- # a second is forwarded untouched. Recompute it, and take the
- # declaration only where it agrees.
- computed = sample_rate * (block_align or channels * (bits // 8))
- byte_rate = computed if computed > 0 and declared_rate != computed else declared_rate
+ # Measure a frame as llama.cpp's decoder (miniaudio) does: by sample width for
+ # byte-aligned PCM, else by nBlockAlign. Trusting either rate field as written let an
+ # inflated one make half an hour read as a quarter of a second, forwarded untouched.
+ frame_bytes = channels * bits // 8 if bits % 8 == 0 else block_align
+ computed = sample_rate * frame_bytes
+ byte_rate = computed or declared_rate
elif chunk == b"data":
if byte_rate <= 0:
return None
@@ -25240,6 +25273,9 @@ async def produce_openai_chat_completions(
except Exception as e:
logger.warning("Audio decode failed: %s", e, exc_info = True)
raise _reject(400, "Could not decode the provided audio file.")
+ # Admission reads the duration from this field's header, which only the forwarded
+ # wav/mp3 is sure to state; an m4a, ogg or flac upload would be charged by its bytes.
+ payload.audio_base64 = audio_b64
# llama-server samples frames but encodes each at the clip's resolution.
video_b64 = None
diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
index 338303fd3ab..6901df1954d 100644
--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
@@ -16,12 +16,21 @@ estimate for a lease that runs up to 25 growing rounds.
import base64
import copy
+import io
+import wave
+import numpy as np
import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+import routes.inference as inference_route
+from auth.authentication import get_current_subject
from models.inference import AnthropicMessagesRequest
from routes.inference import (
_OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,
+ _openai_llama_admission_media_tokens,
_openai_llama_admission_messages_for_estimate,
_openai_llama_admission_tokens,
)
@@ -368,6 +377,230 @@ class TestMediaIsCharged:
assert cost > 2000, f"{field} was charged {cost}, i.e. nothing for the media"
+def _wav_b64(seconds: float, rate: int = 16000) -> str:
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(1)
+ w.setsampwidth(2)
+ w.setframerate(rate)
+ w.writeframes(b"\x00\x00" * int(seconds * rate))
+ return base64.b64encode(buf.getvalue()).decode()
+
+
+def _mp3_b64(frames: int) -> str:
+ # MPEG-1 Layer III, 128 kbps, 44.1 kHz: 417-byte frames of 1152 samples.
+ frame = b"\xff\xfb\x90\x00" + b"\x00" * 413
+ return base64.b64encode(frame * frames).decode()
+
+
+class TestAnAudioTurnIsChargedByItsDuration:
+ """A recording was charged len(base64) // 4, so four seconds of 16 kHz WAV filled a 32768
+ cache and every other chat queued behind it. mtmd charges audio by duration, not bytes."""
+
+ class _Backend:
+ base_url = "http://llama-audio"
+ effective_parallel_slots = 4
+ _kv_cache_context_total = 32768
+ context_length = 32768
+ _mmproj_projector_type = None
+ _extra_args = None
+
+ def test_an_audio_chat_does_not_reserve_the_whole_cache(self):
+ audio = _wav_b64(20)
+ assert len(audio) > 800_000
+ budget = 32768
+ payload = _Payload(
+ messages = [{"role": "user", "content": "transcribe"}],
+ audio_base64 = audio,
+ max_tokens = 256,
+ )
+ cost = _openai_llama_admission_tokens(payload, budget = budget, capacity = 4)
+ assert cost < budget // 4, f"20 s of audio reserved {cost} of {budget}"
+
+ def test_a_text_chat_is_admitted_beside_an_audio_chat(self):
+ async def scenario():
+ queue = LlamaAdmissionQueue("media")
+ config = LlamaAdmissionConfig()
+ leases = []
+ for fields in (
+ {"audio_base64": _wav_b64(5), "content": "transcribe"},
+ {"content": "hi"},
+ ):
+ payload = _Payload(
+ messages = [{"role": "user", "content": fields.pop("content")}],
+ max_tokens = 256,
+ **fields,
+ )
+ reservation = queue.reserve(
+ capacity = 4,
+ config = config,
+ budget = 32768,
+ tokens = _openai_llama_admission_tokens(payload, budget = 32768, capacity = 4),
+ )
+ leases.append(reservation.lease_nowait())
+ return leases
+
+ audio, text = asyncio.run(scenario())
+ assert audio is not None
+ assert text is not None, "a text chat queued behind a 5 s recording"
+
+ @pytest.mark.parametrize(
+ "audio, seconds",
+ [
+ (_wav_b64(1), 1.0),
+ ("data:audio/wav;base64," + _wav_b64(45), 45.0),
+ (_mp3_b64(383), 383 * 1152 / 44100),
+ ],
+ ids = ["wav-1s", "wav-data-uri-45s", "mp3-10s"],
+ )
+ def test_the_charge_bounds_every_audio_projector(self, audio, seconds):
+ cost = _openai_llama_admission_media_tokens(_Payload(audio_base64 = audio))
+ # 25 embeddings a second, and a Whisper encoder pads each clip to a whole 30 s window.
+ whisper_windows = int(seconds // 30) + 1
+ assert cost >= max(25 * seconds, 750 * whisper_windows)
+ assert cost <= 25 * (seconds + 30) + 256
+
+ @pytest.mark.parametrize(
+ "audio",
+ [
+ _image_b64(8),
+ base64.b64encode(b"RIFF\x00\x00\x00\x00WAVEjunk" + b"\x00" * 4096).decode(),
+ "not base64 at all!",
+ ],
+ ids = ["not-audio", "wav-without-data-chunk", "not-base64"],
+ )
+ def test_audio_without_a_stated_duration_keeps_the_byte_charge(self, audio):
+ cost = _openai_llama_admission_media_tokens(_Payload(audio_base64 = audio))
+ assert cost == max(1, len(audio) // 4)
+
+ def test_video_keeps_the_byte_charge(self):
+ clip = _wav_b64(20)
+ cost = _openai_llama_admission_media_tokens(_Payload(video_base64 = clip, messages = []))
+ assert cost == len(clip) // 4
+
+ def test_a_tool_round_does_not_price_the_injected_recording_as_text(self):
+ from routes.inference import _openai_llama_admission_recost
+
+ class _Reservation:
+ def __init__(self, lease):
+ self._lease = lease
+
+ def lease_nowait(self):
+ return self._lease
+
+ audio = _wav_b64(3)
+ payload = _Payload(
+ messages = [{"role": "user", "content": "transcribe"}],
+ audio_base64 = audio,
+ enable_tools = True,
+ max_tokens = 256,
+ )
+ # What _inject_audio_part hands the tool loop.
+ conversation = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "transcribe"},
+ {"type": "input_audio", "input_audio": {"data": audio, "format": "wav"}},
+ ],
+ }
+ ]
+
+ async def run():
+ queue = LlamaAdmissionQueue("test")
+ reservation = queue.reserve(
+ capacity = 4,
+ config = LlamaAdmissionConfig(),
+ budget = 32768,
+ tokens = _openai_llama_admission_tokens(
+ payload, budget = 32768, capacity = 4, tool_loop = True
+ ),
+ )
+ lease = reservation.lease_nowait()
+ assert lease is not None
+ _openai_llama_admission_recost(
+ _Reservation(lease),
+ conversation,
+ request = None,
+ llama_backend = self._Backend(),
+ payload = payload,
+ output_tokens = 256,
+ )
+ return queue.snapshot().committed
+
+ committed = asyncio.run(run())
+ assert committed <= 32768 // 4, f"round zero re-costed a 3 s recording to {committed}"
+
+ def test_the_recording_is_decoded_off_the_event_loop(self, monkeypatch):
+ import threading
+
+ import routes.inference as inference_route
+
+ on_loop = []
+ estimate = inference_route._openai_llama_admission_estimate
+
+ def spy(**kwargs):
+ on_loop.append(threading.current_thread() is threading.main_thread())
+ return estimate(**kwargs)
+
+ monkeypatch.setattr(inference_route, "_openai_llama_admission_estimate", spy)
+ payload = _Payload(
+ messages = [{"role": "user", "content": "transcribe"}],
+ audio_base64 = _wav_b64(1),
+ max_tokens = 64,
+ )
+
+ async def run():
+ reservation, _ = await inference_route._openai_llama_admission_reserve_async(
+ request = None, llama_backend = self._Backend(), payload = payload
+ )
+ reservation.cancel()
+
+ asyncio.run(run())
+ assert on_loop == [False]
+
+ # An ogg header states no length, so only the WAV it is transcoded to can be measured.
+ @pytest.mark.parametrize(
+ "raw",
+ [inference_route._mono_f32_to_wav_bytes(np.zeros(320_000), 16000), b"OggS" * 4**8],
+ ids = ["wav", "ogg"],
+ )
+ def test_an_audio_chat_reserves_a_bound_on_its_duration(self, monkeypatch, raw):
+ from .llama_backend_double import FakeLlamaCppBackend
+
+ class _AudioGguf(FakeLlamaCppBackend):
+ is_vision = _has_audio_input = True
+ _kv_cache_context_total = context_length = 32768
+
+ def generate_chat_completion(self, **_kwargs):
+ yield "ok"
+
+ charged, estimate = [], inference_route._openai_llama_admission_estimate
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", _AudioGguf)
+ monkeypatch.setattr(
+ inference_route,
+ "_openai_llama_admission_estimate",
+ lambda **kw: charged.append(estimate(**kw)) or charged[-1],
+ )
+ monkeypatch.setattr(
+ inference_route, "_decode_audio_mono", lambda _raw: (np.zeros(20 * 16000), 16000)
+ )
+ app = FastAPI()
+ app.include_router(inference_route.router, prefix = "/v1")
+ app.dependency_overrides[get_current_subject] = lambda: "tester"
+ body = {
+ "max_tokens": 256,
+ "messages": [{"role": "user", "content": "hi"}],
+ "audio_base64": base64.b64encode(raw).decode(),
+ }
+
+ response = TestClient(app).post("/v1/chat/completions", json = body)
+
+ assert response.status_code == 200, response.text
+ # 25/s over the clip and a trailing 30 s window, the wrapper, output, and the prompt.
+ assert 0 <= charged[0] - (25 * (20 + 30) + 128 + 256) < 32, charged
+
+
class TestAnAnthropicImageIsChargedLikeAnyOtherImage:
"""/v1/messages reserves from the RAW Anthropic request, so its own image block has to
be compacted too. #9842 fixed this for /v1/chat/completions and left this surface
diff --git a/studio/backend/tests/test_llama_audio_attachment_container.py b/studio/backend/tests/test_llama_audio_attachment_container.py
index a6ab9e3ce34..a65a051f412 100644
--- a/studio/backend/tests/test_llama_audio_attachment_container.py
+++ b/studio/backend/tests/test_llama_audio_attachment_container.py
@@ -691,6 +691,15 @@ def test_a_forwarded_wav_is_bounded_by_its_own_header(monkeypatch):
assert base64.b64decode(encoded) == short_wav
+def test_an_inflated_block_align_cannot_shorten_a_forwarded_wav():
+ wav = bytearray(_wav_header(16_000, 1, 16, 32_000) + bytes(32_000))
+ wav[32:34] = (2_000).to_bytes(2, "little") # nBlockAlign, which the decoder ignores
+ assert inference_route._wav_seconds(bytes(wav)) == 1.0
+ # Unless the samples are not byte-aligned: 12-bit samples pad to nBlockAlign's two bytes.
+ wav[32:36] = (2).to_bytes(2, "little") + (12).to_bytes(2, "little")
+ assert inference_route._wav_seconds(bytes(wav)) == 1.0
+
+
def test_a_forwarded_mp3_is_bounded_by_its_frame_headers(monkeypatch):
monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 60)
try: