Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: accept multiple audio files per message (#12267)

Michael Hanba64bcbce674
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index a9dd6e0f7e6..202c64d90a1 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -14,7 +14,7 @@ import re
import string
import weakref
from dataclasses import dataclass
-from typing import Any, Optional
+from typing import Any, Optional, Sequence
logger = logging.getLogger(__name__)
@@ -2955,13 +2955,15 @@ def messages_with_attached_image(
image: int = 1,
video: bool = False,
audio: Any = None,
+ extra_audio: Sequence[Any] = (),
) -> list:
"""The conversation to render for a turn that carries attached media.
Prepends *system_prompt* as a leading system turn, then injects *image* ``{"type": "image"}``
parts, or a ``{"type": "video"}`` part, plus any *audio* waveform as an ``{"type": "audio"}``
- part, into the LAST user turn and leaves every other turn --
- assistant ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.
+ part (one more per *extra_audio* clip, in order), into the LAST user turn and leaves every
+ other turn -- assistant ``tool_calls`` and ``role="tool"`` results included -- exactly as the
+ caller sent it.
Rebuilding from the newest user TEXT instead dropped the folded system instruction and the
tool history an OpenAI tool loop replays (#10092). Nothing the caller owns is mutated: callers
still read those dicts after generation, and a retry re-renders the same list.
@@ -3014,6 +3016,7 @@ def messages_with_attached_image(
]
if audio is not None:
parts.append({"type": "audio", "audio": audio})
+ parts.extend({"type": "audio", "audio": clip} for clip in extra_audio)
if not parts and not fallback_user_text:
return conversation
for index in range(len(conversation) - 1, -1, -1):
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 671974a3856..3fa64bfae8b 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -1995,6 +1995,7 @@ class InferenceBackend:
repetition_penalty,
use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
+ extra_audio_arrays: Optional[list] = None,
) -> Generator[str, None, None]:
"""Audio-input (ASR) generation: takes an audio numpy array, streams text.
@@ -2015,7 +2016,7 @@ class InferenceBackend:
if not system_prompt:
system_prompt = "You are an assistant that transcribes speech accurately."
- # Gemma 3n format — audio goes INTO apply_chat_template
+ # Gemma 3n format: audio goes INTO apply_chat_template, one item per clip in order.
audio_messages = messages_with_attached_image(
alternating_turns(messages),
system_prompt = system_prompt,
@@ -2023,6 +2024,7 @@ class InferenceBackend:
structured_content = True,
image = 0,
audio = audio_array,
+ extra_audio = extra_audio_arrays or (),
)
# Direct processor render like the vision path, so neutralize here too, with
@@ -2156,6 +2158,7 @@ class InferenceBackend:
self,
audio_array,
cancel_event = None,
+ extra_audio_arrays: Optional[list] = None,
) -> Generator[str, None, None]:
"""Whisper ASR: takes an audio numpy array, yields transcribed text through the pipeline
built at model load."""
@@ -2169,13 +2172,17 @@ class InferenceBackend:
yield "Error: Whisper pipeline not initialized"
return
+ clips = [audio_array, *(extra_audio_arrays or [])]
try:
- with self._generation_lock:
- result = whisper_pipe({"raw": audio_array, "sampling_rate": 16000})
+ for index, clip in enumerate(clips):
+ if cancel_event is not None and cancel_event.is_set():
+ return
+ with self._generation_lock:
+ result = whisper_pipe({"raw": clip, "sampling_rate": 16000})
- text = result.get("text", "") if isinstance(result, dict) else str(result)
- if text:
- yield text
+ text = result.get("text", "") if isinstance(result, dict) else str(result)
+ if text:
+ yield f"\n\n{text}" if index else text
except Exception as e:
logger.error(f"Whisper ASR error: {e}")
yield f"Error: {str(e)}"
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 341de5b5ae2..486559598f6 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -6406,6 +6406,7 @@ class MLXInferenceBackend:
use_adapter = None,
cancel_event = None,
stop = None,
+ extra_audio_arrays = None,
**_sampler,
):
"""Audio-input chat (omni models): waveform in, incremental text deltas out (the audio route
@@ -6418,6 +6419,10 @@ class MLXInferenceBackend:
"no verified audio-capable tower/processor was detected at load."
)
+ # mlx-vlm silently drops all but the first clip; the route refuses this earlier.
+ if extra_audio_arrays:
+ raise RuntimeError("This MLX model takes one audio file per message.")
+
from mlx_vlm import stream_generate as vlm_stream
if not system_prompt:
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 65399c1b681..e94437d90f5 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3319,6 +3319,7 @@ class InferenceOrchestrator:
audio_array,
cancel_event = None,
stats_holder: Optional[dict] = None,
+ extra_audio_arrays: Optional[list] = None,
) -> Generator[str, None, None]:
"""Whisper ASR: sends audio to the subprocess and yields text."""
yield from self._generate_audio_input_inner(
@@ -3328,6 +3329,7 @@ class InferenceOrchestrator:
system_prompt = "",
cancel_event = cancel_event,
stats_holder = stats_holder,
+ extra_audio_arrays = extra_audio_arrays,
)
def generate_audio_input_response(
@@ -3345,6 +3347,7 @@ class InferenceOrchestrator:
cancel_event = None,
stats_holder: Optional[dict] = None,
stop = None,
+ extra_audio_arrays: Optional[list] = None,
) -> Generator[str, None, None]:
"""Audio input generation (e.g. Gemma 3n): streams text tokens."""
yield from self._generate_audio_input_inner(
@@ -3362,6 +3365,7 @@ class InferenceOrchestrator:
cancel_event = cancel_event,
stats_holder = stats_holder,
stop = stop,
+ extra_audio_arrays = extra_audio_arrays,
)
def _generate_audio_input_inner(
@@ -3380,6 +3384,7 @@ class InferenceOrchestrator:
cancel_event = None,
stats_holder: Optional[dict] = None,
stop = None,
+ extra_audio_arrays: Optional[list] = None,
) -> Generator[str, None, None]:
"""Shared inner logic for audio input generation (Whisper + ASR). ``stats_holder``: as in
generate_chat_response, caller-owned and filled on gen_done with the worker's usage /
@@ -3402,15 +3407,16 @@ class InferenceOrchestrator:
return
request_id = str(uuid.uuid4())
- # numpy array -> list for mp.Queue serialization
- audio_data = (
- audio_array.tolist() if hasattr(audio_array, "tolist") else list(audio_array)
- )
+ import numpy as np
+
+ # Raw float32 bytes per clip; far cheaper to pickle than tolist().
+ clips = [audio_array, *(extra_audio_arrays or [])]
+ audio_clips = [np.asarray(clip, dtype = np.float32).tobytes() for clip in clips]
cmd = {
"type": "generate_audio_input",
"request_id": request_id,
- "audio_data": audio_data,
+ "audio_clips": audio_clips,
"audio_type": audio_type,
"messages": messages or [],
"system_prompt": system_prompt,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 70c44c563fe..6475449246f 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1635,8 +1635,15 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
try:
import numpy as np
- # numpy arrays can't go through mp.Queue, so decode from list.
- audio_array = np.array(cmd["audio_data"], dtype = np.float32)
+ # "audio_data" is the older single-clip list form.
+ if "audio_clips" in cmd:
+ # Copy: frombuffer views are read-only.
+ clips = [np.frombuffer(clip, dtype = np.float32).copy() for clip in cmd["audio_clips"]]
+ else:
+ clips = [np.array(cmd["audio_data"], dtype = np.float32)]
+ audio_array = clips[0]
+ # Passed only when present, so single-clip calls are unchanged.
+ extra_audio_kwargs = {"extra_audio_arrays": clips[1:]} if len(clips) > 1 else {}
audio_type = cmd.get("audio_type")
@@ -1647,6 +1654,7 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
generator = backend.generate_whisper_response(
audio_array = audio_array,
cancel_event = cancel_event,
+ **extra_audio_kwargs,
)
else:
audio_kwargs = {
@@ -1670,7 +1678,7 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
backend, "stop", "generate_audio_input_response"
):
audio_kwargs["stop"] = cmd["stop"]
- generator = backend.generate_audio_input_response(**audio_kwargs)
+ generator = backend.generate_audio_input_response(**audio_kwargs, **extra_audio_kwargs)
logger.info("Starting audio input generation for request_id=%s", request_id)
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 9dd446d418a..d25c1e15a77 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -2403,6 +2403,13 @@ class ChatCompletionRequest(BaseModel):
None,
description = "[x-unsloth] Base64-encoded audio (wav/mp3/ogg/flac/m4a) for audio-input models",
)
+ extra_audio_base64: Optional[List[str]] = Field(
+ None,
+ description = (
+ "[x-unsloth] Further recordings after audio_base64, in order, for models that take "
+ "several clips in one message. Size and duration caps apply to all clips together."
+ ),
+ )
audio_instructions: Optional[str] = Field(
None,
description = (
@@ -2841,6 +2848,19 @@ class ChatCompletionRequest(BaseModel):
msg.tool_call_id = picked
return self
+ @model_validator(mode = "after")
+ def _promote_extra_audio(self) -> "ChatCompletionRequest":
+ """Keep ``audio_base64`` the first clip whenever any clip is attached.
+
+ Every capability, size and routing check keys on that field, so a request carrying
+ only ``extra_audio_base64`` must not slip past them as audio-free.
+ """
+ extra = [clip for clip in self.extra_audio_base64 or [] if clip]
+ if not self.audio_base64 and extra:
+ self.audio_base64 = extra.pop(0)
+ self.extra_audio_base64 = extra or None
+ return self
+
@model_validator(mode = "after")
def _map_thinking_to_enable_thinking(self) -> "ChatCompletionRequest":
return resolve_thinking_onto_enable_thinking(self)
diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.py
index 99e1e7f5a45..e09db90a9d4 100644
--- a/studio/backend/routes/chat_generation_runs.py
+++ b/studio/backend/routes/chat_generation_runs.py
@@ -71,6 +71,7 @@ _EXTERNAL_ROUTING_FIELDS = {
_MEDIA_FIELDS = {
"image_base64",
"audio_base64",
+ "extra_audio_base64",
"video_base64",
}
_SQLITE_MAX_INTEGER = 9_223_372_036_854_775_807
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index af36406f0b1..dc01bf014bf 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2282,9 +2282,9 @@ def _openai_llama_admission_media_tokens(
extra += max(0, message_image_parts) * image_tokens
if _legacy_image_is_distinct(payload):
extra += image_tokens
- audio = getattr(payload, "audio_base64", None)
- if isinstance(audio, str) and audio:
- extra += _openai_llama_admission_audio_tokens(audio)
+ for audio in _request_audio_clips(payload):
+ if isinstance(audio, str) and audio:
+ extra += _openai_llama_admission_audio_tokens(audio)
# _inject_video_part splices the legacy clip into the conversation as input_video before the
# loop starts, so during a recost the clips below already include it and charging the field
# too priced it exactly twice.
@@ -8963,6 +8963,10 @@ _AUDIO_IMAGE_INPUT_DETAIL = (
"This model takes audio or an image in one message, not both. Send the image on its own turn."
)
_AUDIO_VIDEO_INPUT_DETAIL = "This model takes audio or a video in one message, not both."
+_MLX_MULTI_AUDIO_DETAIL = (
+ "This MLX model takes one audio file per message. Send the recordings on separate turns, "
+ "or load the GGUF build of the model to send several at once."
+)
async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: bool) -> None:
@@ -8971,15 +8975,17 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo
Only for a swap that is about to run: these exist so a request the target cannot
serve fails before the load evicts the resident model, and where no swap happens the
serving branch decides for itself. A GGUF target is validated through
- :func:`_prepare_audio_for_llama`, which accepts a ``data:`` URI and the containers
+ :func:`_prepare_audio_clips_for_llama`, which accepts a ``data:`` URI and the containers
torchaudio cannot open, and supports ``continue_final_message``. A non-GGUF target is served by
:func:`_decode_audio_base64`, so the upload is validated with that same decoder and
- the array handed back under ``decoded`` for the audio branch to reuse.
+ the arrays handed back under ``decoded`` for the audio branch to reuse. Every clip is
+ validated, in order, so a bad second recording fails before the load too.
"""
+ clips = audio_preflight["clips"]
if target_is_gguf:
try:
audio_preflight["prepared"] = await asyncio.to_thread(
- _prepare_audio_for_llama, audio_preflight["b64"]
+ _prepare_audio_clips_for_llama, clips
)
except _DecodedAudioTooLongError:
# A limit the caller can act on. Reading as "could not be decoded"
@@ -9009,6 +9015,11 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo
)
if audio_preflight.get("has_image"):
raise HTTPException(status_code = 400, detail = _AUDIO_IMAGE_INPUT_DETAIL)
+ from core.inference.local_model_resolver import _host_serves_mlx
+
+ # A non-GGUF target on an MLX host loads on MLX, which takes one clip.
+ if len(clips) > 1 and _host_serves_mlx():
+ raise HTTPException(status_code = 400, detail = _MLX_MULTI_AUDIO_DETAIL)
if not _audio_decoder_is_available():
raise HTTPException(
status_code = 400,
@@ -9021,9 +9032,7 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo
),
)
try:
- audio_preflight["decoded"] = await asyncio.to_thread(
- _decode_audio_base64, audio_preflight["b64"]
- )
+ audio_preflight["decoded"] = await asyncio.to_thread(_decode_audio_clips, clips)
except _DecodedAudioTooLongError:
raise HTTPException(
status_code = 413,
@@ -22225,7 +22234,7 @@ def _decode_audio_base64(b64: str) -> "np.ndarray":
channels = max(1, int(getattr(probe, "num_channels", 1) or 1))
except Exception: # noqa: BLE001 - a container info cannot read is still loadable
rate, frames, channels = 0, 0, 1
- limit = rate * _MAX_AUDIO_SECONDS
+ limit = rate * _audio_seconds_cap()
if limit and frames > limit:
raise _DecodedAudioTooLongError(
f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
@@ -22236,14 +22245,14 @@ def _decode_audio_base64(b64: str) -> "np.ndarray":
# multichannel file -- goes to the bounded reader, which downmixes as it
# goes and holds mono frames only. Streaming beats refusing: the file is
# inside the clock, it is just too wide to hold at once.
- if limit and frames and frames * channels <= _MAX_DECODED_SAMPLES:
+ if limit and frames and frames * channels <= _decoded_samples_cap():
# One frame past the cap, so a container that misreports its length
# is still never fully read. Both caps apply: info() is the value
# being distrusted here, so an understated num_frames must not let
# the read run to the rate-relative limit, which at 192 kHz is four
# times the sample ceiling. A file that fits is unaffected: its
# length is under both.
- read_frames = min(limit, _MAX_DECODED_SAMPLES // channels)
+ read_frames = min(limit, _decoded_samples_cap() // channels)
waveform, sr = torchaudio.load(tmp_path, num_frames = read_frames + 1)
else:
import torch
@@ -22271,8 +22280,8 @@ def _decode_audio_base64(b64: str) -> "np.ndarray":
os.unlink(tmp_path)
# Backstop for a container that reported neither rate nor length.
- if (sr > 0 and waveform.shape[-1] > sr * _MAX_AUDIO_SECONDS) or (
- waveform.shape[-1] * waveform.shape[0] > _MAX_DECODED_SAMPLES
+ if (sr > 0 and waveform.shape[-1] > sr * _audio_seconds_cap()) or (
+ waveform.shape[-1] * waveform.shape[0] > _decoded_samples_cap()
):
raise _DecodedAudioTooLongError(
f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
@@ -22313,6 +22322,9 @@ _REMOTE_VIDEO_REFUSAL = (
"Remote video URLs are not supported. Send the clip as a data URI instead.",
)
_MAX_AUDIO_SECONDS = 30 * 60
+# Byte and duration caps cover all clips together; the count bounds per-clip decode cost.
+_MAX_AUDIO_CLIPS_PER_REQUEST = 8
+_AUDIO_CLIP_B64_SLACK_CHARS = 64
# The duration cap alone is rate-relative, so a high-rate container retains far
# more memory for the same 30 minutes: at 48 kHz that is 86M float32 samples,
# and np.concatenate doubles it. 48 kHz covers ordinary uploads, so this ceiling
@@ -22333,6 +22345,88 @@ class _DecodedAudioTooLongError(ValueError):
"""Decoded audio crossed the duration cap before it could be buffered."""
+# (seconds, samples) the current decode may use: what earlier clips in the request left over.
+_AUDIO_BUDGET_LEFT: "contextvars.ContextVar[Optional[tuple[int, int]]]" = contextvars.ContextVar(
+ "_AUDIO_BUDGET_LEFT", default = None
+)
+
+
+def _audio_seconds_cap() -> int:
+ left = _AUDIO_BUDGET_LEFT.get()
+ return _MAX_AUDIO_SECONDS if left is None else min(left[0], _MAX_AUDIO_SECONDS)
+
+
+def _decoded_samples_cap() -> int:
+ left = _AUDIO_BUDGET_LEFT.get()
+ if left is None or _MAX_AUDIO_SECONDS <= 0:
+ return _MAX_DECODED_SAMPLES
+ scaled = _MAX_DECODED_SAMPLES * _audio_seconds_cap() // _MAX_AUDIO_SECONDS
+ return min(scaled, left[1])
+
+
+def _decode_within(
+ seconds_used: float,
+ decode,
+ *args,
+ samples_used: int = 0,
+):
+ """Run ``decode`` capped to the duration and samples the earlier clips left."""
+ seconds_left = math.ceil(_MAX_AUDIO_SECONDS - seconds_used)
+ samples_left = _MAX_DECODED_SAMPLES - samples_used
+ if seconds_left <= 0 or samples_left <= 0:
+ raise _DecodedAudioTooLongError("combined audio exceeds the duration cap")
+ token = _AUDIO_BUDGET_LEFT.set((seconds_left, samples_left))
+ try:
+ return decode(*args)
+ finally:
+ _AUDIO_BUDGET_LEFT.reset(token)
+
+
+def _request_audio_clips(payload) -> list[str]:
+ """Every recording the request carries, ``audio_base64`` first."""
+ first = getattr(payload, "audio_base64", None)
+ if not first:
+ return []
+ return [first, *(getattr(payload, "extra_audio_base64", None) or [])]
+
+
+def _audio_too_large_detail(clip_count: int) -> str:
+ if clip_count > 1:
+ return "Audio files are too large (max ~25 MB per message, all files together)."
+ return "Audio file is too large (max ~25 MB)."
+
+
+def _request_audio_rejection(payload) -> Optional[tuple[int, str]]:
+ """Refuse too many clips or too many combined bytes."""
+ clips = _request_audio_clips(payload)
+ if len(clips) > _MAX_AUDIO_CLIPS_PER_REQUEST:
+ return (
+ 400,
+ f"Too many audio files in one message (max {_MAX_AUDIO_CLIPS_PER_REQUEST}).",
+ )
+ # Slack for per-clip base64 padding and data: headers.
+ budget = _MAX_AUDIO_B64_CHARS + _AUDIO_CLIP_B64_SLACK_CHARS * (len(clips) - 1)
+ if sum(len(clip) for clip in clips) > budget:
+ return (413, _audio_too_large_detail(len(clips)))
+ return None
+
+
+def _check_decoded_audio_budget(arrays: list) -> None:
+ """Apply the duration cap to all decoded 16 kHz clips together."""
+ if sum(len(array) for array in arrays) > 16000 * _MAX_AUDIO_SECONDS:
+ raise _DecodedAudioTooLongError("combined audio exceeds the duration cap")
+
+
+def _decode_audio_clips(clips: list[str]) -> list:
+ """Decode clips in order, each capped to the duration the earlier ones left."""
+ arrays: list = [_decode_audio_base64(clips[0])]
+ for clip in clips[1:]:
+ used = sum(len(array) for array in arrays) / 16000
+ arrays.append(_decode_within(used, _decode_audio_base64, clip))
+ _check_decoded_audio_budget(arrays)
+ return arrays
+
+
def _audio_too_long_detail() -> str:
"""The one answer an overlong upload gets, whichever path found it out."""
return f"Audio is too long (max {_MAX_AUDIO_SECONDS // 60} minutes)."
@@ -22420,14 +22514,14 @@ def _decode_audio_with_torchcodec(path: str) -> "tuple[Any, int]":
"decoded within the size limit; convert it to wav or mp3"
)
duration = float(getattr(metadata, "duration_seconds", 0.0) or 0.0)
- if duration > _MAX_AUDIO_SECONDS:
+ if duration > _audio_seconds_cap():
raise _DecodedAudioTooLongError(
f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
)
# Both ceilings, and a second past the clock so a container understating its
# own duration is still cut rather than believed.
channels = max(1, int(getattr(metadata, "num_channels", 1) or 1))
- seconds = min(_MAX_AUDIO_SECONDS + 1, _MAX_DECODED_SAMPLES / (rate * channels) + 1)
+ seconds = min(_audio_seconds_cap() + 1, _decoded_samples_cap() / (rate * channels) + 1)
samples = decoder.get_samples_played_in_range(0.0, seconds)
return samples.data, rate
@@ -22665,23 +22759,26 @@ def _resample_mono_linear(arr: "np.ndarray", source_rate: int, target_rate: int)
def _fit_transcoded_audio_to_wav_cap(
- arr: "np.ndarray", sample_rate: int
+ arr: "np.ndarray",
+ sample_rate: int,
+ cap: Optional[int] = None,
) -> "tuple[np.ndarray, int]":
- """Downsample only when needed so transcoded WAV stays within the upload cap."""
+ """Downsample only when needed so transcoded WAV fits ``cap`` (default: the upload cap)."""
+ cap = _MAX_AUDIO_RAW_BYTES if cap is None else cap
if sample_rate <= 0:
raise ValueError("decoded audio has an invalid sample rate")
wav_bytes = _WAV_HEADER_BYTES + len(arr) * 2
- if wav_bytes <= _MAX_AUDIO_RAW_BYTES:
+ if wav_bytes <= cap:
return arr, sample_rate
duration = len(arr) / float(sample_rate)
- max_samples = max(1, (_MAX_AUDIO_RAW_BYTES - _WAV_HEADER_BYTES) // 2)
+ max_samples = max(1, (cap - _WAV_HEADER_BYTES) // 2)
target_rate = int(max_samples // duration)
if target_rate < _MIN_TRANSCODE_AUDIO_SAMPLE_RATE:
raise ValueError("decoded audio exceeds the transcoded WAV size limit")
target_rate = min(sample_rate, target_rate)
fitted = _resample_mono_linear(arr, sample_rate, target_rate)
- if _WAV_HEADER_BYTES + len(fitted) * 2 > _MAX_AUDIO_RAW_BYTES:
+ if _WAV_HEADER_BYTES + len(fitted) * 2 > cap:
raise ValueError("decoded audio exceeds the transcoded WAV size limit")
return fitted, target_rate
@@ -22767,7 +22864,7 @@ def _decoded_sample_ceiling(sample_rate: int) -> int:
thirty minutes' worth of memory. Nothing may allocate past the smaller of
them, because that is the point at which the decode refuses anyway.
"""
- return min(sample_rate * _MAX_AUDIO_SECONDS, _MAX_DECODED_SAMPLES)
+ return min(sample_rate * _audio_seconds_cap(), _decoded_samples_cap())
def _av_expected_samples(container, sample_rate: int, ceiling: int) -> int:
@@ -22928,13 +23025,15 @@ def _decode_audio_mono(raw: bytes) -> "tuple[np.ndarray, int]":
"this audio file does not report a sample rate, so it cannot "
"be decoded within the size limit; convert it to wav or mp3"
)
- window = min(float(_MAX_AUDIO_SECONDS + 1), _MAX_DECODED_SAMPLES / probe_rate + 1)
+ window = min(
+ float(_audio_seconds_cap() + 1), _decoded_samples_cap() / probe_rate + 1
+ )
arr, sr = librosa.load(tmp_path, sr = None, mono = True, duration = window)
finally:
os.unlink(tmp_path)
if arr.ndim > 1:
arr = arr.mean(axis = 1)
- if (sr > 0 and len(arr) > sr * _MAX_AUDIO_SECONDS) or len(arr) > _MAX_DECODED_SAMPLES:
+ if (sr > 0 and len(arr) > sr * _audio_seconds_cap()) or len(arr) > _decoded_samples_cap():
raise _DecodedAudioTooLongError(
f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
)
@@ -22949,33 +23048,106 @@ def _prepare_audio_for_llama(b64: str) -> tuple[str, str]:
PCM payload inflation). Other containers (m4a/ogg/webm/flac) are decoded to
a mono WAV. Blocking; call via a thread from async paths.
"""
- if b64.startswith("data:"):
- b64 = b64.split(",", 1)[1] if "," in b64 else ""
- raw = base64.b64decode(b64)
- passthrough = _sniff_audio_container(raw)
- if passthrough is not None:
- # Forwarding skips every bounded decoder, so the duration cap has to be
- # applied from the headers instead. A 16 kbps MP3 holds hours inside the
- # 25 MB upload cap, and llama-server was left to decode all of it. A
- # 25 MB upload cap, and llama-server was left to decode all of it.
- seconds = _passthrough_audio_seconds(raw, passthrough, _MAX_AUDIO_SECONDS)
- if seconds is not None and seconds > _MAX_AUDIO_SECONDS:
+ return _prepare_audio_clips_for_llama([b64])[0]
+
+
+def _prepare_audio_clips_for_llama(clips: list[str]) -> list[tuple[str, str]]:
+ """Prepare clips for llama-server in order, under one shared byte and duration budget.
+
+ wav/mp3 pass through; other formats are decoded first, then transcoded to WAV under one
+ shared rate ceiling, so the result does not depend on attachment order.
+ Blocking; call via a thread from async paths.
+ """
+ stripped = [_strip_audio_data_uri(clip) for clip in clips]
+ raws = [base64.b64decode(clip) for clip in stripped]
+ passthrough = [_llama_passthrough_audio(raw) for raw in raws]
+ wav_budget = _MAX_AUDIO_RAW_BYTES - sum(
+ len(raw) for raw, kept in zip(raws, passthrough) if kept is not None
+ )
+ total_seconds = 0.0
+ samples_held = 0
+ decoded: dict[int, tuple] = {}
+ for index, (raw, kept) in enumerate(zip(raws, passthrough)):
+ if kept is not None:
+ seconds = kept[1]
+ else:
+ arr, sr = (
+ _decode_within(total_seconds, _decode_audio_mono, raw, samples_used = samples_held)
+ if index
+ else _decode_audio_mono(raw)
+ )
+ decoded[index] = (arr, sr)
+ samples_held += len(arr)
+ seconds = len(arr) / float(sr) if sr else 0.0
+ total_seconds += seconds
+ if total_seconds > _MAX_AUDIO_SECONDS:
raise _DecodedAudioTooLongError(
f"audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
)
- # Headers that cannot state a length do not earn a free pass. Forwarding
- # them anyway meant the cap held only for containers honest enough to
- # describe themselves, which is the wrong way round: four junk bytes in
- # an MPEG stream, or a WAV with no data chunk, ended the header walk and
- # took the whole recording through with it. Decoding costs a transcode
- # and nothing else, and puts the file back under both ceilings, so it
- # still reaches the model.
- if seconds is not None:
- return b64, passthrough
+ shared_rate = (
+ _shared_wav_rate([(len(a), sr) for a, sr in decoded.values()], wav_budget)
+ if len(decoded) > 1
+ else None
+ )
+ prepared: list[tuple[str, str]] = []
+ for index, (clip, kept) in enumerate(zip(stripped, passthrough)):
+ if kept is not None:
+ prepared.append((clip, kept[0]))
+ continue
+ arr, sr = decoded.pop(index)
+ if shared_rate is None:
+ arr, sr = _fit_transcoded_audio_to_wav_cap(arr, sr, cap = wav_budget)
+ elif shared_rate < sr:
+ if shared_rate < _MIN_TRANSCODE_AUDIO_SAMPLE_RATE:
+ raise ValueError("decoded audio exceeds the transcoded WAV size limit")
+ arr, sr = _resample_mono_linear(arr, sr, shared_rate), shared_rate
+ prepared.append((base64.b64encode(_mono_f32_to_wav_bytes(arr, sr)).decode("ascii"), "wav"))
+ return prepared
+
+
+def _shared_wav_rate(clips: list[tuple[int, int]], budget: int) -> int:
+ """Highest rate at which every (samples, rate) clip, capped to it, fits ``budget`` as WAV."""
+
+ def samples_at(n: int, sr: int, rate: int) -> int:
+ # _resample_mono_linear's own output length, so the budget holds exactly.
+ return n if rate >= sr or sr <= 0 else max(1, int(round(n / float(sr) * rate)))
+
+ def total(rate: int) -> int:
+ return sum(_WAV_HEADER_BYTES + 2 * samples_at(n, sr, rate) for n, sr in clips)
+
+ low, high = 0, max(sr for _, sr in clips)
+ if total(high) <= budget:
+ return high
+ while low < high:
+ mid = (low + high + 1) // 2
+ if total(mid) <= budget:
+ low = mid
+ else:
+ high = mid - 1
+ return low
- arr, sr = _decode_audio_mono(raw)
- arr, sr = _fit_transcoded_audio_to_wav_cap(arr, sr)
- return base64.b64encode(_mono_f32_to_wav_bytes(arr, sr)).decode("ascii"), "wav"
+
+def _strip_audio_data_uri(b64: str) -> str:
+ if b64.startswith("data:"):
+ return b64.split(",", 1)[1] if "," in b64 else ""
+ return b64
+
+
+def _llama_passthrough_audio(raw: bytes) -> Optional[tuple[str, float]]:
+ """(container, seconds) if the upload can be forwarded as is, else None."""
+ passthrough = _sniff_audio_container(raw)
+ if passthrough is None:
+ return None
+ # Forwarding skips the bounded decoders: cap duration from headers (16 kbps MP3 = hours).
+ seconds = _passthrough_audio_seconds(raw, passthrough, _MAX_AUDIO_SECONDS)
+ if seconds is not None and seconds > _MAX_AUDIO_SECONDS:
+ raise _DecodedAudioTooLongError(
+ f"audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"
+ )
+ # No stated length (junk MPEG bytes, WAV without data chunk) means decode, not forward.
+ if seconds is None:
+ return None
+ return passthrough, seconds
_VIDEO_INPUT_REFUSAL = (
@@ -23098,7 +23270,7 @@ def _reject_unsupported_content_parts(payload) -> None:
def _reject_misplaced_audio_parts(payload) -> None:
- """Refuse a recording the single ``audio_base64`` field cannot carry faithfully.
+ """Refuse recordings the ``audio_base64`` / ``extra_audio_base64`` fields cannot carry faithfully.
Placement is decided here rather than in the lift because the lift runs behind routing --
the preview route reaches it only after taking the preview lock and loading a checkpoint,
@@ -23125,13 +23297,13 @@ def _reject_misplaced_audio_parts(payload) -> None:
for part in msg.content
if isinstance(part, InputAudioContentPart)
]
- if len(parts) > 1:
+ if len(parts) > _MAX_AUDIO_CLIPS_PER_REQUEST:
_raise_unsupported_openai_parameter(
"messages",
- "Only one audio recording per request is supported, and this one carries "
- f"{len(parts)}.",
+ f"At most {_MAX_AUDIO_CLIPS_PER_REQUEST} audio recordings per request are supported, "
+ f"and this one carries {len(parts)}.",
)
- if parts and parts[0][0] != last_user:
+ if any(index != last_user for index, _ in parts):
_raise_unsupported_openai_parameter(
"messages",
"Audio input is supported on the latest user message, and this one carries it on an "
@@ -23172,18 +23344,18 @@ def _messages_have_embedded_image(messages) -> bool:
def _normalise_chat_content_parts(payload) -> None:
- """Lift the request's ``input_audio`` part onto ``audio_base64``, in place.
+ """Lift the request's ``input_audio`` parts onto ``audio_base64`` / ``extra_audio_base64``.
- Everything that makes audio safe to serve reads that field: the capability check that keeps
+ Everything that makes audio safe to serve reads those fields: the capability check that keeps
a text-only target from being loaded for it, the size bound, the decoder, the duration limit,
- and /chat/count_tokens' refusal. So the part is lifted rather than left standing -- a part the
- field never sees is a recording none of those checks can act on.
+ and /chat/count_tokens' refusal. So the parts are lifted rather than left standing -- a part
+ the fields never see is a recording none of those checks can act on.
- ``_reject_misplaced_audio_parts`` has already refused every shape the single field cannot
- carry faithfully, so what reaches here is one recording on the latest user turn. An explicit
- ``audio_base64`` still wins over a part.
+ ``_reject_misplaced_audio_parts`` has already refused every shape the fields cannot carry
+ faithfully, so what reaches here is up to the clip cap on the latest user turn, in order. An
+ explicit ``audio_base64`` still wins over parts.
"""
- lifted = None
+ lifted: list[str] = []
for msg in payload.messages:
if not isinstance(msg.content, list):
continue
@@ -23191,13 +23363,14 @@ def _normalise_chat_content_parts(payload) -> None:
for part in msg.content:
if isinstance(part, InputAudioContentPart):
if msg.role == "user" and part.input_audio.data:
- lifted = part.input_audio.data
+ lifted.append(part.input_audio.data)
continue
kept.append(part)
if len(kept) != len(msg.content):
msg.content = kept
if lifted and not getattr(payload, "audio_base64", None):
- payload.audio_base64 = lifted
+ payload.audio_base64 = lifted[0]
+ payload.extra_audio_base64 = lifted[1:] or None
def _message_video_urls(messages) -> list[str]:
@@ -26426,8 +26599,9 @@ async def produce_openai_chat_completions(
),
)
# target-independent, unlike the format-dependent checks the switch itself runs.
- if payload.audio_base64 and len(payload.audio_base64) > _MAX_AUDIO_B64_CHARS:
- raise HTTPException(status_code = 413, detail = "Audio file is too large (max ~25 MB).")
+ _audio_rejection = _request_audio_rejection(payload)
+ if _audio_rejection is not None:
+ raise HTTPException(status_code = _audio_rejection[0], detail = _audio_rejection[1])
# Reject streaming n>1 before the switch: only the non-streaming GGUF path
# returns multiple choices, so stream=true + n>1 is invalid on every local
# serving path (the external path already rejected it before its early
@@ -26476,7 +26650,7 @@ async def produce_openai_chat_completions(
# the rest of the audio checks depend on the target's format, so the switch runs them.
_audio_preflight = (
{
- "b64": payload.audio_base64,
+ "clips": _request_audio_clips(payload),
"continue_final": _continue_final_message(payload),
"has_image": _images_in_last_user_message(payload.messages)
or _legacy_image_is_distinct(payload),
@@ -26748,6 +26922,15 @@ async def produce_openai_chat_completions(
)
api_monitor.fail(monitor_id, _audio_unsupported_detail)
raise HTTPException(status_code = 400, detail = _audio_unsupported_detail)
+ # mlx-vlm drops or misplaces extra clips, so refuse rather than ignore them.
+ # The pre-switch check may not have run, so bound clips before decoding.
+ _audio_rejection = _request_audio_rejection(payload)
+ if _audio_rejection is not None:
+ api_monitor.fail(monitor_id, _audio_rejection[1])
+ raise HTTPException(status_code = _audio_rejection[0], detail = _audio_rejection[1])
+ if payload.extra_audio_base64 and model_info.get("is_mlx"):
+ api_monitor.fail(monitor_id, _MLX_MULTI_AUDIO_DETAIL)
+ raise HTTPException(status_code = 400, detail = _MLX_MULTI_AUDIO_DETAIL)
# ── Audio INPUT path: decode WAV and route to audio input generation ──
if payload.audio_base64 and model_info.get("has_audio_input"):
@@ -26770,10 +26953,15 @@ async def produce_openai_chat_completions(
try:
# Decoded before the switch; only a path that skipped that preflight
# (no automatic load could run) still has to do it here.
- audio_array = (
+ audio_arrays = (
_predecoded_audio
if _predecoded_audio is not None
- else _decode_audio_base64(payload.audio_base64)
+ else await asyncio.to_thread(_decode_audio_clips, _request_audio_clips(payload))
+ )
+ audio_array = audio_arrays[0]
+ # Passed only when present, so single-clip calls are unchanged.
+ extra_audio_kwargs = (
+ {"extra_audio_arrays": audio_arrays[1:]} if len(audio_arrays) > 1 else {}
)
system_prompt, chat_messages, _ = await _extract_content_parts_async(
payload.messages
@@ -26814,6 +27002,7 @@ async def produce_openai_chat_completions(
audio_array = audio_array,
cancel_event = cancel_event,
stats_holder = _audio_stats_holder,
+ **extra_audio_kwargs,
)
return backend.generate_audio_input_response(
messages = chat_messages,
@@ -26830,6 +27019,7 @@ async def produce_openai_chat_completions(
cancel_event = cancel_event,
stats_holder = _audio_stats_holder,
stop = normalized_stop,
+ **extra_audio_kwargs,
)
if payload.stream:
@@ -27258,21 +27448,23 @@ async def produce_openai_chat_completions(
# audio encoder); other containers are transcoded to WAV here. The part
# is injected into the message list below so it rides through both the
# plain and tool-calling paths, exactly like image_url parts.
- audio_b64 = None
- audio_format = "wav"
+ prepared_audio: list[tuple[str, str]] = []
if payload.audio_base64:
if not getattr(llama_backend, "_has_audio_input", False):
raise _reject(
400,
"Audio provided but current GGUF model does not support audio input.",
)
- if len(payload.audio_base64) > _MAX_AUDIO_B64_CHARS:
- raise _reject(413, "Audio file is too large (max ~25 MB).")
+ audio_rejection = _request_audio_rejection(payload)
+ if audio_rejection is not None:
+ raise _reject(*audio_rejection)
try:
- audio_b64, audio_format = (
+ prepared_audio = (
_preprepared_audio
if _preprepared_audio is not None
- else await asyncio.to_thread(_prepare_audio_for_llama, payload.audio_base64)
+ else await asyncio.to_thread(
+ _prepare_audio_clips_for_llama, _request_audio_clips(payload)
+ )
)
except _DecodedAudioTooLongError as e:
# A valid file that is simply too long reports the limit, as the
@@ -27282,9 +27474,10 @@ async def produce_openai_chat_completions(
except Exception as e:
logger.warning("Audio decode failed: %s", e, exc_info = True)
raise _reject(400, "Could not decode the provided audio file.")
- # Admission reads the duration from this field's header, which only the forwarded
+ # Admission reads the duration from these fields' headers, which only the forwarded
# wav/mp3 is sure to state; an m4a, ogg or flac upload would be charged by its bytes.
- payload.audio_base64 = audio_b64
+ payload.audio_base64 = prepared_audio[0][0]
+ payload.extra_audio_base64 = [clip for clip, _ in prepared_audio[1:]] or None
# llama-server samples frames but encodes each at the clip's resolution.
video_b64 = None
@@ -27317,7 +27510,7 @@ async def produce_openai_chat_completions(
gguf_messages, request, thread_id = getattr(payload, "thread_id", None)
)
image_b64 = None
- if audio_b64:
+ for audio_b64, audio_format in prepared_audio:
_inject_audio_part(gguf_messages, audio_b64, audio_format)
if video_b64:
_inject_video_part(gguf_messages, video_b64)
@@ -36282,7 +36475,7 @@ async def chat_count_tokens(
detail = "Cannot count tokens for messages containing images.",
)
# Same for audio: the completion injects the recording, this cannot.
- if getattr(payload, "audio_base64", None):
+ if getattr(payload, "audio_base64", None) or getattr(payload, "extra_audio_base64", None):
raise HTTPException(
status_code = 503,
detail = "Cannot count tokens for messages containing audio.",
diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.py
index 2df2a937f1f..96aaa3dbf7d 100644
--- a/studio/backend/tests/test_chat_input_audio_part.py
+++ b/studio/backend/tests/test_chat_input_audio_part.py
@@ -69,17 +69,102 @@ def test_an_explicit_audio_base64_wins():
assert payload.audio_base64 == AUDIO_B64
-def test_two_recordings_are_refused_rather_than_reduced_to_one():
- """``audio_base64`` holds one recording, so a second was silently discarded.
+def _multi_audio_message(*datas, text = "compare these"):
+ return {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": text},
+ *(
+ {"type": "input_audio", "input_audio": {"data": data, "format": "wav"}}
+ for data in datas
+ ),
+ ],
+ }
- A request to compare two clips would have been answered from the last one alone.
- """
+
+def test_several_recordings_on_the_latest_turn_are_all_lifted_in_order():
+ """Models like Gemma 4 take several clips at once; none may be dropped or reordered."""
+ payload = _request(_multi_audio_message("Zmlyc3Q=", "c2Vjb25k", "dGhpcmQ="))
+
+ _reject_unsupported_content_parts(payload)
+ _normalise_chat_content_parts(payload)
+
+ assert payload.audio_base64 == "Zmlyc3Q="
+ assert payload.extra_audio_base64 == ["c2Vjb25k", "dGhpcmQ="]
+ assert inference_route._request_audio_clips(payload) == ["Zmlyc3Q=", "c2Vjb25k", "dGhpcmQ="]
+ assert [p.type for p in payload.messages[0].content] == ["text"]
+
+
+def test_recordings_split_across_turns_are_refused_rather_than_reduced():
+ """Only the latest turn's clips reach the model, so an older clip would be answered without."""
payload = _request(_audio_message(data = "Zmlyc3Q="), _audio_message(data = "c2Vjb25k"))
with pytest.raises(HTTPException) as exc:
_reject_unsupported_content_parts(payload)
assert exc.value.status_code == 400
- assert "one audio recording" in str(exc.value.detail)
+ assert "earlier turn" in str(exc.value.detail)
+
+
+def test_more_recordings_than_the_cap_are_refused():
+ datas = ["QUFB"] * (inference_route._MAX_AUDIO_CLIPS_PER_REQUEST + 1)
+ payload = _request(_multi_audio_message(*datas))
+
+ with pytest.raises(HTTPException) as exc:
+ _reject_unsupported_content_parts(payload)
+ assert exc.value.status_code == 400
+ assert f"At most {inference_route._MAX_AUDIO_CLIPS_PER_REQUEST}" in str(exc.value.detail)
+
+
+def test_extra_audio_alone_is_promoted_onto_the_audio_field():
+ """Every capability and size check keys on audio_base64, so extras can never bypass them."""
+ payload = _request(
+ {"role": "user", "content": "hi"}, extra_audio_base64 = ["", "Zmlyc3Q=", "c2Vjb25k"]
+ )
+
+ assert payload.audio_base64 == "Zmlyc3Q="
+ assert payload.extra_audio_base64 == ["c2Vjb25k"]
+
+
+def test_the_size_cap_covers_all_clips_together(monkeypatch):
+ """N clips never cost more upload or decode memory than one maximal clip does."""
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_B64_CHARS", 10)
+ monkeypatch.setattr(inference_route, "_AUDIO_CLIP_B64_SLACK_CHARS", 2)
+ one = _request({"role": "user", "content": "hi"}, audio_base64 = "A" * 10)
+ assert inference_route._request_audio_rejection(one) is None
+ within_slack = _request(
+ {"role": "user", "content": "hi"}, audio_base64 = "A" * 6, extra_audio_base64 = ["A" * 6]
+ )
+ assert inference_route._request_audio_rejection(within_slack) is None
+
+ two = _request(
+ {"role": "user", "content": "hi"}, audio_base64 = "A" * 7, extra_audio_base64 = ["A" * 6]
+ )
+ status, detail = inference_route._request_audio_rejection(two)
+ assert status == 413
+ assert "all files together" in detail
+
+
+def test_the_count_cap_is_checked_on_the_fields_too():
+ payload = _request(
+ {"role": "user", "content": "hi"},
+ audio_base64 = "QUFB",
+ extra_audio_base64 = ["QUFB"] * inference_route._MAX_AUDIO_CLIPS_PER_REQUEST,
+ )
+ status, detail = inference_route._request_audio_rejection(payload)
+ assert status == 400
+ assert "Too many audio files" in detail
+
+
+def test_the_duration_cap_covers_all_decoded_clips_together(monkeypatch):
+ import numpy as np
+
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)
+ monkeypatch.setattr(
+ inference_route, "_decode_audio_base64", lambda _b64: np.zeros(12_000, np.float32)
+ )
+ assert len(inference_route._decode_audio_clips(["a"])) == 1
+ with pytest.raises(inference_route._DecodedAudioTooLongError):
+ inference_route._decode_audio_clips(["a", "b"])
def test_an_audio_part_on_a_non_user_role_is_refused():
@@ -509,3 +594,239 @@ def test_the_text_only_checkpoint_refusal_precedes_the_branch_that_consumes_audi
# the refusal has to precede the consuming branch, or it never runs
assert branch < consume
assert "cannot read audio input" in source[branch:consume]
+
+
+def _wav_b64(seconds: float, rate: int = 16000) -> str:
+ import base64
+ import io
+ import wave
+
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as wav:
+ wav.setnchannels(1)
+ wav.setsampwidth(2)
+ wav.setframerate(rate)
+ wav.writeframes(b"\x00\x00" * int(seconds * rate))
+ return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+def test_gguf_clips_pass_through_in_order():
+ first, second = _wav_b64(0.5), _wav_b64(0.25)
+ prepared = inference_route._prepare_audio_clips_for_llama(
+ [first, f"data:audio/wav;base64,{second}"]
+ )
+ assert prepared == [(first, "wav"), (second, "wav")]
+
+
+def test_gguf_duration_cap_covers_all_clips_together(monkeypatch):
+ """Each clip alone is under the cap; together they are not, as on the decoded path."""
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)
+ clip = _wav_b64(0.75)
+ assert len(inference_route._prepare_audio_clips_for_llama([clip])) == 1
+ with pytest.raises(inference_route._DecodedAudioTooLongError):
+ inference_route._prepare_audio_clips_for_llama([clip, clip])
+
+
+def test_transcoded_gguf_clips_share_one_wav_budget(monkeypatch):
+ """Several m4a/ogg clips must not each grow into a cap-sized WAV for llama-server."""
+ import base64
+
+ import numpy as np
+
+ caps = []
+
+ def _fit(
+ arr,
+ sr,
+ cap = None,
+ ):
+ caps.append(cap)
+ return arr, sr
+
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_RAW_BYTES", 1000)
+ monkeypatch.setattr(inference_route, "_sniff_audio_container", lambda _raw: None)
+ monkeypatch.setattr(
+ inference_route, "_decode_audio_mono", lambda _raw: (np.zeros(100, np.float32), 16000)
+ )
+ monkeypatch.setattr(inference_route, "_fit_transcoded_audio_to_wav_cap", _fit)
+
+ inference_route._prepare_audio_for_llama("QUFB")
+ assert caps == [1000] # one clip keeps the whole cap, exactly as before
+
+ # Clips that fit together are neither refit nor resampled.
+ caps.clear()
+ prepared = inference_route._prepare_audio_clips_for_llama(["QUFB", "QUFB", "QUFB"])
+ assert caps == []
+ assert [len(base64.b64decode(data)) for data, _ in prepared] == [44 + 100 * 2] * 3
+
+ # Pass-through clips are forwarded at their upload size, so transcodes share what they leave.
+ caps.clear()
+ monkeypatch.setattr(
+ inference_route, "_sniff_audio_container", lambda raw: "wav" if raw == b"WAV!" else None
+ )
+ monkeypatch.setattr(inference_route, "_passthrough_audio_seconds", lambda *_a: 0.1)
+ inference_route._prepare_audio_clips_for_llama(["V0FWIQ==", "QUFB"])
+ assert caps == [1000 - 4]
+
+
+def test_the_wav_budget_does_not_depend_on_clip_order(monkeypatch):
+ """A long clip beside a short one fits by downsampling both to one shared rate.
+
+ An equal split starved the long clip below the minimum rate whenever it came first.
+ """
+ import base64
+ import io
+ import wave
+
+ import numpy as np
+
+ long_clip = np.zeros(10 * 16000, np.float32)
+ short_clip = np.zeros(16000, np.float32)
+ monkeypatch.setattr(inference_route, "_sniff_audio_container", lambda _raw: None)
+ monkeypatch.setattr(
+ inference_route,
+ "_decode_audio_mono",
+ lambda raw: ((long_clip if raw == b"LNG" else short_clip), 16000),
+ )
+ # Room for 11 seconds at 9 kHz, well under the 16 kHz the clips arrive at.
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_RAW_BYTES", 2 * 44 + 2 * 11 * 9000)
+
+ def rates(prepared):
+ out = []
+ for data, fmt in prepared:
+ assert fmt == "wav"
+ with wave.open(io.BytesIO(base64.b64decode(data))) as wav:
+ out.append(wav.getframerate())
+ return out
+
+ long_b64 = base64.b64encode(b"LNG").decode()
+ short_b64 = base64.b64encode(b"SHT").decode()
+ assert rates(inference_route._prepare_audio_clips_for_llama([long_b64, short_b64])) == [
+ 9000,
+ 9000,
+ ]
+ assert rates(inference_route._prepare_audio_clips_for_llama([short_b64, long_b64])) == [
+ 9000,
+ 9000,
+ ]
+
+
+def test_clips_below_the_shared_rate_are_left_alone():
+ n_low, n_high = 8000 * 1200, 48000 * 300
+ budget = 25 * 1024 * 1024
+ rate = inference_route._shared_wav_rate([(n_low, 8000), (n_high, 48000)], budget)
+ # The 8 kHz clip keeps its full size; the 48 kHz one still clears the 8 kHz floor.
+ assert 8000 <= rate < 48000
+ assert 2 * 44 + 2 * n_low + 2 * round(n_high / 48000 * rate) <= budget
+
+
+def test_the_shared_rate_is_used_as_is_at_the_floor(monkeypatch):
+ """A budget that allows exactly 8 kHz must not come out at 7999 Hz and be refused."""
+ import base64
+ import io
+ import wave
+
+ import numpy as np
+
+ sr = 44100
+ lengths = {b"ONE": sr * 7 + 1, b"TWO": sr * 3 + 7}
+ monkeypatch.setattr(inference_route, "_sniff_audio_container", lambda _raw: None)
+ monkeypatch.setattr(
+ inference_route,
+ "_decode_audio_mono",
+ lambda raw: (np.zeros(lengths[raw], np.float32), sr),
+ )
+ budget = sum(44 + 2 * round(n / float(sr) * 8000) for n in lengths.values())
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_RAW_BYTES", budget)
+
+ prepared = inference_route._prepare_audio_clips_for_llama(
+ [base64.b64encode(b"ONE").decode(), base64.b64encode(b"TWO").decode()]
+ )
+ wavs = [base64.b64decode(data) for data, _ in prepared]
+ assert [wave.open(io.BytesIO(w)).getframerate() for w in wavs] == [8000, 8000]
+ assert sum(len(w) for w in wavs) <= budget
+
+
+def test_held_clips_count_against_the_sample_ceiling(monkeypatch):
+ """Decoded clips are held until the budget is split, so they share one sample ceiling."""
+ seen = []
+
+ def _decode(_raw):
+ seen.append(inference_route._decoded_samples_cap())
+ return inference_route._decoded_samples_cap
+
+ monkeypatch.setattr(inference_route, "_MAX_DECODED_SAMPLES", 1000)
+ inference_route._decode_within(0.0, _decode, b"", samples_used = 600)
+ assert seen == [400]
+ with pytest.raises(inference_route._DecodedAudioTooLongError):
+ inference_route._decode_within(0.0, _decode, b"", samples_used = 1000)
+
+
+class _CapturingAudioBackend:
+ def __init__(self):
+ self.calls = []
+
+ def generate_audio_input_response(self, **kwargs):
+ self.calls.append(kwargs)
+ return iter(())
+
+
+def _run_worker_audio(clips):
+ import numpy as np
+ from types import SimpleNamespace
+
+ from core.inference import worker
+
+ backend = _CapturingAudioBackend()
+ sent = []
+ worker._handle_generate_audio_input(
+ backend,
+ {
+ "request_id": "r",
+ "audio_clips": [np.asarray(c, dtype = np.float32).tobytes() for c in clips],
+ },
+ SimpleNamespace(put = lambda item, *a, **k: sent.append(item)),
+ SimpleNamespace(is_set = lambda: False),
+ )
+ assert not [m for m in sent if m.get("type") == "gen_error"], sent
+ return backend.calls[0]
+
+
+def test_a_single_clip_reaches_the_backend_as_before():
+ """A plain list of samples is ONE waveform, never a list of one-sample clips."""
+ call = _run_worker_audio([[0.0, 0.1, -0.1]])
+ assert list(call["audio_array"]) == pytest.approx([0.0, 0.1, -0.1])
+ assert "extra_audio_arrays" not in call
+
+
+def test_extra_clips_reach_the_backend_in_order():
+ call = _run_worker_audio([[0.1], [0.2, 0.2], [0.3]])
+ assert list(call["audio_array"]) == pytest.approx([0.1])
+ assert [list(c) for c in call["extra_audio_arrays"]] == [
+ pytest.approx([0.2, 0.2]),
+ pytest.approx([0.3]),
+ ]
+
+
+def test_extra_clips_join_the_latest_user_turn_in_order():
+ from core.inference.chat_template_helpers import messages_with_attached_image
+
+ history = [
+ {"role": "user", "content": "earlier"},
+ {"role": "assistant", "content": "reply"},
+ {"role": "user", "content": "compare these"},
+ ]
+ rendered = messages_with_attached_image(
+ history, structured_content = True, image = 0, audio = "a", extra_audio = ["b", "c"]
+ )
+ assert rendered[0]["content"] == [{"type": "text", "text": "earlier"}]
+ assert rendered[-1]["content"] == [
+ {"type": "audio", "audio": "a"},
+ {"type": "audio", "audio": "b"},
+ {"type": "audio", "audio": "c"},
+ {"type": "text", "text": "compare these"},
+ ]
+ # No extras: exactly the single-clip render.
+ single = messages_with_attached_image(history, structured_content = True, image = 0, audio = "a")
+ assert single[-1]["content"][0] == {"type": "audio", "audio": "a"}
+ assert len(single[-1]["content"]) == 2
diff --git a/studio/backend/tests/test_llama_audio_attachment_container.py b/studio/backend/tests/test_llama_audio_attachment_container.py
index ee30843f84b..035f0a30d6f 100644
--- a/studio/backend/tests/test_llama_audio_attachment_container.py
+++ b/studio/backend/tests/test_llama_audio_attachment_container.py
@@ -35,12 +35,12 @@ def test_aac_and_mp2_are_transcoded_instead_of_forwarded_as_mp3(monkeypatch):
monkeypatch.setattr(
inference_route,
"_decode_audio_mono",
- lambda raw: (decoded.append(raw) or object(), 16_000),
+ lambda raw: (decoded.append(raw) or np.zeros(0, np.float32), 16_000),
)
monkeypatch.setattr(
inference_route,
"_fit_transcoded_audio_to_wav_cap",
- lambda array, sample_rate: (array, sample_rate),
+ lambda array, sample_rate, **_: (array, sample_rate),
)
monkeypatch.setattr(
inference_route,
@@ -939,13 +939,15 @@ def test_a_switch_preflight_answers_an_overlong_upload_like_the_serving_path(mon
def _too_long(_b64):
raise inference_route._DecodedAudioTooLongError("decoded audio exceeds the limit")
- monkeypatch.setattr(inference_route, "_prepare_audio_for_llama", _too_long)
+ monkeypatch.setattr(inference_route, "_prepare_audio_clips_for_llama", _too_long)
monkeypatch.setattr(inference_route, "_decode_audio_base64", _too_long)
monkeypatch.setattr(inference_route, "_audio_decoder_is_available", lambda: True)
for target_is_gguf in (True, False):
try:
- asyncio.run(inference_route._preflight_audio_for_switch({"b64": "x"}, target_is_gguf))
+ asyncio.run(
+ inference_route._preflight_audio_for_switch({"clips": ["x"]}, target_is_gguf)
+ )
except HTTPException as error:
assert error.status_code == 413, target_is_gguf
assert error.detail == inference_route._audio_too_long_detail()
@@ -962,13 +964,15 @@ def test_a_switch_preflight_still_reports_undecodable_audio_as_a_bad_value(monke
def _broken(_b64):
raise ValueError("not audio at all")
- monkeypatch.setattr(inference_route, "_prepare_audio_for_llama", _broken)
+ monkeypatch.setattr(inference_route, "_prepare_audio_clips_for_llama", _broken)
monkeypatch.setattr(inference_route, "_decode_audio_base64", _broken)
monkeypatch.setattr(inference_route, "_audio_decoder_is_available", lambda: True)
for target_is_gguf in (True, False):
try:
- asyncio.run(inference_route._preflight_audio_for_switch({"b64": "x"}, target_is_gguf))
+ asyncio.run(
+ inference_route._preflight_audio_for_switch({"clips": ["x"]}, target_is_gguf)
+ )
except HTTPException as error:
assert error.status_code == 400, target_is_gguf
else:
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 472fe237925..93bd8b35ffe 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -10820,11 +10820,11 @@ def test_the_audio_preflight_only_binds_a_non_gguf_target(monkeypatch):
audio.setframerate(16000)
audio.writeframes(b"\x00\x00" * 16)
preflight = {
- "b64": f"data:audio/wav;base64,{_b64.b64encode(wav.getvalue()).decode()}",
+ "clips": [f"data:audio/wav;base64,{_b64.b64encode(wav.getvalue()).decode()}"],
"continue_final": True,
}
asyncio.run(inference_route._preflight_audio_for_switch(preflight, True))
- assert preflight["prepared"][1] == "wav"
+ assert preflight["prepared"][0][1] == "wav"
assert "decoded" not in preflight
# the non-GGUF branch runs _decode_audio_base64, so it refuses the same input, before the load.
@@ -10877,7 +10877,7 @@ def test_a_prior_turn_image_does_not_block_a_non_gguf_audio_switch(monkeypatch):
require_image = True,
require_audio_input = True,
audio_preflight = {
- "b64": "valid",
+ "clips": ["valid"],
"continue_final": False,
"has_image": False,
},
@@ -11337,7 +11337,7 @@ def test_mixed_audio_and_image_is_rejected_before_a_non_gguf_switch(monkeypatch)
require_image = True,
require_audio_input = True,
audio_preflight = {
- "b64": "AAAA",
+ "clips": ["AAAA"],
"continue_final": False,
"has_image": True,
},
@@ -11382,7 +11382,7 @@ def test_audio_beside_a_clip_is_rejected_before_a_non_gguf_switch(monkeypatch):
require_audio_input = True,
require_video = True,
audio_preflight = {
- "b64": "AAAA",
+ "clips": ["AAAA"],
"continue_final": True,
"has_image": True,
"has_video": True,
@@ -11422,7 +11422,7 @@ def test_the_gguf_audio_preflight_takes_the_base64_llama_cpp_takes():
try:
asyncio.run(
inference_route._preflight_audio_for_switch(
- {"b64": encoded, "continue_final": False}, True
+ {"clips": [encoded], "continue_final": False}, True
)
)
except HTTPException:
@@ -11432,7 +11432,7 @@ def test_the_gguf_audio_preflight_takes_the_base64_llama_cpp_takes():
with pytest.raises(HTTPException) as exc:
asyncio.run(
inference_route._preflight_audio_for_switch(
- {"b64": bad, "continue_final": False}, True
+ {"clips": [bad], "continue_final": False}, True
)
)
assert exc.value.status_code == 400, bad
@@ -11457,7 +11457,7 @@ def test_non_audio_bytes_are_rejected_before_a_gguf_switch(monkeypatch):
"tester",
require_audio_input = True,
audio_preflight = {
- "b64": _b64.b64encode(b"not audio").decode(),
+ "clips": [_b64.b64encode(b"not audio").decode()],
"continue_final": False,
},
)
@@ -11476,7 +11476,7 @@ def test_a_non_gguf_audio_target_is_refused_without_a_decoder(monkeypatch):
with pytest.raises(HTTPException) as exc:
asyncio.run(
inference_route._preflight_audio_for_switch(
- {"b64": "AAAA", "continue_final": False}, False
+ {"clips": ["AAAA"], "continue_final": False}, False
)
)
assert exc.value.status_code == 400
@@ -11497,14 +11497,14 @@ def test_non_audio_bytes_are_rejected_before_a_non_gguf_switch(monkeypatch):
with pytest.raises(HTTPException) as exc:
asyncio.run(
inference_route._preflight_audio_for_switch(
- {"b64": "AAAA", "continue_final": False}, False
+ {"clips": ["AAAA"], "continue_final": False}, False
)
)
assert exc.value.status_code == 400
- preflight = {"b64": "GOOD", "continue_final": False}
+ preflight = {"clips": ["GOOD"], "continue_final": False}
asyncio.run(inference_route._preflight_audio_for_switch(preflight, False))
- assert preflight["decoded"] == "pcm"
+ assert preflight["decoded"] == ["pcm"]
def test_a_gguf_only_host_does_not_need_torchaudio_to_accept_audio(monkeypatch):
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 94a787d39a0..2151a6cda1d 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -3190,10 +3190,9 @@ const Composer: FC<{
try {
await aui.composer().addAttachment(file);
} catch {
- // Chat-wide, not per file (no audio model, too large, already
- // attached), and every adapter path toasted: stop quietly.
+ // The adapter toasted. Keep going: a later, smaller clip may still fit.
if (stillThisComposer()) cancelQueuedSendRef.current?.();
- return;
+ continue;
}
}
}
@@ -3242,8 +3241,8 @@ const Composer: FC<{
};
}, [nativeAttachmentTargetKey, aui]);
- // Same drain as audio, one queue over: one clip per message, and the send
- // gate has to hold across the read either way.
+ // Same drain as audio, one queue over: video is one clip per message, and the
+ // send gate has to hold across the read either way.
useEffect(() => {
if (!nativeAttachmentTargetKey) {
return;
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 661ad7a4086..abd3dfa9873 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -1698,24 +1698,23 @@ export function messagesUsePrivateContent(messages: RunMessages): boolean {
});
}
-export function findLatestUserAudioBase64(
- messages: RunMessages,
- includePendingAudio = true,
-): string | undefined {
+/** Every clip on the newest user message, in the order it was attached. */
+function latestUserAudioClips(messages: RunMessages): string[] {
+ const clips: string[] = [];
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i];
if (!message || message.role !== "user") continue;
for (const part of message.content ?? []) {
const base64 = extractAudioPartBase64(part);
- if (base64) return base64;
+ if (base64) clips.push(base64);
}
if ("attachments" in message) {
for (const attachment of message.attachments ?? []) {
for (const part of attachment.content ?? []) {
const base64 = extractAudioPartBase64(part);
- if (base64) return base64;
+ if (base64) clips.push(base64);
}
}
}
@@ -1726,6 +1725,15 @@ export function findLatestUserAudioBase64(
// retranscribe the stale clip. Matches the consumed-on-send semantics of the legacy pendingAudio.
break;
}
+ return clips;
+}
+
+export function findLatestUserAudioBase64(
+ messages: RunMessages,
+ includePendingAudio = true,
+): string | undefined {
+ const [first] = latestUserAudioClips(messages);
+ if (first) return first;
const pendingAudio = includePendingAudio
? useChatRuntimeStore.getState().pendingAudioBase64
@@ -1733,6 +1741,14 @@ export function findLatestUserAudioBase64(
return pendingAudio ?? undefined;
}
+/** Clips after the first, for extra_audio_base64. Undefined when empty so text turns stay durable. */
+export function findLatestUserExtraAudioBase64(
+ messages: RunMessages,
+): string[] | undefined {
+ const extra = latestUserAudioClips(messages).slice(1);
+ return extra.length > 0 ? extra : undefined;
+}
+
function extractVideoPartBase64(
part: { type: string } | null | undefined,
): string | undefined {
@@ -5200,6 +5216,7 @@ export function createOpenAIStreamAdapter(
externalModelLabel(params.checkpoint) ||
params.checkpoint,
audio: Boolean(findLatestUserAudioBase64(survivingMessages, false)),
+ audioCount: latestUserAudioClips(survivingMessages).length,
video: Boolean(videoBase64),
});
if (attachedMediaReason) {
@@ -6495,6 +6512,8 @@ export function createOpenAIStreamAdapter(
currentTurnMessages,
!queuedRunSettings && !continuation,
),
+ extra_audio_base64:
+ findLatestUserExtraAudioBase64(currentTurnMessages),
video_base64: findLatestUserVideoBase64(currentTurnMessages),
cancel_id: cancelId,
...(sandboxSessionId ? { session_id: sandboxSessionId } : {}),
diff --git a/studio/frontend/src/features/chat/audio-attachment-adapter.ts b/studio/frontend/src/features/chat/audio-attachment-adapter.ts
index baecd8d2f6b..4325a016bfc 100644
--- a/studio/frontend/src/features/chat/audio-attachment-adapter.ts
+++ b/studio/frontend/src/features/chat/audio-attachment-adapter.ts
@@ -4,7 +4,8 @@
import {
AUDIO_ATTACHMENT_ACCEPT,
fileToBase64,
- getAudioSizeError,
+ getAudioAddError,
+ maxAudioFilesFor,
} from "@/lib/audio-utils";
import type {
Attachment,
@@ -17,13 +18,15 @@ import { externalModelLabel } from "./lib/external-model-label";
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
// crypto.randomUUID is undefined in non-secure contexts (HTTP over a LAN IP).
-function newAttachmentId(): string {
+export function newAttachmentId(): string {
if (typeof globalThis.crypto?.randomUUID === "function") {
return globalThis.crypto.randomUUID();
}
return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
}
+const AUDIO_ADD_TOAST_ID = "audio-attachment-limit";
+
// A loaded model without audio rejects at add(); with none loaded, the send path checks it later.
export class AudioAttachmentAdapter implements AttachmentAdapter {
// MIME is unreliable for some containers (m4a), so also match by extension. No .webm extension:
@@ -31,7 +34,8 @@ export class AudioAttachmentAdapter implements AttachmentAdapter {
// MIME. .mp4 and .m4v stay off for the same reason; only .m4a is audio-only. Not the picker list:
// this decides routing, and .3gp is in that list only so a dialog can offer a recording.
accept = AUDIO_ATTACHMENT_ACCEPT;
- private readonly attachmentIds = new Set<string>();
+ // Pending clip sizes by id; caps cover all clips in a message.
+ private readonly attachmentSizes = new Map<string, number>();
async add({ file }: { file: File }): Promise<PendingAttachment> {
const state = useChatRuntimeStore.getState();
@@ -50,22 +54,32 @@ export class AudioAttachmentAdapter implements AttachmentAdapter {
unavailableReason = `${label} cannot accept audio. Load an audio-input model before attaching audio files.`;
}
if (unavailableReason) {
- toast.error(unavailableReason);
+ toast.error(unavailableReason, { id: AUDIO_ADD_TOAST_ID });
throw new Error(unavailableReason);
}
- const sizeReason = getAudioSizeError(file.size);
- if (sizeReason) {
- toast.error(sizeReason);
- throw new Error(sizeReason);
+ // A staged store clip would be dropped if sent alongside attachments.
+ if (state.pendingAudioBase64) {
+ const stagedReason =
+ "Send or remove the staged audio clip before attaching more audio.";
+ toast.error(stagedReason, { id: AUDIO_ADD_TOAST_ID });
+ throw new Error(stagedReason);
}
- if (this.attachmentIds.size > 0 || state.pendingAudioBase64) {
- const duplicateReason = "Only one audio file can be attached per message.";
- toast.error(duplicateReason);
- throw new Error(duplicateReason);
+ let totalSize = 0;
+ for (const size of this.attachmentSizes.values()) totalSize += size;
+ const addReason = getAudioAddError(
+ this.attachmentSizes.size,
+ totalSize,
+ file.size,
+ maxAudioFilesFor(modelLoaded ? activeModel : undefined),
+ );
+ if (addReason) {
+ // Shared id so a large batch toasts once.
+ toast.error(addReason, { id: AUDIO_ADD_TOAST_ID });
+ throw new Error(addReason);
}
const id = newAttachmentId();
- this.attachmentIds.add(id);
+ this.attachmentSizes.set(id, file.size);
return {
id,
type: "file",
@@ -90,12 +104,12 @@ export class AudioAttachmentAdapter implements AttachmentAdapter {
status: { type: "complete" },
};
} finally {
- this.attachmentIds.delete(attachment.id);
+ this.attachmentSizes.delete(attachment.id);
}
}
remove(attachment: Attachment): Promise<void> {
- this.attachmentIds.delete(attachment.id);
+ this.attachmentSizes.delete(attachment.id);
return Promise.resolve();
}
}
diff --git a/studio/frontend/src/features/chat/lib/attached-media-gate.ts b/studio/frontend/src/features/chat/lib/attached-media-gate.ts
index 018a42be6ee..788005180b9 100644
--- a/studio/frontend/src/features/chat/lib/attached-media-gate.ts
+++ b/studio/frontend/src/features/chat/lib/attached-media-gate.ts
@@ -6,18 +6,26 @@ export function attachedMediaUnavailableReason({
checkpoint,
modelLabel,
audio,
+ audioCount = audio ? 1 : 0,
video,
}: {
- activeModel: { hasAudioInput?: boolean; hasVideoInput?: boolean } | undefined;
+ activeModel:
+ | { hasAudioInput?: boolean; hasVideoInput?: boolean; isMlx?: boolean }
+ | undefined;
checkpoint: string | null | undefined;
modelLabel: string;
audio: boolean;
+ audioCount?: number;
video: boolean;
}): string | null {
if (!checkpoint || (!audio && !video)) return null;
if (audio && !activeModel?.hasAudioInput) {
return `${modelLabel} cannot accept audio. Load an audio-input model, or remove the audio file.`;
}
+ // MLX takes one clip per message (see maxAudioFilesFor).
+ if (audio && audioCount > 1 && activeModel?.isMlx) {
+ return `${modelLabel} takes one audio file per message. Remove the extra audio files, or load a GGUF model to send several.`;
+ }
if (video && !activeModel?.hasVideoInput) {
return `${modelLabel} cannot accept video. Load a model that reads video, or remove the video.`;
}
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 0bdbb5b9271..b4fe427a3d4 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -61,10 +61,11 @@ import {
AUDIO_PICKER_ACCEPT,
isAudioAttachmentFile,
fileToBase64,
- getAudioSizeError,
+ getAudioAddError,
} from "@/lib/audio-utils";
import { isTauri } from "@/lib/api-base";
import { classifiedAttachmentFiles, isVideoFile } from "@/lib/video-utils";
+import { newAttachmentId } from "./audio-attachment-adapter";
import { isDownloadCancelled } from "@/lib/native-files";
import { isMultimodalResponse } from "./types/api";
import { getImageInputUnavailableReason } from "./utils/image-input-support";
@@ -632,14 +633,19 @@ export function SharedComposer({
const [comparing, setComparing] = useState(false);
const [pendingImages, setPendingImages] = useState<PendingImage[]>([]);
const [convertingImages, setConvertingImages] = useState(0);
- const [pendingAudio, setPendingAudio] = useState<{
- name: string;
- base64: string;
- contentType: string;
- } | null>(null);
+ const [pendingAudio, setPendingAudio] = useState<
+ {
+ id: string;
+ name: string;
+ base64: string;
+ contentType: string;
+ size: number;
+ }[]
+ >([]);
const textRef = useRef(text);
const pendingImagesRef = useRef(pendingImages);
const pendingAudioRef = useRef(pendingAudio);
+ const readingAudioRef = useRef(new Map<string, number>());
const setCurrentText = useCallback(
(value: string | ((previous: string) => string)) => {
const next =
@@ -649,8 +655,6 @@ export function SharedComposer({
},
[],
);
- // Audio files still being read into base64, which pendingAudio cannot see yet.
- const audioDecodingRef = useRef(0);
// Attachments still classifying or converting; a ref so runPromptList reads it synchronously.
const attachingRef = useRef(0);
useEffect(() => {
@@ -993,6 +997,18 @@ export function SharedComposer({
const clearPendingAudioStore = useChatRuntimeStore(
(s) => s.clearPendingAudio,
);
+ // Mirror the first clip into the store's single-clip slot.
+ const mirroredAudioRef = useRef(false);
+ useEffect(() => {
+ const [first] = pendingAudio;
+ if (first) {
+ setPendingAudioStore(first.base64, first.name);
+ mirroredAudioRef.current = true;
+ } else if (mirroredAudioRef.current) {
+ clearPendingAudioStore();
+ mirroredAudioRef.current = false;
+ }
+ }, [pendingAudio, setPendingAudioStore, clearPendingAudioStore]);
const {
isDictating,
@@ -1114,30 +1130,46 @@ export function SharedComposer({
const files = await classifiedAttachmentFiles(input);
const next: PendingImage[] = [];
let droppedImageForUnavailable = false;
- let audioSizeError: string | null = null;
+ let audioAddError: string | null = null;
let videoUnsupported = false;
let conversionError: string | null = null;
for (let i = 0; i < files.length; i++) {
const file = files[i];
if (!file) continue;
if (isAudioAttachmentFile(file)) {
- const sizeError = getAudioSizeError(file.size);
- if (sizeError) {
- audioSizeError ??= sizeError;
+ // Caps count staged clips and ones still being read by any batch.
+ const counted = [
+ ...pendingAudioRef.current.map((clip) => clip.size),
+ ...readingAudioRef.current.values(),
+ ];
+ const addError = getAudioAddError(
+ counted.length,
+ counted.reduce((total, size) => total + size, 0),
+ file.size,
+ );
+ if (addError) {
+ audioAddError ??= addError;
continue;
}
- audioDecodingRef.current += 1;
- fileToBase64(file)
- .then((base64) => {
- setPendingAudio({ name: file.name, base64, contentType: file.type });
- setPendingAudioStore(base64, file.name);
- })
- .catch(() => {
- toast.error("Could not read that audio file.");
- })
- .finally(() => {
- audioDecodingRef.current -= 1;
- });
+ const clip = {
+ id: newAttachmentId(),
+ name: file.name,
+ base64: "",
+ contentType: file.type,
+ size: file.size,
+ };
+ readingAudioRef.current.set(clip.id, clip.size);
+ try {
+ clip.base64 = await fileToBase64(file);
+ } catch {
+ toast.error("Could not read that audio file.");
+ continue;
+ } finally {
+ readingAudioRef.current.delete(clip.id);
+ }
+ // Updated now, not on commit, so the next check already counts it.
+ pendingAudioRef.current = [...pendingAudioRef.current, clip];
+ setPendingAudio((prev) => [...prev, clip]);
continue;
}
// video_base64 targets the single loaded GGUF, so at most one side of a compare could answer. Say
@@ -1168,8 +1200,8 @@ export function SharedComposer({
if (droppedImageForUnavailable && attachUnavailableReason) {
toast.error(attachUnavailableReason);
}
- if (audioSizeError) {
- toast.error(audioSizeError);
+ if (audioAddError) {
+ toast.error(audioAddError);
}
if (conversionError) {
toast.error(conversionError);
@@ -1181,15 +1213,19 @@ export function SharedComposer({
}
setPendingImages((prev) => [...prev, ...next]);
},
- [setPendingAudioStore, attachUnavailableReason],
+ [attachUnavailableReason],
);
+ // State mirror of attachingRef: holds sends until a whole batch is read.
+ const [addingFiles, setAddingFiles] = useState(0);
const trackAttaching = useCallback(async (work: () => Promise<void>) => {
attachingRef.current += 1;
+ setAddingFiles((count) => count + 1);
try {
await work();
} finally {
attachingRef.current -= 1;
+ setAddingFiles((count) => count - 1);
}
}, []);
@@ -1265,7 +1301,7 @@ export function SharedComposer({
useEffect(() => () => clearStuckImeTimer(), []);
async function send() {
- if (composingRef.current || convertingImages > 0) {
+ if (composingRef.current || convertingImages > 0 || addingFiles > 0) {
resetPromptQueue();
return;
}
@@ -1273,7 +1309,7 @@ export function SharedComposer({
const submittedImages = pendingImages;
const submittedAudio = pendingAudio;
const msg = submittedText.trim();
- if (!msg && submittedImages.length === 0 && !submittedAudio) {
+ if (!msg && submittedImages.length === 0 && submittedAudio.length === 0) {
resetPromptQueue();
return;
}
@@ -1326,11 +1362,11 @@ export function SharedComposer({
} catch {
}
}
- if (submittedAudio) {
+ for (const clip of submittedAudio) {
content.push({
type: "audio",
- name: submittedAudio.name,
- audio: `data:${submittedAudio.contentType};base64,${submittedAudio.base64}`,
+ name: clip.name,
+ audio: `data:${clip.contentType};base64,${clip.base64}`,
});
}
if (msg) {
@@ -1386,7 +1422,7 @@ export function SharedComposer({
const clearSubmittedDraft = () => {
setCurrentText("");
setPendingImages([]);
- setPendingAudio(null);
+ setPendingAudio([]);
clearPendingAudioStore();
textareaRef.current?.focus();
};
@@ -2217,11 +2253,12 @@ export function SharedComposer({
const canSend =
(text.trim().length > 0 ||
pendingImages.length > 0 ||
- pendingAudio !== null) &&
+ pendingAudio.length > 0) &&
!busy &&
!isComposing &&
!isDictating &&
convertingImages === 0 &&
+ addingFiles === 0 &&
!sendUnavailableReason;
// Compare mode swaps this composer in for the single-chat one and only one is ever on screen, so the
@@ -2311,8 +2348,7 @@ export function SharedComposer({
// Only the first send would carry staged attachments; refuse rather than clear.
if (
pendingImagesRef.current.length > 0 ||
- pendingAudioRef.current ||
- audioDecodingRef.current > 0 ||
+ pendingAudioRef.current.length > 0 ||
attachingRef.current > 0
) {
toast.error("Remove the staged attachment before running a list", {
@@ -2583,20 +2619,22 @@ export function SharedComposer({
onRemove={() => removePendingImage(id)}
/>
))}
- {pendingAudio && (
+ {pendingAudio.map((clip) => (
<div
+ key={clip.id}
data-composer-attachment="audio"
className="flex items-center gap-2 rounded-lg border border-[color-mix(in_oklab,var(--foreground)_calc(20%*var(--contrast-edge-gain,1)),transparent)] bg-muted px-3 py-1.5 text-xs"
>
<HeadphonesIcon className="size-3.5 text-muted-foreground" />
<span data-reload-snapshot-sensitive className="max-w-48 truncate">
- {pendingAudio.name}
+ {clip.name}
</span>
<button
type="button"
onClick={() => {
- setPendingAudio(null);
- clearPendingAudioStore();
+ setPendingAudio((prev) =>
+ prev.filter((other) => other.id !== clip.id),
+ );
}}
className="flex size-4 items-center justify-center rounded-full hover:bg-destructive hover:text-destructive-foreground"
aria-label="Remove audio"
@@ -2604,7 +2642,7 @@ export function SharedComposer({
<XIcon className="size-3" />
</button>
</div>
- )}
+ ))}
</div>
{skillMentions.popover}
<ComposerDraftPreview text={text} />
@@ -2684,6 +2722,7 @@ export function SharedComposer({
ref={audioInputRef}
type="file"
accept={AUDIO_PICKER_ACCEPT}
+ multiple
className="hidden"
onChange={(e) => {
void addFiles(e.target.files);
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 029491032ef..f2810d00aca 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -625,6 +625,8 @@ export interface OpenAIChatCompletionsRequest {
seed?: number;
image_base64?: string;
audio_base64?: string;
+ /** Further clips after audio_base64, in attach order. */
+ extra_audio_base64?: string[];
video_base64?: string;
use_adapter?: boolean | string | null;
enable_thinking?: boolean | null;
diff --git a/studio/frontend/src/features/native-intents/drop-paths.ts b/studio/frontend/src/features/native-intents/drop-paths.ts
index eee7677e064..df322210d76 100644
--- a/studio/frontend/src/features/native-intents/drop-paths.ts
+++ b/studio/frontend/src/features/native-intents/drop-paths.ts
@@ -15,6 +15,7 @@ import {
isTextAttachmentName,
} from "../chat/text-attachment-accept.ts";
import { RAG_UPLOAD_ACCEPT } from "../rag/types/rag.ts";
+import { MAX_AUDIO_FILES } from "../../lib/audio-utils.ts";
const DOC_EXTS = RAG_UPLOAD_ACCEPT.split(",").map((ext) =>
ext.trim().toLowerCase(),
@@ -70,7 +71,7 @@ const VIDEO_EXTS = CHAT_VIDEO_DROP_ACCEPT.split(",").map((ext) =>
);
/** What the window actually takes, for the rejection toast and the overlay. */
-export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, one of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;
+export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, up to ${MAX_AUDIO_FILES} of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;
/** Last path segment of a native path, for display and extension checks. */
export function nativeFileName(path: string): string {
@@ -128,8 +129,8 @@ export function classifyDropPaths(paths: string[]): NativeDropClass {
) {
return { kind: "unsupported" };
}
- // The audio adapter takes one clip per message; a larger batch never attaches.
- if (audio.length > 1) {
+ // Over the per-message cap a batch would only partly attach. The adapter applies per-model limits.
+ if (audio.length > MAX_AUDIO_FILES) {
return { kind: "unsupported" };
}
// Same for video: one clip expands into a run of frames, so a batch would
diff --git a/studio/frontend/src/lib/audio-utils.ts b/studio/frontend/src/lib/audio-utils.ts
index bdf95412d93..6944f51bdf5 100644
--- a/studio/frontend/src/lib/audio-utils.ts
+++ b/studio/frontend/src/lib/audio-utils.ts
@@ -47,6 +47,31 @@ export function getAudioSizeError(size: number): string | null {
: null;
}
+// Keep in sync with _MAX_AUDIO_CLIPS_PER_REQUEST. MAX_AUDIO_SIZE covers all clips together.
+export const MAX_AUDIO_FILES = 8;
+
+/** Max clips per message for a model. MLX takes one. */
+export function maxAudioFilesFor(model: { isMlx?: boolean } | undefined): number {
+ return model?.isMlx ? 1 : MAX_AUDIO_FILES;
+}
+
+export function getAudioAddError(
+ count: number,
+ totalSize: number,
+ size: number,
+ maxFiles: number = MAX_AUDIO_FILES,
+): string | null {
+ if (count >= maxFiles) {
+ return maxFiles === 1
+ ? "This model takes one audio file per message. Load a GGUF model to send several."
+ : `Up to ${maxFiles} audio files can be attached per message.`;
+ }
+ if (count > 0 && totalSize + size > MAX_AUDIO_SIZE) {
+ return `Audio files together exceed the ${MAX_AUDIO_SIZE_LABEL} per-message limit`;
+ }
+ return getAudioSizeError(size);
+}
+
export function fileToBase64(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
diff --git a/studio/frontend/tests/chat-attachment-display.test.ts b/studio/frontend/tests/chat-attachment-display.test.ts
index 5235bd34c8d..721c234fc80 100644
--- a/studio/frontend/tests/chat-attachment-display.test.ts
+++ b/studio/frontend/tests/chat-attachment-display.test.ts
@@ -6,6 +6,12 @@ import assert from "node:assert/strict";
import test from "node:test";
import { attachedMediaUnavailableReason } from "../src/features/chat/lib/attached-media-gate.ts";
+import {
+ getAudioAddError,
+ MAX_AUDIO_FILES,
+ MAX_AUDIO_SIZE,
+ maxAudioFilesFor,
+} from "../src/lib/audio-utils.ts";
import {
ATTACHMENT_KIND_ICON_CLASS,
attachmentFileKind,
@@ -184,6 +190,44 @@ test("audio or video attached before a model loaded is checked when sent", () =>
);
});
+test("several clips are sent to GGUF and transformers, but refused on MLX", () => {
+ const base = { checkpoint: "unsloth/gemma-4-E4B-it", modelLabel: "Gemma" };
+ const gguf = { hasAudioInput: true, isMlx: false };
+ const mlx = { hasAudioInput: true, isMlx: true };
+ assert.equal(
+ attachedMediaUnavailableReason({ ...base, activeModel: gguf, audio: true, audioCount: 3, video: false }),
+ null,
+ );
+ assert.equal(
+ attachedMediaUnavailableReason({ ...base, activeModel: mlx, audio: true, audioCount: 1, video: false }),
+ null,
+ );
+ assert.match(
+ attachedMediaUnavailableReason({ ...base, activeModel: mlx, audio: true, audioCount: 2, video: false }) ?? "",
+ /^Gemma takes one audio file per message\./,
+ );
+});
+
+test("audio caps cover a message's clips together", () => {
+ assert.equal(getAudioAddError(0, 0, 1024), null);
+ assert.equal(getAudioAddError(3, 10 * 1024 * 1024, 10 * 1024 * 1024), null);
+ assert.match(
+ getAudioAddError(1, 20 * 1024 * 1024, 10 * 1024 * 1024) ?? "",
+ /together exceed/,
+ );
+ assert.match(getAudioAddError(0, 0, MAX_AUDIO_SIZE + 1) ?? "", /exceeds/);
+ assert.match(
+ getAudioAddError(MAX_AUDIO_FILES, 0, 1) ?? "",
+ new RegExp(`Up to ${MAX_AUDIO_FILES} audio files`),
+ );
+ assert.equal(maxAudioFilesFor({ isMlx: true }), 1);
+ assert.equal(maxAudioFilesFor(undefined), MAX_AUDIO_FILES);
+ assert.match(
+ getAudioAddError(1, 0, 1, maxAudioFilesFor({ isMlx: true })) ?? "",
+ /one audio file per message/,
+ );
+});
+
test("attaching no longer needs a loaded model, only a capable one when one is loaded", async () => {
const audio = await readSrcAsync("features/chat/audio-attachment-adapter.ts");
const video = await readSrcAsync("features/chat/video-attachment-adapter.ts");
diff --git a/studio/frontend/tests/native-drop-paths.test.ts b/studio/frontend/tests/native-drop-paths.test.ts
index 27c715077ff..88ff8e20004 100644
--- a/studio/frontend/tests/native-drop-paths.test.ts
+++ b/studio/frontend/tests/native-drop-paths.test.ts
@@ -53,6 +53,7 @@ import {
} from "../src/features/native-intents/drop-paths.ts";
import type { NativeIntent } from "../src/features/native-intents/types.ts";
import { RAG_UPLOAD_ACCEPT } from "../src/features/rag/types/rag.ts";
+import { MAX_AUDIO_FILES } from "../src/lib/audio-utils.ts";
import { MAX_REFERENCE_BYTES } from "../src/features/video/reference-budget.ts";
import {
VIDEO_ACCEPT,
@@ -535,22 +536,39 @@ test("a single audio file routes to chat audio attachments", () => {
]);
});
-// One clip per message, so a larger batch is turned away before it is read.
-test("multi-audio drops are rejected before they are routed", () => {
+// Models like Gemma 4 take several clips in one message, so a batch routes whole.
+test("multi-audio drops route to chat audio attachments", () => {
const dropped = classifyDropPaths([
"/clips/take.WAV",
"/clips/note.mp3",
"/clips/voice.flac",
]);
- assert.equal(dropped.kind, "unsupported");
+ assert.equal(dropped.kind, "audio");
+ assert.deepEqual(dropped.kind === "audio" ? dropped.paths : [], [
+ "/clips/take.WAV",
+ "/clips/note.mp3",
+ "/clips/voice.flac",
+ ]);
});
-test("a second clip alongside other attachments is rejected too", () => {
+test("several clips alongside other attachments route together", () => {
const dropped = classifyDropPaths([
"/docs/a.pdf",
"/clips/note.mp3",
"/clips/voice.flac",
]);
+ assert.equal(dropped.kind, "attach");
+ assert.deepEqual(dropped.kind === "attach" ? dropped.audio : [], [
+ "/clips/note.mp3",
+ "/clips/voice.flac",
+ ]);
+});
+
+// Past the per-message cap a batch would attach only partly, so it is turned away before it is read.
+test("an audio batch over the per-message cap is rejected", () => {
+ const dropped = classifyDropPaths(
+ Array.from({ length: MAX_AUDIO_FILES + 1 }, (_, i) => `/clips/${i}.wav`),
+ );
assert.equal(dropped.kind, "unsupported");
});
diff --git a/studio/frontend/tests/reload-snapshot.test.ts b/studio/frontend/tests/reload-snapshot.test.ts
index 8ac8a198bc8..117688ae680 100644
--- a/studio/frontend/tests/reload-snapshot.test.ts
+++ b/studio/frontend/tests/reload-snapshot.test.ts
@@ -1391,7 +1391,7 @@ test("carries live form state, except what sensitive fields hide", () => {
);
assert.match(
sharedComposerSource,
- /data-reload-snapshot-sensitive[\s\S]*?pendingAudio\.name/,
+ /pendingAudio\.map\([\s\S]*?data-reload-snapshot-sensitive[\s\S]*?\{clip\.name\}/,
);
// Both carriers: the tooltip on the name, and the accessible name on the
// remove button, a sibling no ancestor marker would reach.