Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: accept multiple audio files per message (#12267)
Michael Hanba64bcbce674
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex a9dd6e0f7e6..202c64d90a1 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -14,7 +14,7 @@ import reimport stringimport weakreffrom dataclasses import dataclass-from typing import Any, Optional+from typing import Any, Optional, Sequencelogger = logging.getLogger(__name__)@@ -2955,13 +2955,15 @@ def messages_with_attached_image(image: int = 1,video: bool = False,audio: Any = None,+ extra_audio: Sequence[Any] = (),) -> list:"""The conversation to render for a turn that carries attached media.Prepends *system_prompt* as a leading system turn, then injects *image* ``{"type": "image"}``parts, or a ``{"type": "video"}`` part, plus any *audio* waveform as an ``{"type": "audio"}``- part, into the LAST user turn and leaves every other turn --- assistant ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.+ part (one more per *extra_audio* clip, in order), into the LAST user turn and leaves every+ other turn -- assistant ``tool_calls`` and ``role="tool"`` results included -- exactly as the+ caller sent it.Rebuilding from the newest user TEXT instead dropped the folded system instruction and thetool history an OpenAI tool loop replays (#10092). Nothing the caller owns is mutated: callersstill read those dicts after generation, and a retry re-renders the same list.@@ -3014,6 +3016,7 @@ def messages_with_attached_image(]if audio is not None:parts.append({"type": "audio", "audio": audio})+ parts.extend({"type": "audio", "audio": clip} for clip in extra_audio)if not parts and not fallback_user_text:return conversationfor index in range(len(conversation) - 1, -1, -1):diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 671974a3856..3fa64bfae8b 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -1995,6 +1995,7 @@ class InferenceBackend:repetition_penalty,use_adapter: Optional[Union[bool, str]] = None,cancel_event = None,+ extra_audio_arrays: Optional[list] = None,) -> Generator[str, None, None]:"""Audio-input (ASR) generation: takes an audio numpy array, streams text.@@ -2015,7 +2016,7 @@ class InferenceBackend:if not system_prompt:system_prompt = "You are an assistant that transcribes speech accurately."- # Gemma 3n format — audio goes INTO apply_chat_template+ # Gemma 3n format: audio goes INTO apply_chat_template, one item per clip in order.audio_messages = messages_with_attached_image(alternating_turns(messages),system_prompt = system_prompt,@@ -2023,6 +2024,7 @@ class InferenceBackend:structured_content = True,image = 0,audio = audio_array,+ extra_audio = extra_audio_arrays or (),)# Direct processor render like the vision path, so neutralize here too, with@@ -2156,6 +2158,7 @@ class InferenceBackend:self,audio_array,cancel_event = None,+ extra_audio_arrays: Optional[list] = None,) -> Generator[str, None, None]:"""Whisper ASR: takes an audio numpy array, yields transcribed text through the pipelinebuilt at model load."""@@ -2169,13 +2172,17 @@ class InferenceBackend:yield "Error: Whisper pipeline not initialized"return+ clips = [audio_array, *(extra_audio_arrays or [])]try:- with self._generation_lock:- result = whisper_pipe({"raw": audio_array, "sampling_rate": 16000})+ for index, clip in enumerate(clips):+ if cancel_event is not None and cancel_event.is_set():+ return+ with self._generation_lock:+ result = whisper_pipe({"raw": clip, "sampling_rate": 16000})- text = result.get("text", "") if isinstance(result, dict) else str(result)- if text:- yield text+ text = result.get("text", "") if isinstance(result, dict) else str(result)+ if text:+ yield f"\n\n{text}" if index else textexcept Exception as e:logger.error(f"Whisper ASR error: {e}")yield f"Error: {str(e)}"diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 341de5b5ae2..486559598f6 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -6406,6 +6406,7 @@ class MLXInferenceBackend:use_adapter = None,cancel_event = None,stop = None,+ extra_audio_arrays = None,**_sampler,):"""Audio-input chat (omni models): waveform in, incremental text deltas out (the audio route@@ -6418,6 +6419,10 @@ class MLXInferenceBackend:"no verified audio-capable tower/processor was detected at load.")+ # mlx-vlm silently drops all but the first clip; the route refuses this earlier.+ if extra_audio_arrays:+ raise RuntimeError("This MLX model takes one audio file per message.")+from mlx_vlm import stream_generate as vlm_streamif not system_prompt:diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 65399c1b681..e94437d90f5 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -3319,6 +3319,7 @@ class InferenceOrchestrator:audio_array,cancel_event = None,stats_holder: Optional[dict] = None,+ extra_audio_arrays: Optional[list] = None,) -> Generator[str, None, None]:"""Whisper ASR: sends audio to the subprocess and yields text."""yield from self._generate_audio_input_inner(@@ -3328,6 +3329,7 @@ class InferenceOrchestrator:system_prompt = "",cancel_event = cancel_event,stats_holder = stats_holder,+ extra_audio_arrays = extra_audio_arrays,)def generate_audio_input_response(@@ -3345,6 +3347,7 @@ class InferenceOrchestrator:cancel_event = None,stats_holder: Optional[dict] = None,stop = None,+ extra_audio_arrays: Optional[list] = None,) -> Generator[str, None, None]:"""Audio input generation (e.g. Gemma 3n): streams text tokens."""yield from self._generate_audio_input_inner(@@ -3362,6 +3365,7 @@ class InferenceOrchestrator:cancel_event = cancel_event,stats_holder = stats_holder,stop = stop,+ extra_audio_arrays = extra_audio_arrays,)def _generate_audio_input_inner(@@ -3380,6 +3384,7 @@ class InferenceOrchestrator:cancel_event = None,stats_holder: Optional[dict] = None,stop = None,+ extra_audio_arrays: Optional[list] = None,) -> Generator[str, None, None]:"""Shared inner logic for audio input generation (Whisper + ASR). ``stats_holder``: as ingenerate_chat_response, caller-owned and filled on gen_done with the worker's usage /@@ -3402,15 +3407,16 @@ class InferenceOrchestrator:returnrequest_id = str(uuid.uuid4())- # numpy array -> list for mp.Queue serialization- audio_data = (- audio_array.tolist() if hasattr(audio_array, "tolist") else list(audio_array)- )+ import numpy as np++ # Raw float32 bytes per clip; far cheaper to pickle than tolist().+ clips = [audio_array, *(extra_audio_arrays or [])]+ audio_clips = [np.asarray(clip, dtype = np.float32).tobytes() for clip in clips]cmd = {"type": "generate_audio_input","request_id": request_id,- "audio_data": audio_data,+ "audio_clips": audio_clips,"audio_type": audio_type,"messages": messages or [],"system_prompt": system_prompt,diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 70c44c563fe..6475449246f 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1635,8 +1635,15 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_evetry:import numpy as np- # numpy arrays can't go through mp.Queue, so decode from list.- audio_array = np.array(cmd["audio_data"], dtype = np.float32)+ # "audio_data" is the older single-clip list form.+ if "audio_clips" in cmd:+ # Copy: frombuffer views are read-only.+ clips = [np.frombuffer(clip, dtype = np.float32).copy() for clip in cmd["audio_clips"]]+ else:+ clips = [np.array(cmd["audio_data"], dtype = np.float32)]+ audio_array = clips[0]+ # Passed only when present, so single-clip calls are unchanged.+ extra_audio_kwargs = {"extra_audio_arrays": clips[1:]} if len(clips) > 1 else {}audio_type = cmd.get("audio_type")@@ -1647,6 +1654,7 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_evegenerator = backend.generate_whisper_response(audio_array = audio_array,cancel_event = cancel_event,+ **extra_audio_kwargs,)else:audio_kwargs = {@@ -1670,7 +1678,7 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_evebackend, "stop", "generate_audio_input_response"):audio_kwargs["stop"] = cmd["stop"]- generator = backend.generate_audio_input_response(**audio_kwargs)+ generator = backend.generate_audio_input_response(**audio_kwargs, **extra_audio_kwargs)logger.info("Starting audio input generation for request_id=%s", request_id)diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 9dd446d418a..d25c1e15a77 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -2403,6 +2403,13 @@ class ChatCompletionRequest(BaseModel):None,description = "[x-unsloth] Base64-encoded audio (wav/mp3/ogg/flac/m4a) for audio-input models",)+ extra_audio_base64: Optional[List[str]] = Field(+ None,+ description = (+ "[x-unsloth] Further recordings after audio_base64, in order, for models that take "+ "several clips in one message. Size and duration caps apply to all clips together."+ ),+ )audio_instructions: Optional[str] = Field(None,description = (@@ -2841,6 +2848,19 @@ class ChatCompletionRequest(BaseModel):msg.tool_call_id = pickedreturn self+ @model_validator(mode = "after")+ def _promote_extra_audio(self) -> "ChatCompletionRequest":+ """Keep ``audio_base64`` the first clip whenever any clip is attached.++ Every capability, size and routing check keys on that field, so a request carrying+ only ``extra_audio_base64`` must not slip past them as audio-free.+ """+ extra = [clip for clip in self.extra_audio_base64 or [] if clip]+ if not self.audio_base64 and extra:+ self.audio_base64 = extra.pop(0)+ self.extra_audio_base64 = extra or None+ return self+@model_validator(mode = "after")def _map_thinking_to_enable_thinking(self) -> "ChatCompletionRequest":return resolve_thinking_onto_enable_thinking(self)diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.pyindex 99e1e7f5a45..e09db90a9d4 100644--- a/studio/backend/routes/chat_generation_runs.py+++ b/studio/backend/routes/chat_generation_runs.py@@ -71,6 +71,7 @@ _EXTERNAL_ROUTING_FIELDS = {_MEDIA_FIELDS = {"image_base64","audio_base64",+ "extra_audio_base64","video_base64",}_SQLITE_MAX_INTEGER = 9_223_372_036_854_775_807diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex af36406f0b1..dc01bf014bf 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -2282,9 +2282,9 @@ def _openai_llama_admission_media_tokens(extra += max(0, message_image_parts) * image_tokensif _legacy_image_is_distinct(payload):extra += image_tokens- audio = getattr(payload, "audio_base64", None)- if isinstance(audio, str) and audio:- extra += _openai_llama_admission_audio_tokens(audio)+ for audio in _request_audio_clips(payload):+ if isinstance(audio, str) and audio:+ extra += _openai_llama_admission_audio_tokens(audio)# _inject_video_part splices the legacy clip into the conversation as input_video before the# loop starts, so during a recost the clips below already include it and charging the field# too priced it exactly twice.@@ -8963,6 +8963,10 @@ _AUDIO_IMAGE_INPUT_DETAIL = ("This model takes audio or an image in one message, not both. Send the image on its own turn.")_AUDIO_VIDEO_INPUT_DETAIL = "This model takes audio or a video in one message, not both."+_MLX_MULTI_AUDIO_DETAIL = (+ "This MLX model takes one audio file per message. Send the recordings on separate turns, "+ "or load the GGUF build of the model to send several at once."+)async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: bool) -> None:@@ -8971,15 +8975,17 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: booOnly for a swap that is about to run: these exist so a request the target cannotserve fails before the load evicts the resident model, and where no swap happens theserving branch decides for itself. A GGUF target is validated through- :func:`_prepare_audio_for_llama`, which accepts a ``data:`` URI and the containers+ :func:`_prepare_audio_clips_for_llama`, which accepts a ``data:`` URI and the containerstorchaudio cannot open, and supports ``continue_final_message``. A non-GGUF target is served by:func:`_decode_audio_base64`, so the upload is validated with that same decoder and- the array handed back under ``decoded`` for the audio branch to reuse.+ the arrays handed back under ``decoded`` for the audio branch to reuse. Every clip is+ validated, in order, so a bad second recording fails before the load too."""+ clips = audio_preflight["clips"]if target_is_gguf:try:audio_preflight["prepared"] = await asyncio.to_thread(- _prepare_audio_for_llama, audio_preflight["b64"]+ _prepare_audio_clips_for_llama, clips)except _DecodedAudioTooLongError:# A limit the caller can act on. Reading as "could not be decoded"@@ -9009,6 +9015,11 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo)if audio_preflight.get("has_image"):raise HTTPException(status_code = 400, detail = _AUDIO_IMAGE_INPUT_DETAIL)+ from core.inference.local_model_resolver import _host_serves_mlx++ # A non-GGUF target on an MLX host loads on MLX, which takes one clip.+ if len(clips) > 1 and _host_serves_mlx():+ raise HTTPException(status_code = 400, detail = _MLX_MULTI_AUDIO_DETAIL)if not _audio_decoder_is_available():raise HTTPException(status_code = 400,@@ -9021,9 +9032,7 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo),)try:- audio_preflight["decoded"] = await asyncio.to_thread(- _decode_audio_base64, audio_preflight["b64"]- )+ audio_preflight["decoded"] = await asyncio.to_thread(_decode_audio_clips, clips)except _DecodedAudioTooLongError:raise HTTPException(status_code = 413,@@ -22225,7 +22234,7 @@ def _decode_audio_base64(b64: str) -> "np.ndarray":channels = max(1, int(getattr(probe, "num_channels", 1) or 1))except Exception: # noqa: BLE001 - a container info cannot read is still loadablerate, frames, channels = 0, 0, 1- limit = rate * _MAX_AUDIO_SECONDS+ limit = rate * _audio_seconds_cap()if limit and frames > limit:raise _DecodedAudioTooLongError(f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"@@ -22236,14 +22245,14 @@ def _decode_audio_base64(b64: str) -> "np.ndarray":# multichannel file -- goes to the bounded reader, which downmixes as it# goes and holds mono frames only. Streaming beats refusing: the file is# inside the clock, it is just too wide to hold at once.- if limit and frames and frames * channels <= _MAX_DECODED_SAMPLES:+ if limit and frames and frames * channels <= _decoded_samples_cap():# One frame past the cap, so a container that misreports its length# is still never fully read. Both caps apply: info() is the value# being distrusted here, so an understated num_frames must not let# the read run to the rate-relative limit, which at 192 kHz is four# times the sample ceiling. A file that fits is unaffected: its# length is under both.- read_frames = min(limit, _MAX_DECODED_SAMPLES // channels)+ read_frames = min(limit, _decoded_samples_cap() // channels)waveform, sr = torchaudio.load(tmp_path, num_frames = read_frames + 1)else:import torch@@ -22271,8 +22280,8 @@ def _decode_audio_base64(b64: str) -> "np.ndarray":os.unlink(tmp_path)# Backstop for a container that reported neither rate nor length.- if (sr > 0 and waveform.shape[-1] > sr * _MAX_AUDIO_SECONDS) or (- waveform.shape[-1] * waveform.shape[0] > _MAX_DECODED_SAMPLES+ if (sr > 0 and waveform.shape[-1] > sr * _audio_seconds_cap()) or (+ waveform.shape[-1] * waveform.shape[0] > _decoded_samples_cap()):raise _DecodedAudioTooLongError(f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"@@ -22313,6 +22322,9 @@ _REMOTE_VIDEO_REFUSAL = ("Remote video URLs are not supported. Send the clip as a data URI instead.",)_MAX_AUDIO_SECONDS = 30 * 60+# Byte and duration caps cover all clips together; the count bounds per-clip decode cost.+_MAX_AUDIO_CLIPS_PER_REQUEST = 8+_AUDIO_CLIP_B64_SLACK_CHARS = 64# The duration cap alone is rate-relative, so a high-rate container retains far# more memory for the same 30 minutes: at 48 kHz that is 86M float32 samples,# and np.concatenate doubles it. 48 kHz covers ordinary uploads, so this ceiling@@ -22333,6 +22345,88 @@ class _DecodedAudioTooLongError(ValueError):"""Decoded audio crossed the duration cap before it could be buffered."""+# (seconds, samples) the current decode may use: what earlier clips in the request left over.+_AUDIO_BUDGET_LEFT: "contextvars.ContextVar[Optional[tuple[int, int]]]" = contextvars.ContextVar(+ "_AUDIO_BUDGET_LEFT", default = None+)+++def _audio_seconds_cap() -> int:+ left = _AUDIO_BUDGET_LEFT.get()+ return _MAX_AUDIO_SECONDS if left is None else min(left[0], _MAX_AUDIO_SECONDS)+++def _decoded_samples_cap() -> int:+ left = _AUDIO_BUDGET_LEFT.get()+ if left is None or _MAX_AUDIO_SECONDS <= 0:+ return _MAX_DECODED_SAMPLES+ scaled = _MAX_DECODED_SAMPLES * _audio_seconds_cap() // _MAX_AUDIO_SECONDS+ return min(scaled, left[1])+++def _decode_within(+ seconds_used: float,+ decode,+ *args,+ samples_used: int = 0,+):+ """Run ``decode`` capped to the duration and samples the earlier clips left."""+ seconds_left = math.ceil(_MAX_AUDIO_SECONDS - seconds_used)+ samples_left = _MAX_DECODED_SAMPLES - samples_used+ if seconds_left <= 0 or samples_left <= 0:+ raise _DecodedAudioTooLongError("combined audio exceeds the duration cap")+ token = _AUDIO_BUDGET_LEFT.set((seconds_left, samples_left))+ try:+ return decode(*args)+ finally:+ _AUDIO_BUDGET_LEFT.reset(token)+++def _request_audio_clips(payload) -> list[str]:+ """Every recording the request carries, ``audio_base64`` first."""+ first = getattr(payload, "audio_base64", None)+ if not first:+ return []+ return [first, *(getattr(payload, "extra_audio_base64", None) or [])]+++def _audio_too_large_detail(clip_count: int) -> str:+ if clip_count > 1:+ return "Audio files are too large (max ~25 MB per message, all files together)."+ return "Audio file is too large (max ~25 MB)."+++def _request_audio_rejection(payload) -> Optional[tuple[int, str]]:+ """Refuse too many clips or too many combined bytes."""+ clips = _request_audio_clips(payload)+ if len(clips) > _MAX_AUDIO_CLIPS_PER_REQUEST:+ return (+ 400,+ f"Too many audio files in one message (max {_MAX_AUDIO_CLIPS_PER_REQUEST}).",+ )+ # Slack for per-clip base64 padding and data: headers.+ budget = _MAX_AUDIO_B64_CHARS + _AUDIO_CLIP_B64_SLACK_CHARS * (len(clips) - 1)+ if sum(len(clip) for clip in clips) > budget:+ return (413, _audio_too_large_detail(len(clips)))+ return None+++def _check_decoded_audio_budget(arrays: list) -> None:+ """Apply the duration cap to all decoded 16 kHz clips together."""+ if sum(len(array) for array in arrays) > 16000 * _MAX_AUDIO_SECONDS:+ raise _DecodedAudioTooLongError("combined audio exceeds the duration cap")+++def _decode_audio_clips(clips: list[str]) -> list:+ """Decode clips in order, each capped to the duration the earlier ones left."""+ arrays: list = [_decode_audio_base64(clips[0])]+ for clip in clips[1:]:+ used = sum(len(array) for array in arrays) / 16000+ arrays.append(_decode_within(used, _decode_audio_base64, clip))+ _check_decoded_audio_budget(arrays)+ return arrays++def _audio_too_long_detail() -> str:"""The one answer an overlong upload gets, whichever path found it out."""return f"Audio is too long (max {_MAX_AUDIO_SECONDS // 60} minutes)."@@ -22420,14 +22514,14 @@ def _decode_audio_with_torchcodec(path: str) -> "tuple[Any, int]":"decoded within the size limit; convert it to wav or mp3")duration = float(getattr(metadata, "duration_seconds", 0.0) or 0.0)- if duration > _MAX_AUDIO_SECONDS:+ if duration > _audio_seconds_cap():raise _DecodedAudioTooLongError(f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit")# Both ceilings, and a second past the clock so a container understating its# own duration is still cut rather than believed.channels = max(1, int(getattr(metadata, "num_channels", 1) or 1))- seconds = min(_MAX_AUDIO_SECONDS + 1, _MAX_DECODED_SAMPLES / (rate * channels) + 1)+ seconds = min(_audio_seconds_cap() + 1, _decoded_samples_cap() / (rate * channels) + 1)samples = decoder.get_samples_played_in_range(0.0, seconds)return samples.data, rate@@ -22665,23 +22759,26 @@ def _resample_mono_linear(arr: "np.ndarray", source_rate: int, target_rate: int)def _fit_transcoded_audio_to_wav_cap(- arr: "np.ndarray", sample_rate: int+ arr: "np.ndarray",+ sample_rate: int,+ cap: Optional[int] = None,) -> "tuple[np.ndarray, int]":- """Downsample only when needed so transcoded WAV stays within the upload cap."""+ """Downsample only when needed so transcoded WAV fits ``cap`` (default: the upload cap)."""+ cap = _MAX_AUDIO_RAW_BYTES if cap is None else capif sample_rate <= 0:raise ValueError("decoded audio has an invalid sample rate")wav_bytes = _WAV_HEADER_BYTES + len(arr) * 2- if wav_bytes <= _MAX_AUDIO_RAW_BYTES:+ if wav_bytes <= cap:return arr, sample_rateduration = len(arr) / float(sample_rate)- max_samples = max(1, (_MAX_AUDIO_RAW_BYTES - _WAV_HEADER_BYTES) // 2)+ max_samples = max(1, (cap - _WAV_HEADER_BYTES) // 2)target_rate = int(max_samples // duration)if target_rate < _MIN_TRANSCODE_AUDIO_SAMPLE_RATE:raise ValueError("decoded audio exceeds the transcoded WAV size limit")target_rate = min(sample_rate, target_rate)fitted = _resample_mono_linear(arr, sample_rate, target_rate)- if _WAV_HEADER_BYTES + len(fitted) * 2 > _MAX_AUDIO_RAW_BYTES:+ if _WAV_HEADER_BYTES + len(fitted) * 2 > cap:raise ValueError("decoded audio exceeds the transcoded WAV size limit")return fitted, target_rate@@ -22767,7 +22864,7 @@ def _decoded_sample_ceiling(sample_rate: int) -> int:thirty minutes' worth of memory. Nothing may allocate past the smaller ofthem, because that is the point at which the decode refuses anyway."""- return min(sample_rate * _MAX_AUDIO_SECONDS, _MAX_DECODED_SAMPLES)+ return min(sample_rate * _audio_seconds_cap(), _decoded_samples_cap())def _av_expected_samples(container, sample_rate: int, ceiling: int) -> int:@@ -22928,13 +23025,15 @@ def _decode_audio_mono(raw: bytes) -> "tuple[np.ndarray, int]":"this audio file does not report a sample rate, so it cannot ""be decoded within the size limit; convert it to wav or mp3")- window = min(float(_MAX_AUDIO_SECONDS + 1), _MAX_DECODED_SAMPLES / probe_rate + 1)+ window = min(+ float(_audio_seconds_cap() + 1), _decoded_samples_cap() / probe_rate + 1+ )arr, sr = librosa.load(tmp_path, sr = None, mono = True, duration = window)finally:os.unlink(tmp_path)if arr.ndim > 1:arr = arr.mean(axis = 1)- if (sr > 0 and len(arr) > sr * _MAX_AUDIO_SECONDS) or len(arr) > _MAX_DECODED_SAMPLES:+ if (sr > 0 and len(arr) > sr * _audio_seconds_cap()) or len(arr) > _decoded_samples_cap():raise _DecodedAudioTooLongError(f"decoded audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit")@@ -22949,33 +23048,106 @@ def _prepare_audio_for_llama(b64: str) -> tuple[str, str]:PCM payload inflation). Other containers (m4a/ogg/webm/flac) are decoded toa mono WAV. Blocking; call via a thread from async paths."""- if b64.startswith("data:"):- b64 = b64.split(",", 1)[1] if "," in b64 else ""- raw = base64.b64decode(b64)- passthrough = _sniff_audio_container(raw)- if passthrough is not None:- # Forwarding skips every bounded decoder, so the duration cap has to be- # applied from the headers instead. A 16 kbps MP3 holds hours inside the- # 25 MB upload cap, and llama-server was left to decode all of it. A- # 25 MB upload cap, and llama-server was left to decode all of it.- seconds = _passthrough_audio_seconds(raw, passthrough, _MAX_AUDIO_SECONDS)- if seconds is not None and seconds > _MAX_AUDIO_SECONDS:+ return _prepare_audio_clips_for_llama([b64])[0]+++def _prepare_audio_clips_for_llama(clips: list[str]) -> list[tuple[str, str]]:+ """Prepare clips for llama-server in order, under one shared byte and duration budget.++ wav/mp3 pass through; other formats are decoded first, then transcoded to WAV under one+ shared rate ceiling, so the result does not depend on attachment order.+ Blocking; call via a thread from async paths.+ """+ stripped = [_strip_audio_data_uri(clip) for clip in clips]+ raws = [base64.b64decode(clip) for clip in stripped]+ passthrough = [_llama_passthrough_audio(raw) for raw in raws]+ wav_budget = _MAX_AUDIO_RAW_BYTES - sum(+ len(raw) for raw, kept in zip(raws, passthrough) if kept is not None+ )+ total_seconds = 0.0+ samples_held = 0+ decoded: dict[int, tuple] = {}+ for index, (raw, kept) in enumerate(zip(raws, passthrough)):+ if kept is not None:+ seconds = kept[1]+ else:+ arr, sr = (+ _decode_within(total_seconds, _decode_audio_mono, raw, samples_used = samples_held)+ if index+ else _decode_audio_mono(raw)+ )+ decoded[index] = (arr, sr)+ samples_held += len(arr)+ seconds = len(arr) / float(sr) if sr else 0.0+ total_seconds += seconds+ if total_seconds > _MAX_AUDIO_SECONDS:raise _DecodedAudioTooLongError(f"audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit")- # Headers that cannot state a length do not earn a free pass. Forwarding- # them anyway meant the cap held only for containers honest enough to- # describe themselves, which is the wrong way round: four junk bytes in- # an MPEG stream, or a WAV with no data chunk, ended the header walk and- # took the whole recording through with it. Decoding costs a transcode- # and nothing else, and puts the file back under both ceilings, so it- # still reaches the model.- if seconds is not None:- return b64, passthrough+ shared_rate = (+ _shared_wav_rate([(len(a), sr) for a, sr in decoded.values()], wav_budget)+ if len(decoded) > 1+ else None+ )+ prepared: list[tuple[str, str]] = []+ for index, (clip, kept) in enumerate(zip(stripped, passthrough)):+ if kept is not None:+ prepared.append((clip, kept[0]))+ continue+ arr, sr = decoded.pop(index)+ if shared_rate is None:+ arr, sr = _fit_transcoded_audio_to_wav_cap(arr, sr, cap = wav_budget)+ elif shared_rate < sr:+ if shared_rate < _MIN_TRANSCODE_AUDIO_SAMPLE_RATE:+ raise ValueError("decoded audio exceeds the transcoded WAV size limit")+ arr, sr = _resample_mono_linear(arr, sr, shared_rate), shared_rate+ prepared.append((base64.b64encode(_mono_f32_to_wav_bytes(arr, sr)).decode("ascii"), "wav"))+ return prepared+++def _shared_wav_rate(clips: list[tuple[int, int]], budget: int) -> int:+ """Highest rate at which every (samples, rate) clip, capped to it, fits ``budget`` as WAV."""++ def samples_at(n: int, sr: int, rate: int) -> int:+ # _resample_mono_linear's own output length, so the budget holds exactly.+ return n if rate >= sr or sr <= 0 else max(1, int(round(n / float(sr) * rate)))++ def total(rate: int) -> int:+ return sum(_WAV_HEADER_BYTES + 2 * samples_at(n, sr, rate) for n, sr in clips)++ low, high = 0, max(sr for _, sr in clips)+ if total(high) <= budget:+ return high+ while low < high:+ mid = (low + high + 1) // 2+ if total(mid) <= budget:+ low = mid+ else:+ high = mid - 1+ return low- arr, sr = _decode_audio_mono(raw)- arr, sr = _fit_transcoded_audio_to_wav_cap(arr, sr)- return base64.b64encode(_mono_f32_to_wav_bytes(arr, sr)).decode("ascii"), "wav"++def _strip_audio_data_uri(b64: str) -> str:+ if b64.startswith("data:"):+ return b64.split(",", 1)[1] if "," in b64 else ""+ return b64+++def _llama_passthrough_audio(raw: bytes) -> Optional[tuple[str, float]]:+ """(container, seconds) if the upload can be forwarded as is, else None."""+ passthrough = _sniff_audio_container(raw)+ if passthrough is None:+ return None+ # Forwarding skips the bounded decoders: cap duration from headers (16 kbps MP3 = hours).+ seconds = _passthrough_audio_seconds(raw, passthrough, _MAX_AUDIO_SECONDS)+ if seconds is not None and seconds > _MAX_AUDIO_SECONDS:+ raise _DecodedAudioTooLongError(+ f"audio exceeds the {_MAX_AUDIO_SECONDS // 60}-minute limit"+ )+ # No stated length (junk MPEG bytes, WAV without data chunk) means decode, not forward.+ if seconds is None:+ return None+ return passthrough, seconds_VIDEO_INPUT_REFUSAL = (@@ -23098,7 +23270,7 @@ def _reject_unsupported_content_parts(payload) -> None:def _reject_misplaced_audio_parts(payload) -> None:- """Refuse a recording the single ``audio_base64`` field cannot carry faithfully.+ """Refuse recordings the ``audio_base64`` / ``extra_audio_base64`` fields cannot carry faithfully.Placement is decided here rather than in the lift because the lift runs behind routing --the preview route reaches it only after taking the preview lock and loading a checkpoint,@@ -23125,13 +23297,13 @@ def _reject_misplaced_audio_parts(payload) -> None:for part in msg.contentif isinstance(part, InputAudioContentPart)]- if len(parts) > 1:+ if len(parts) > _MAX_AUDIO_CLIPS_PER_REQUEST:_raise_unsupported_openai_parameter("messages",- "Only one audio recording per request is supported, and this one carries "- f"{len(parts)}.",+ f"At most {_MAX_AUDIO_CLIPS_PER_REQUEST} audio recordings per request are supported, "+ f"and this one carries {len(parts)}.",)- if parts and parts[0][0] != last_user:+ if any(index != last_user for index, _ in parts):_raise_unsupported_openai_parameter("messages","Audio input is supported on the latest user message, and this one carries it on an "@@ -23172,18 +23344,18 @@ def _messages_have_embedded_image(messages) -> bool:def _normalise_chat_content_parts(payload) -> None:- """Lift the request's ``input_audio`` part onto ``audio_base64``, in place.+ """Lift the request's ``input_audio`` parts onto ``audio_base64`` / ``extra_audio_base64``.- Everything that makes audio safe to serve reads that field: the capability check that keeps+ Everything that makes audio safe to serve reads those fields: the capability check that keepsa text-only target from being loaded for it, the size bound, the decoder, the duration limit,- and /chat/count_tokens' refusal. So the part is lifted rather than left standing -- a part the- field never sees is a recording none of those checks can act on.+ and /chat/count_tokens' refusal. So the parts are lifted rather than left standing -- a part+ the fields never see is a recording none of those checks can act on.- ``_reject_misplaced_audio_parts`` has already refused every shape the single field cannot- carry faithfully, so what reaches here is one recording on the latest user turn. An explicit- ``audio_base64`` still wins over a part.+ ``_reject_misplaced_audio_parts`` has already refused every shape the fields cannot carry+ faithfully, so what reaches here is up to the clip cap on the latest user turn, in order. An+ explicit ``audio_base64`` still wins over parts."""- lifted = None+ lifted: list[str] = []for msg in payload.messages:if not isinstance(msg.content, list):continue@@ -23191,13 +23363,14 @@ def _normalise_chat_content_parts(payload) -> None:for part in msg.content:if isinstance(part, InputAudioContentPart):if msg.role == "user" and part.input_audio.data:- lifted = part.input_audio.data+ lifted.append(part.input_audio.data)continuekept.append(part)if len(kept) != len(msg.content):msg.content = keptif lifted and not getattr(payload, "audio_base64", None):- payload.audio_base64 = lifted+ payload.audio_base64 = lifted[0]+ payload.extra_audio_base64 = lifted[1:] or Nonedef _message_video_urls(messages) -> list[str]:@@ -26426,8 +26599,9 @@ async def produce_openai_chat_completions(),)# target-independent, unlike the format-dependent checks the switch itself runs.- if payload.audio_base64 and len(payload.audio_base64) > _MAX_AUDIO_B64_CHARS:- raise HTTPException(status_code = 413, detail = "Audio file is too large (max ~25 MB).")+ _audio_rejection = _request_audio_rejection(payload)+ if _audio_rejection is not None:+ raise HTTPException(status_code = _audio_rejection[0], detail = _audio_rejection[1])# Reject streaming n>1 before the switch: only the non-streaming GGUF path# returns multiple choices, so stream=true + n>1 is invalid on every local# serving path (the external path already rejected it before its early@@ -26476,7 +26650,7 @@ async def produce_openai_chat_completions(# the rest of the audio checks depend on the target's format, so the switch runs them._audio_preflight = ({- "b64": payload.audio_base64,+ "clips": _request_audio_clips(payload),"continue_final": _continue_final_message(payload),"has_image": _images_in_last_user_message(payload.messages)or _legacy_image_is_distinct(payload),@@ -26748,6 +26922,15 @@ async def produce_openai_chat_completions()api_monitor.fail(monitor_id, _audio_unsupported_detail)raise HTTPException(status_code = 400, detail = _audio_unsupported_detail)+ # mlx-vlm drops or misplaces extra clips, so refuse rather than ignore them.+ # The pre-switch check may not have run, so bound clips before decoding.+ _audio_rejection = _request_audio_rejection(payload)+ if _audio_rejection is not None:+ api_monitor.fail(monitor_id, _audio_rejection[1])+ raise HTTPException(status_code = _audio_rejection[0], detail = _audio_rejection[1])+ if payload.extra_audio_base64 and model_info.get("is_mlx"):+ api_monitor.fail(monitor_id, _MLX_MULTI_AUDIO_DETAIL)+ raise HTTPException(status_code = 400, detail = _MLX_MULTI_AUDIO_DETAIL)# ── Audio INPUT path: decode WAV and route to audio input generation ──if payload.audio_base64 and model_info.get("has_audio_input"):@@ -26770,10 +26953,15 @@ async def produce_openai_chat_completions(try:# Decoded before the switch; only a path that skipped that preflight# (no automatic load could run) still has to do it here.- audio_array = (+ audio_arrays = (_predecoded_audioif _predecoded_audio is not None- else _decode_audio_base64(payload.audio_base64)+ else await asyncio.to_thread(_decode_audio_clips, _request_audio_clips(payload))+ )+ audio_array = audio_arrays[0]+ # Passed only when present, so single-clip calls are unchanged.+ extra_audio_kwargs = (+ {"extra_audio_arrays": audio_arrays[1:]} if len(audio_arrays) > 1 else {})system_prompt, chat_messages, _ = await _extract_content_parts_async(payload.messages@@ -26814,6 +27002,7 @@ async def produce_openai_chat_completions(audio_array = audio_array,cancel_event = cancel_event,stats_holder = _audio_stats_holder,+ **extra_audio_kwargs,)return backend.generate_audio_input_response(messages = chat_messages,@@ -26830,6 +27019,7 @@ async def produce_openai_chat_completions(cancel_event = cancel_event,stats_holder = _audio_stats_holder,stop = normalized_stop,+ **extra_audio_kwargs,)if payload.stream:@@ -27258,21 +27448,23 @@ async def produce_openai_chat_completions(# audio encoder); other containers are transcoded to WAV here. The part# is injected into the message list below so it rides through both the# plain and tool-calling paths, exactly like image_url parts.- audio_b64 = None- audio_format = "wav"+ prepared_audio: list[tuple[str, str]] = []if payload.audio_base64:if not getattr(llama_backend, "_has_audio_input", False):raise _reject(400,"Audio provided but current GGUF model does not support audio input.",)- if len(payload.audio_base64) > _MAX_AUDIO_B64_CHARS:- raise _reject(413, "Audio file is too large (max ~25 MB).")+ audio_rejection = _request_audio_rejection(payload)+ if audio_rejection is not None:+ raise _reject(*audio_rejection)try:- audio_b64, audio_format = (+ prepared_audio = (_preprepared_audioif _preprepared_audio is not None- else await asyncio.to_thread(_prepare_audio_for_llama, payload.audio_base64)+ else await asyncio.to_thread(+ _prepare_audio_clips_for_llama, _request_audio_clips(payload)+ ))except _DecodedAudioTooLongError as e:# A valid file that is simply too long reports the limit, as the@@ -27282,9 +27474,10 @@ async def produce_openai_chat_completions(except Exception as e:logger.warning("Audio decode failed: %s", e, exc_info = True)raise _reject(400, "Could not decode the provided audio file.")- # Admission reads the duration from this field's header, which only the forwarded+ # Admission reads the duration from these fields' headers, which only the forwarded# wav/mp3 is sure to state; an m4a, ogg or flac upload would be charged by its bytes.- payload.audio_base64 = audio_b64+ payload.audio_base64 = prepared_audio[0][0]+ payload.extra_audio_base64 = [clip for clip, _ in prepared_audio[1:]] or None# llama-server samples frames but encodes each at the clip's resolution.video_b64 = None@@ -27317,7 +27510,7 @@ async def produce_openai_chat_completions(gguf_messages, request, thread_id = getattr(payload, "thread_id", None))image_b64 = None- if audio_b64:+ for audio_b64, audio_format in prepared_audio:_inject_audio_part(gguf_messages, audio_b64, audio_format)if video_b64:_inject_video_part(gguf_messages, video_b64)@@ -36282,7 +36475,7 @@ async def chat_count_tokens(detail = "Cannot count tokens for messages containing images.",)# Same for audio: the completion injects the recording, this cannot.- if getattr(payload, "audio_base64", None):+ if getattr(payload, "audio_base64", None) or getattr(payload, "extra_audio_base64", None):raise HTTPException(status_code = 503,detail = "Cannot count tokens for messages containing audio.",diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.pyindex 2df2a937f1f..96aaa3dbf7d 100644--- a/studio/backend/tests/test_chat_input_audio_part.py+++ b/studio/backend/tests/test_chat_input_audio_part.py@@ -69,17 +69,102 @@ def test_an_explicit_audio_base64_wins():assert payload.audio_base64 == AUDIO_B64-def test_two_recordings_are_refused_rather_than_reduced_to_one():- """``audio_base64`` holds one recording, so a second was silently discarded.+def _multi_audio_message(*datas, text = "compare these"):+ return {+ "role": "user",+ "content": [+ {"type": "text", "text": text},+ *(+ {"type": "input_audio", "input_audio": {"data": data, "format": "wav"}}+ for data in datas+ ),+ ],+ }- A request to compare two clips would have been answered from the last one alone.- """++def test_several_recordings_on_the_latest_turn_are_all_lifted_in_order():+ """Models like Gemma 4 take several clips at once; none may be dropped or reordered."""+ payload = _request(_multi_audio_message("Zmlyc3Q=", "c2Vjb25k", "dGhpcmQ="))++ _reject_unsupported_content_parts(payload)+ _normalise_chat_content_parts(payload)++ assert payload.audio_base64 == "Zmlyc3Q="+ assert payload.extra_audio_base64 == ["c2Vjb25k", "dGhpcmQ="]+ assert inference_route._request_audio_clips(payload) == ["Zmlyc3Q=", "c2Vjb25k", "dGhpcmQ="]+ assert [p.type for p in payload.messages[0].content] == ["text"]+++def test_recordings_split_across_turns_are_refused_rather_than_reduced():+ """Only the latest turn's clips reach the model, so an older clip would be answered without."""payload = _request(_audio_message(data = "Zmlyc3Q="), _audio_message(data = "c2Vjb25k"))with pytest.raises(HTTPException) as exc:_reject_unsupported_content_parts(payload)assert exc.value.status_code == 400- assert "one audio recording" in str(exc.value.detail)+ assert "earlier turn" in str(exc.value.detail)+++def test_more_recordings_than_the_cap_are_refused():+ datas = ["QUFB"] * (inference_route._MAX_AUDIO_CLIPS_PER_REQUEST + 1)+ payload = _request(_multi_audio_message(*datas))++ with pytest.raises(HTTPException) as exc:+ _reject_unsupported_content_parts(payload)+ assert exc.value.status_code == 400+ assert f"At most {inference_route._MAX_AUDIO_CLIPS_PER_REQUEST}" in str(exc.value.detail)+++def test_extra_audio_alone_is_promoted_onto_the_audio_field():+ """Every capability and size check keys on audio_base64, so extras can never bypass them."""+ payload = _request(+ {"role": "user", "content": "hi"}, extra_audio_base64 = ["", "Zmlyc3Q=", "c2Vjb25k"]+ )++ assert payload.audio_base64 == "Zmlyc3Q="+ assert payload.extra_audio_base64 == ["c2Vjb25k"]+++def test_the_size_cap_covers_all_clips_together(monkeypatch):+ """N clips never cost more upload or decode memory than one maximal clip does."""+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_B64_CHARS", 10)+ monkeypatch.setattr(inference_route, "_AUDIO_CLIP_B64_SLACK_CHARS", 2)+ one = _request({"role": "user", "content": "hi"}, audio_base64 = "A" * 10)+ assert inference_route._request_audio_rejection(one) is None+ within_slack = _request(+ {"role": "user", "content": "hi"}, audio_base64 = "A" * 6, extra_audio_base64 = ["A" * 6]+ )+ assert inference_route._request_audio_rejection(within_slack) is None++ two = _request(+ {"role": "user", "content": "hi"}, audio_base64 = "A" * 7, extra_audio_base64 = ["A" * 6]+ )+ status, detail = inference_route._request_audio_rejection(two)+ assert status == 413+ assert "all files together" in detail+++def test_the_count_cap_is_checked_on_the_fields_too():+ payload = _request(+ {"role": "user", "content": "hi"},+ audio_base64 = "QUFB",+ extra_audio_base64 = ["QUFB"] * inference_route._MAX_AUDIO_CLIPS_PER_REQUEST,+ )+ status, detail = inference_route._request_audio_rejection(payload)+ assert status == 400+ assert "Too many audio files" in detail+++def test_the_duration_cap_covers_all_decoded_clips_together(monkeypatch):+ import numpy as np++ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)+ monkeypatch.setattr(+ inference_route, "_decode_audio_base64", lambda _b64: np.zeros(12_000, np.float32)+ )+ assert len(inference_route._decode_audio_clips(["a"])) == 1+ with pytest.raises(inference_route._DecodedAudioTooLongError):+ inference_route._decode_audio_clips(["a", "b"])def test_an_audio_part_on_a_non_user_role_is_refused():@@ -509,3 +594,239 @@ def test_the_text_only_checkpoint_refusal_precedes_the_branch_that_consumes_audi# the refusal has to precede the consuming branch, or it never runsassert branch < consumeassert "cannot read audio input" in source[branch:consume]+++def _wav_b64(seconds: float, rate: int = 16000) -> str:+ import base64+ import io+ import wave++ buf = io.BytesIO()+ with wave.open(buf, "wb") as wav:+ wav.setnchannels(1)+ wav.setsampwidth(2)+ wav.setframerate(rate)+ wav.writeframes(b"\x00\x00" * int(seconds * rate))+ return base64.b64encode(buf.getvalue()).decode("ascii")+++def test_gguf_clips_pass_through_in_order():+ first, second = _wav_b64(0.5), _wav_b64(0.25)+ prepared = inference_route._prepare_audio_clips_for_llama(+ [first, f"data:audio/wav;base64,{second}"]+ )+ assert prepared == [(first, "wav"), (second, "wav")]+++def test_gguf_duration_cap_covers_all_clips_together(monkeypatch):+ """Each clip alone is under the cap; together they are not, as on the decoded path."""+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_SECONDS", 1)+ clip = _wav_b64(0.75)+ assert len(inference_route._prepare_audio_clips_for_llama([clip])) == 1+ with pytest.raises(inference_route._DecodedAudioTooLongError):+ inference_route._prepare_audio_clips_for_llama([clip, clip])+++def test_transcoded_gguf_clips_share_one_wav_budget(monkeypatch):+ """Several m4a/ogg clips must not each grow into a cap-sized WAV for llama-server."""+ import base64++ import numpy as np++ caps = []++ def _fit(+ arr,+ sr,+ cap = None,+ ):+ caps.append(cap)+ return arr, sr++ monkeypatch.setattr(inference_route, "_MAX_AUDIO_RAW_BYTES", 1000)+ monkeypatch.setattr(inference_route, "_sniff_audio_container", lambda _raw: None)+ monkeypatch.setattr(+ inference_route, "_decode_audio_mono", lambda _raw: (np.zeros(100, np.float32), 16000)+ )+ monkeypatch.setattr(inference_route, "_fit_transcoded_audio_to_wav_cap", _fit)++ inference_route._prepare_audio_for_llama("QUFB")+ assert caps == [1000] # one clip keeps the whole cap, exactly as before++ # Clips that fit together are neither refit nor resampled.+ caps.clear()+ prepared = inference_route._prepare_audio_clips_for_llama(["QUFB", "QUFB", "QUFB"])+ assert caps == []+ assert [len(base64.b64decode(data)) for data, _ in prepared] == [44 + 100 * 2] * 3++ # Pass-through clips are forwarded at their upload size, so transcodes share what they leave.+ caps.clear()+ monkeypatch.setattr(+ inference_route, "_sniff_audio_container", lambda raw: "wav" if raw == b"WAV!" else None+ )+ monkeypatch.setattr(inference_route, "_passthrough_audio_seconds", lambda *_a: 0.1)+ inference_route._prepare_audio_clips_for_llama(["V0FWIQ==", "QUFB"])+ assert caps == [1000 - 4]+++def test_the_wav_budget_does_not_depend_on_clip_order(monkeypatch):+ """A long clip beside a short one fits by downsampling both to one shared rate.++ An equal split starved the long clip below the minimum rate whenever it came first.+ """+ import base64+ import io+ import wave++ import numpy as np++ long_clip = np.zeros(10 * 16000, np.float32)+ short_clip = np.zeros(16000, np.float32)+ monkeypatch.setattr(inference_route, "_sniff_audio_container", lambda _raw: None)+ monkeypatch.setattr(+ inference_route,+ "_decode_audio_mono",+ lambda raw: ((long_clip if raw == b"LNG" else short_clip), 16000),+ )+ # Room for 11 seconds at 9 kHz, well under the 16 kHz the clips arrive at.+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_RAW_BYTES", 2 * 44 + 2 * 11 * 9000)++ def rates(prepared):+ out = []+ for data, fmt in prepared:+ assert fmt == "wav"+ with wave.open(io.BytesIO(base64.b64decode(data))) as wav:+ out.append(wav.getframerate())+ return out++ long_b64 = base64.b64encode(b"LNG").decode()+ short_b64 = base64.b64encode(b"SHT").decode()+ assert rates(inference_route._prepare_audio_clips_for_llama([long_b64, short_b64])) == [+ 9000,+ 9000,+ ]+ assert rates(inference_route._prepare_audio_clips_for_llama([short_b64, long_b64])) == [+ 9000,+ 9000,+ ]+++def test_clips_below_the_shared_rate_are_left_alone():+ n_low, n_high = 8000 * 1200, 48000 * 300+ budget = 25 * 1024 * 1024+ rate = inference_route._shared_wav_rate([(n_low, 8000), (n_high, 48000)], budget)+ # The 8 kHz clip keeps its full size; the 48 kHz one still clears the 8 kHz floor.+ assert 8000 <= rate < 48000+ assert 2 * 44 + 2 * n_low + 2 * round(n_high / 48000 * rate) <= budget+++def test_the_shared_rate_is_used_as_is_at_the_floor(monkeypatch):+ """A budget that allows exactly 8 kHz must not come out at 7999 Hz and be refused."""+ import base64+ import io+ import wave++ import numpy as np++ sr = 44100+ lengths = {b"ONE": sr * 7 + 1, b"TWO": sr * 3 + 7}+ monkeypatch.setattr(inference_route, "_sniff_audio_container", lambda _raw: None)+ monkeypatch.setattr(+ inference_route,+ "_decode_audio_mono",+ lambda raw: (np.zeros(lengths[raw], np.float32), sr),+ )+ budget = sum(44 + 2 * round(n / float(sr) * 8000) for n in lengths.values())+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_RAW_BYTES", budget)++ prepared = inference_route._prepare_audio_clips_for_llama(+ [base64.b64encode(b"ONE").decode(), base64.b64encode(b"TWO").decode()]+ )+ wavs = [base64.b64decode(data) for data, _ in prepared]+ assert [wave.open(io.BytesIO(w)).getframerate() for w in wavs] == [8000, 8000]+ assert sum(len(w) for w in wavs) <= budget+++def test_held_clips_count_against_the_sample_ceiling(monkeypatch):+ """Decoded clips are held until the budget is split, so they share one sample ceiling."""+ seen = []++ def _decode(_raw):+ seen.append(inference_route._decoded_samples_cap())+ return inference_route._decoded_samples_cap++ monkeypatch.setattr(inference_route, "_MAX_DECODED_SAMPLES", 1000)+ inference_route._decode_within(0.0, _decode, b"", samples_used = 600)+ assert seen == [400]+ with pytest.raises(inference_route._DecodedAudioTooLongError):+ inference_route._decode_within(0.0, _decode, b"", samples_used = 1000)+++class _CapturingAudioBackend:+ def __init__(self):+ self.calls = []++ def generate_audio_input_response(self, **kwargs):+ self.calls.append(kwargs)+ return iter(())+++def _run_worker_audio(clips):+ import numpy as np+ from types import SimpleNamespace++ from core.inference import worker++ backend = _CapturingAudioBackend()+ sent = []+ worker._handle_generate_audio_input(+ backend,+ {+ "request_id": "r",+ "audio_clips": [np.asarray(c, dtype = np.float32).tobytes() for c in clips],+ },+ SimpleNamespace(put = lambda item, *a, **k: sent.append(item)),+ SimpleNamespace(is_set = lambda: False),+ )+ assert not [m for m in sent if m.get("type") == "gen_error"], sent+ return backend.calls[0]+++def test_a_single_clip_reaches_the_backend_as_before():+ """A plain list of samples is ONE waveform, never a list of one-sample clips."""+ call = _run_worker_audio([[0.0, 0.1, -0.1]])+ assert list(call["audio_array"]) == pytest.approx([0.0, 0.1, -0.1])+ assert "extra_audio_arrays" not in call+++def test_extra_clips_reach_the_backend_in_order():+ call = _run_worker_audio([[0.1], [0.2, 0.2], [0.3]])+ assert list(call["audio_array"]) == pytest.approx([0.1])+ assert [list(c) for c in call["extra_audio_arrays"]] == [+ pytest.approx([0.2, 0.2]),+ pytest.approx([0.3]),+ ]+++def test_extra_clips_join_the_latest_user_turn_in_order():+ from core.inference.chat_template_helpers import messages_with_attached_image++ history = [+ {"role": "user", "content": "earlier"},+ {"role": "assistant", "content": "reply"},+ {"role": "user", "content": "compare these"},+ ]+ rendered = messages_with_attached_image(+ history, structured_content = True, image = 0, audio = "a", extra_audio = ["b", "c"]+ )+ assert rendered[0]["content"] == [{"type": "text", "text": "earlier"}]+ assert rendered[-1]["content"] == [+ {"type": "audio", "audio": "a"},+ {"type": "audio", "audio": "b"},+ {"type": "audio", "audio": "c"},+ {"type": "text", "text": "compare these"},+ ]+ # No extras: exactly the single-clip render.+ single = messages_with_attached_image(history, structured_content = True, image = 0, audio = "a")+ assert single[-1]["content"][0] == {"type": "audio", "audio": "a"}+ assert len(single[-1]["content"]) == 2diff --git a/studio/backend/tests/test_llama_audio_attachment_container.py b/studio/backend/tests/test_llama_audio_attachment_container.pyindex ee30843f84b..035f0a30d6f 100644--- a/studio/backend/tests/test_llama_audio_attachment_container.py+++ b/studio/backend/tests/test_llama_audio_attachment_container.py@@ -35,12 +35,12 @@ def test_aac_and_mp2_are_transcoded_instead_of_forwarded_as_mp3(monkeypatch):monkeypatch.setattr(inference_route,"_decode_audio_mono",- lambda raw: (decoded.append(raw) or object(), 16_000),+ lambda raw: (decoded.append(raw) or np.zeros(0, np.float32), 16_000),)monkeypatch.setattr(inference_route,"_fit_transcoded_audio_to_wav_cap",- lambda array, sample_rate: (array, sample_rate),+ lambda array, sample_rate, **_: (array, sample_rate),)monkeypatch.setattr(inference_route,@@ -939,13 +939,15 @@ def test_a_switch_preflight_answers_an_overlong_upload_like_the_serving_path(mondef _too_long(_b64):raise inference_route._DecodedAudioTooLongError("decoded audio exceeds the limit")- monkeypatch.setattr(inference_route, "_prepare_audio_for_llama", _too_long)+ monkeypatch.setattr(inference_route, "_prepare_audio_clips_for_llama", _too_long)monkeypatch.setattr(inference_route, "_decode_audio_base64", _too_long)monkeypatch.setattr(inference_route, "_audio_decoder_is_available", lambda: True)for target_is_gguf in (True, False):try:- asyncio.run(inference_route._preflight_audio_for_switch({"b64": "x"}, target_is_gguf))+ asyncio.run(+ inference_route._preflight_audio_for_switch({"clips": ["x"]}, target_is_gguf)+ )except HTTPException as error:assert error.status_code == 413, target_is_ggufassert error.detail == inference_route._audio_too_long_detail()@@ -962,13 +964,15 @@ def test_a_switch_preflight_still_reports_undecodable_audio_as_a_bad_value(monkedef _broken(_b64):raise ValueError("not audio at all")- monkeypatch.setattr(inference_route, "_prepare_audio_for_llama", _broken)+ monkeypatch.setattr(inference_route, "_prepare_audio_clips_for_llama", _broken)monkeypatch.setattr(inference_route, "_decode_audio_base64", _broken)monkeypatch.setattr(inference_route, "_audio_decoder_is_available", lambda: True)for target_is_gguf in (True, False):try:- asyncio.run(inference_route._preflight_audio_for_switch({"b64": "x"}, target_is_gguf))+ asyncio.run(+ inference_route._preflight_audio_for_switch({"clips": ["x"]}, target_is_gguf)+ )except HTTPException as error:assert error.status_code == 400, target_is_ggufelse:diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 472fe237925..93bd8b35ffe 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -10820,11 +10820,11 @@ def test_the_audio_preflight_only_binds_a_non_gguf_target(monkeypatch):audio.setframerate(16000)audio.writeframes(b"\x00\x00" * 16)preflight = {- "b64": f"data:audio/wav;base64,{_b64.b64encode(wav.getvalue()).decode()}",+ "clips": [f"data:audio/wav;base64,{_b64.b64encode(wav.getvalue()).decode()}"],"continue_final": True,}asyncio.run(inference_route._preflight_audio_for_switch(preflight, True))- assert preflight["prepared"][1] == "wav"+ assert preflight["prepared"][0][1] == "wav"assert "decoded" not in preflight# the non-GGUF branch runs _decode_audio_base64, so it refuses the same input, before the load.@@ -10877,7 +10877,7 @@ def test_a_prior_turn_image_does_not_block_a_non_gguf_audio_switch(monkeypatch):require_image = True,require_audio_input = True,audio_preflight = {- "b64": "valid",+ "clips": ["valid"],"continue_final": False,"has_image": False,},@@ -11337,7 +11337,7 @@ def test_mixed_audio_and_image_is_rejected_before_a_non_gguf_switch(monkeypatch)require_image = True,require_audio_input = True,audio_preflight = {- "b64": "AAAA",+ "clips": ["AAAA"],"continue_final": False,"has_image": True,},@@ -11382,7 +11382,7 @@ def test_audio_beside_a_clip_is_rejected_before_a_non_gguf_switch(monkeypatch):require_audio_input = True,require_video = True,audio_preflight = {- "b64": "AAAA",+ "clips": ["AAAA"],"continue_final": True,"has_image": True,"has_video": True,@@ -11422,7 +11422,7 @@ def test_the_gguf_audio_preflight_takes_the_base64_llama_cpp_takes():try:asyncio.run(inference_route._preflight_audio_for_switch(- {"b64": encoded, "continue_final": False}, True+ {"clips": [encoded], "continue_final": False}, True))except HTTPException:@@ -11432,7 +11432,7 @@ def test_the_gguf_audio_preflight_takes_the_base64_llama_cpp_takes():with pytest.raises(HTTPException) as exc:asyncio.run(inference_route._preflight_audio_for_switch(- {"b64": bad, "continue_final": False}, True+ {"clips": [bad], "continue_final": False}, True))assert exc.value.status_code == 400, bad@@ -11457,7 +11457,7 @@ def test_non_audio_bytes_are_rejected_before_a_gguf_switch(monkeypatch):"tester",require_audio_input = True,audio_preflight = {- "b64": _b64.b64encode(b"not audio").decode(),+ "clips": [_b64.b64encode(b"not audio").decode()],"continue_final": False,},)@@ -11476,7 +11476,7 @@ def test_a_non_gguf_audio_target_is_refused_without_a_decoder(monkeypatch):with pytest.raises(HTTPException) as exc:asyncio.run(inference_route._preflight_audio_for_switch(- {"b64": "AAAA", "continue_final": False}, False+ {"clips": ["AAAA"], "continue_final": False}, False))assert exc.value.status_code == 400@@ -11497,14 +11497,14 @@ def test_non_audio_bytes_are_rejected_before_a_non_gguf_switch(monkeypatch):with pytest.raises(HTTPException) as exc:asyncio.run(inference_route._preflight_audio_for_switch(- {"b64": "AAAA", "continue_final": False}, False+ {"clips": ["AAAA"], "continue_final": False}, False))assert exc.value.status_code == 400- preflight = {"b64": "GOOD", "continue_final": False}+ preflight = {"clips": ["GOOD"], "continue_final": False}asyncio.run(inference_route._preflight_audio_for_switch(preflight, False))- assert preflight["decoded"] == "pcm"+ assert preflight["decoded"] == ["pcm"]def test_a_gguf_only_host_does_not_need_torchaudio_to_accept_audio(monkeypatch):diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsxindex 94a787d39a0..2151a6cda1d 100644--- a/studio/frontend/src/components/assistant-ui/thread.tsx+++ b/studio/frontend/src/components/assistant-ui/thread.tsx@@ -3190,10 +3190,9 @@ const Composer: FC<{try {await aui.composer().addAttachment(file);} catch {- // Chat-wide, not per file (no audio model, too large, already- // attached), and every adapter path toasted: stop quietly.+ // The adapter toasted. Keep going: a later, smaller clip may still fit.if (stillThisComposer()) cancelQueuedSendRef.current?.();- return;+ continue;}}}@@ -3242,8 +3241,8 @@ const Composer: FC<{};}, [nativeAttachmentTargetKey, aui]);- // Same drain as audio, one queue over: one clip per message, and the send- // gate has to hold across the read either way.+ // Same drain as audio, one queue over: video is one clip per message, and the+ // send gate has to hold across the read either way.useEffect(() => {if (!nativeAttachmentTargetKey) {return;diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 661ad7a4086..abd3dfa9873 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -1698,24 +1698,23 @@ export function messagesUsePrivateContent(messages: RunMessages): boolean {});}-export function findLatestUserAudioBase64(- messages: RunMessages,- includePendingAudio = true,-): string | undefined {+/** Every clip on the newest user message, in the order it was attached. */+function latestUserAudioClips(messages: RunMessages): string[] {+ const clips: string[] = [];for (let i = messages.length - 1; i >= 0; i -= 1) {const message = messages[i];if (!message || message.role !== "user") continue;for (const part of message.content ?? []) {const base64 = extractAudioPartBase64(part);- if (base64) return base64;+ if (base64) clips.push(base64);}if ("attachments" in message) {for (const attachment of message.attachments ?? []) {for (const part of attachment.content ?? []) {const base64 = extractAudioPartBase64(part);- if (base64) return base64;+ if (base64) clips.push(base64);}}}@@ -1726,6 +1725,15 @@ export function findLatestUserAudioBase64(// retranscribe the stale clip. Matches the consumed-on-send semantics of the legacy pendingAudio.break;}+ return clips;+}++export function findLatestUserAudioBase64(+ messages: RunMessages,+ includePendingAudio = true,+): string | undefined {+ const [first] = latestUserAudioClips(messages);+ if (first) return first;const pendingAudio = includePendingAudio? useChatRuntimeStore.getState().pendingAudioBase64@@ -1733,6 +1741,14 @@ export function findLatestUserAudioBase64(return pendingAudio ?? undefined;}+/** Clips after the first, for extra_audio_base64. Undefined when empty so text turns stay durable. */+export function findLatestUserExtraAudioBase64(+ messages: RunMessages,+): string[] | undefined {+ const extra = latestUserAudioClips(messages).slice(1);+ return extra.length > 0 ? extra : undefined;+}+function extractVideoPartBase64(part: { type: string } | null | undefined,): string | undefined {@@ -5200,6 +5216,7 @@ export function createOpenAIStreamAdapter(externalModelLabel(params.checkpoint) ||params.checkpoint,audio: Boolean(findLatestUserAudioBase64(survivingMessages, false)),+ audioCount: latestUserAudioClips(survivingMessages).length,video: Boolean(videoBase64),});if (attachedMediaReason) {@@ -6495,6 +6512,8 @@ export function createOpenAIStreamAdapter(currentTurnMessages,!queuedRunSettings && !continuation,),+ extra_audio_base64:+ findLatestUserExtraAudioBase64(currentTurnMessages),video_base64: findLatestUserVideoBase64(currentTurnMessages),cancel_id: cancelId,...(sandboxSessionId ? { session_id: sandboxSessionId } : {}),diff --git a/studio/frontend/src/features/chat/audio-attachment-adapter.ts b/studio/frontend/src/features/chat/audio-attachment-adapter.tsindex baecd8d2f6b..4325a016bfc 100644--- a/studio/frontend/src/features/chat/audio-attachment-adapter.ts+++ b/studio/frontend/src/features/chat/audio-attachment-adapter.ts@@ -4,7 +4,8 @@import {AUDIO_ATTACHMENT_ACCEPT,fileToBase64,- getAudioSizeError,+ getAudioAddError,+ maxAudioFilesFor,} from "@/lib/audio-utils";import type {Attachment,@@ -17,13 +18,15 @@ import { externalModelLabel } from "./lib/external-model-label";import { useChatRuntimeStore } from "./stores/chat-runtime-store";// crypto.randomUUID is undefined in non-secure contexts (HTTP over a LAN IP).-function newAttachmentId(): string {+export function newAttachmentId(): string {if (typeof globalThis.crypto?.randomUUID === "function") {return globalThis.crypto.randomUUID();}return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;}+const AUDIO_ADD_TOAST_ID = "audio-attachment-limit";+// A loaded model without audio rejects at add(); with none loaded, the send path checks it later.export class AudioAttachmentAdapter implements AttachmentAdapter {// MIME is unreliable for some containers (m4a), so also match by extension. No .webm extension:@@ -31,7 +34,8 @@ export class AudioAttachmentAdapter implements AttachmentAdapter {// MIME. .mp4 and .m4v stay off for the same reason; only .m4a is audio-only. Not the picker list:// this decides routing, and .3gp is in that list only so a dialog can offer a recording.accept = AUDIO_ATTACHMENT_ACCEPT;- private readonly attachmentIds = new Set<string>();+ // Pending clip sizes by id; caps cover all clips in a message.+ private readonly attachmentSizes = new Map<string, number>();async add({ file }: { file: File }): Promise<PendingAttachment> {const state = useChatRuntimeStore.getState();@@ -50,22 +54,32 @@ export class AudioAttachmentAdapter implements AttachmentAdapter {unavailableReason = `${label} cannot accept audio. Load an audio-input model before attaching audio files.`;}if (unavailableReason) {- toast.error(unavailableReason);+ toast.error(unavailableReason, { id: AUDIO_ADD_TOAST_ID });throw new Error(unavailableReason);}- const sizeReason = getAudioSizeError(file.size);- if (sizeReason) {- toast.error(sizeReason);- throw new Error(sizeReason);+ // A staged store clip would be dropped if sent alongside attachments.+ if (state.pendingAudioBase64) {+ const stagedReason =+ "Send or remove the staged audio clip before attaching more audio.";+ toast.error(stagedReason, { id: AUDIO_ADD_TOAST_ID });+ throw new Error(stagedReason);}- if (this.attachmentIds.size > 0 || state.pendingAudioBase64) {- const duplicateReason = "Only one audio file can be attached per message.";- toast.error(duplicateReason);- throw new Error(duplicateReason);+ let totalSize = 0;+ for (const size of this.attachmentSizes.values()) totalSize += size;+ const addReason = getAudioAddError(+ this.attachmentSizes.size,+ totalSize,+ file.size,+ maxAudioFilesFor(modelLoaded ? activeModel : undefined),+ );+ if (addReason) {+ // Shared id so a large batch toasts once.+ toast.error(addReason, { id: AUDIO_ADD_TOAST_ID });+ throw new Error(addReason);}const id = newAttachmentId();- this.attachmentIds.add(id);+ this.attachmentSizes.set(id, file.size);return {id,type: "file",@@ -90,12 +104,12 @@ export class AudioAttachmentAdapter implements AttachmentAdapter {status: { type: "complete" },};} finally {- this.attachmentIds.delete(attachment.id);+ this.attachmentSizes.delete(attachment.id);}}remove(attachment: Attachment): Promise<void> {- this.attachmentIds.delete(attachment.id);+ this.attachmentSizes.delete(attachment.id);return Promise.resolve();}}diff --git a/studio/frontend/src/features/chat/lib/attached-media-gate.ts b/studio/frontend/src/features/chat/lib/attached-media-gate.tsindex 018a42be6ee..788005180b9 100644--- a/studio/frontend/src/features/chat/lib/attached-media-gate.ts+++ b/studio/frontend/src/features/chat/lib/attached-media-gate.ts@@ -6,18 +6,26 @@ export function attachedMediaUnavailableReason({checkpoint,modelLabel,audio,+ audioCount = audio ? 1 : 0,video,}: {- activeModel: { hasAudioInput?: boolean; hasVideoInput?: boolean } | undefined;+ activeModel:+ | { hasAudioInput?: boolean; hasVideoInput?: boolean; isMlx?: boolean }+ | undefined;checkpoint: string | null | undefined;modelLabel: string;audio: boolean;+ audioCount?: number;video: boolean;}): string | null {if (!checkpoint || (!audio && !video)) return null;if (audio && !activeModel?.hasAudioInput) {return `${modelLabel} cannot accept audio. Load an audio-input model, or remove the audio file.`;}+ // MLX takes one clip per message (see maxAudioFilesFor).+ if (audio && audioCount > 1 && activeModel?.isMlx) {+ return `${modelLabel} takes one audio file per message. Remove the extra audio files, or load a GGUF model to send several.`;+ }if (video && !activeModel?.hasVideoInput) {return `${modelLabel} cannot accept video. Load a model that reads video, or remove the video.`;}diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex 0bdbb5b9271..b4fe427a3d4 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -61,10 +61,11 @@ import {AUDIO_PICKER_ACCEPT,isAudioAttachmentFile,fileToBase64,- getAudioSizeError,+ getAudioAddError,} from "@/lib/audio-utils";import { isTauri } from "@/lib/api-base";import { classifiedAttachmentFiles, isVideoFile } from "@/lib/video-utils";+import { newAttachmentId } from "./audio-attachment-adapter";import { isDownloadCancelled } from "@/lib/native-files";import { isMultimodalResponse } from "./types/api";import { getImageInputUnavailableReason } from "./utils/image-input-support";@@ -632,14 +633,19 @@ export function SharedComposer({const [comparing, setComparing] = useState(false);const [pendingImages, setPendingImages] = useState<PendingImage[]>([]);const [convertingImages, setConvertingImages] = useState(0);- const [pendingAudio, setPendingAudio] = useState<{- name: string;- base64: string;- contentType: string;- } | null>(null);+ const [pendingAudio, setPendingAudio] = useState<+ {+ id: string;+ name: string;+ base64: string;+ contentType: string;+ size: number;+ }[]+ >([]);const textRef = useRef(text);const pendingImagesRef = useRef(pendingImages);const pendingAudioRef = useRef(pendingAudio);+ const readingAudioRef = useRef(new Map<string, number>());const setCurrentText = useCallback((value: string | ((previous: string) => string)) => {const next =@@ -649,8 +655,6 @@ export function SharedComposer({},[],);- // Audio files still being read into base64, which pendingAudio cannot see yet.- const audioDecodingRef = useRef(0);// Attachments still classifying or converting; a ref so runPromptList reads it synchronously.const attachingRef = useRef(0);useEffect(() => {@@ -993,6 +997,18 @@ export function SharedComposer({const clearPendingAudioStore = useChatRuntimeStore((s) => s.clearPendingAudio,);+ // Mirror the first clip into the store's single-clip slot.+ const mirroredAudioRef = useRef(false);+ useEffect(() => {+ const [first] = pendingAudio;+ if (first) {+ setPendingAudioStore(first.base64, first.name);+ mirroredAudioRef.current = true;+ } else if (mirroredAudioRef.current) {+ clearPendingAudioStore();+ mirroredAudioRef.current = false;+ }+ }, [pendingAudio, setPendingAudioStore, clearPendingAudioStore]);const {isDictating,@@ -1114,30 +1130,46 @@ export function SharedComposer({const files = await classifiedAttachmentFiles(input);const next: PendingImage[] = [];let droppedImageForUnavailable = false;- let audioSizeError: string | null = null;+ let audioAddError: string | null = null;let videoUnsupported = false;let conversionError: string | null = null;for (let i = 0; i < files.length; i++) {const file = files[i];if (!file) continue;if (isAudioAttachmentFile(file)) {- const sizeError = getAudioSizeError(file.size);- if (sizeError) {- audioSizeError ??= sizeError;+ // Caps count staged clips and ones still being read by any batch.+ const counted = [+ ...pendingAudioRef.current.map((clip) => clip.size),+ ...readingAudioRef.current.values(),+ ];+ const addError = getAudioAddError(+ counted.length,+ counted.reduce((total, size) => total + size, 0),+ file.size,+ );+ if (addError) {+ audioAddError ??= addError;continue;}- audioDecodingRef.current += 1;- fileToBase64(file)- .then((base64) => {- setPendingAudio({ name: file.name, base64, contentType: file.type });- setPendingAudioStore(base64, file.name);- })- .catch(() => {- toast.error("Could not read that audio file.");- })- .finally(() => {- audioDecodingRef.current -= 1;- });+ const clip = {+ id: newAttachmentId(),+ name: file.name,+ base64: "",+ contentType: file.type,+ size: file.size,+ };+ readingAudioRef.current.set(clip.id, clip.size);+ try {+ clip.base64 = await fileToBase64(file);+ } catch {+ toast.error("Could not read that audio file.");+ continue;+ } finally {+ readingAudioRef.current.delete(clip.id);+ }+ // Updated now, not on commit, so the next check already counts it.+ pendingAudioRef.current = [...pendingAudioRef.current, clip];+ setPendingAudio((prev) => [...prev, clip]);continue;}// video_base64 targets the single loaded GGUF, so at most one side of a compare could answer. Say@@ -1168,8 +1200,8 @@ export function SharedComposer({if (droppedImageForUnavailable && attachUnavailableReason) {toast.error(attachUnavailableReason);}- if (audioSizeError) {- toast.error(audioSizeError);+ if (audioAddError) {+ toast.error(audioAddError);}if (conversionError) {toast.error(conversionError);@@ -1181,15 +1213,19 @@ export function SharedComposer({}setPendingImages((prev) => [...prev, ...next]);},- [setPendingAudioStore, attachUnavailableReason],+ [attachUnavailableReason],);+ // State mirror of attachingRef: holds sends until a whole batch is read.+ const [addingFiles, setAddingFiles] = useState(0);const trackAttaching = useCallback(async (work: () => Promise<void>) => {attachingRef.current += 1;+ setAddingFiles((count) => count + 1);try {await work();} finally {attachingRef.current -= 1;+ setAddingFiles((count) => count - 1);}}, []);@@ -1265,7 +1301,7 @@ export function SharedComposer({useEffect(() => () => clearStuckImeTimer(), []);async function send() {- if (composingRef.current || convertingImages > 0) {+ if (composingRef.current || convertingImages > 0 || addingFiles > 0) {resetPromptQueue();return;}@@ -1273,7 +1309,7 @@ export function SharedComposer({const submittedImages = pendingImages;const submittedAudio = pendingAudio;const msg = submittedText.trim();- if (!msg && submittedImages.length === 0 && !submittedAudio) {+ if (!msg && submittedImages.length === 0 && submittedAudio.length === 0) {resetPromptQueue();return;}@@ -1326,11 +1362,11 @@ export function SharedComposer({} catch {}}- if (submittedAudio) {+ for (const clip of submittedAudio) {content.push({type: "audio",- name: submittedAudio.name,- audio: `data:${submittedAudio.contentType};base64,${submittedAudio.base64}`,+ name: clip.name,+ audio: `data:${clip.contentType};base64,${clip.base64}`,});}if (msg) {@@ -1386,7 +1422,7 @@ export function SharedComposer({const clearSubmittedDraft = () => {setCurrentText("");setPendingImages([]);- setPendingAudio(null);+ setPendingAudio([]);clearPendingAudioStore();textareaRef.current?.focus();};@@ -2217,11 +2253,12 @@ export function SharedComposer({const canSend =(text.trim().length > 0 ||pendingImages.length > 0 ||- pendingAudio !== null) &&+ pendingAudio.length > 0) &&!busy &&!isComposing &&!isDictating &&convertingImages === 0 &&+ addingFiles === 0 &&!sendUnavailableReason;// Compare mode swaps this composer in for the single-chat one and only one is ever on screen, so the@@ -2311,8 +2348,7 @@ export function SharedComposer({// Only the first send would carry staged attachments; refuse rather than clear.if (pendingImagesRef.current.length > 0 ||- pendingAudioRef.current ||- audioDecodingRef.current > 0 ||+ pendingAudioRef.current.length > 0 ||attachingRef.current > 0) {toast.error("Remove the staged attachment before running a list", {@@ -2583,20 +2619,22 @@ export function SharedComposer({onRemove={() => removePendingImage(id)}/>))}- {pendingAudio && (+ {pendingAudio.map((clip) => (<div+ key={clip.id}data-composer-attachment="audio"className="flex items-center gap-2 rounded-lg border border-[color-mix(in_oklab,var(--foreground)_calc(20%*var(--contrast-edge-gain,1)),transparent)] bg-muted px-3 py-1.5 text-xs"><HeadphonesIcon className="size-3.5 text-muted-foreground" /><span data-reload-snapshot-sensitive className="max-w-48 truncate">- {pendingAudio.name}+ {clip.name}</span><buttontype="button"onClick={() => {- setPendingAudio(null);- clearPendingAudioStore();+ setPendingAudio((prev) =>+ prev.filter((other) => other.id !== clip.id),+ );}}className="flex size-4 items-center justify-center rounded-full hover:bg-destructive hover:text-destructive-foreground"aria-label="Remove audio"@@ -2604,7 +2642,7 @@ export function SharedComposer({<XIcon className="size-3" /></button></div>- )}+ ))}</div>{skillMentions.popover}<ComposerDraftPreview text={text} />@@ -2684,6 +2722,7 @@ export function SharedComposer({ref={audioInputRef}type="file"accept={AUDIO_PICKER_ACCEPT}+ multipleclassName="hidden"onChange={(e) => {void addFiles(e.target.files);diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 029491032ef..f2810d00aca 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -625,6 +625,8 @@ export interface OpenAIChatCompletionsRequest {seed?: number;image_base64?: string;audio_base64?: string;+ /** Further clips after audio_base64, in attach order. */+ extra_audio_base64?: string[];video_base64?: string;use_adapter?: boolean | string | null;enable_thinking?: boolean | null;diff --git a/studio/frontend/src/features/native-intents/drop-paths.ts b/studio/frontend/src/features/native-intents/drop-paths.tsindex eee7677e064..df322210d76 100644--- a/studio/frontend/src/features/native-intents/drop-paths.ts+++ b/studio/frontend/src/features/native-intents/drop-paths.ts@@ -15,6 +15,7 @@ import {isTextAttachmentName,} from "../chat/text-attachment-accept.ts";import { RAG_UPLOAD_ACCEPT } from "../rag/types/rag.ts";+import { MAX_AUDIO_FILES } from "../../lib/audio-utils.ts";const DOC_EXTS = RAG_UPLOAD_ACCEPT.split(",").map((ext) =>ext.trim().toLowerCase(),@@ -70,7 +71,7 @@ const VIDEO_EXTS = CHAT_VIDEO_DROP_ACCEPT.split(",").map((ext) =>);/** What the window actually takes, for the rejection toast and the overlay. */-export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, one of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;+export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, up to ${MAX_AUDIO_FILES} of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;/** Last path segment of a native path, for display and extension checks. */export function nativeFileName(path: string): string {@@ -128,8 +129,8 @@ export function classifyDropPaths(paths: string[]): NativeDropClass {) {return { kind: "unsupported" };}- // The audio adapter takes one clip per message; a larger batch never attaches.- if (audio.length > 1) {+ // Over the per-message cap a batch would only partly attach. The adapter applies per-model limits.+ if (audio.length > MAX_AUDIO_FILES) {return { kind: "unsupported" };}// Same for video: one clip expands into a run of frames, so a batch woulddiff --git a/studio/frontend/src/lib/audio-utils.ts b/studio/frontend/src/lib/audio-utils.tsindex bdf95412d93..6944f51bdf5 100644--- a/studio/frontend/src/lib/audio-utils.ts+++ b/studio/frontend/src/lib/audio-utils.ts@@ -47,6 +47,31 @@ export function getAudioSizeError(size: number): string | null {: null;}+// Keep in sync with _MAX_AUDIO_CLIPS_PER_REQUEST. MAX_AUDIO_SIZE covers all clips together.+export const MAX_AUDIO_FILES = 8;++/** Max clips per message for a model. MLX takes one. */+export function maxAudioFilesFor(model: { isMlx?: boolean } | undefined): number {+ return model?.isMlx ? 1 : MAX_AUDIO_FILES;+}++export function getAudioAddError(+ count: number,+ totalSize: number,+ size: number,+ maxFiles: number = MAX_AUDIO_FILES,+): string | null {+ if (count >= maxFiles) {+ return maxFiles === 1+ ? "This model takes one audio file per message. Load a GGUF model to send several."+ : `Up to ${maxFiles} audio files can be attached per message.`;+ }+ if (count > 0 && totalSize + size > MAX_AUDIO_SIZE) {+ return `Audio files together exceed the ${MAX_AUDIO_SIZE_LABEL} per-message limit`;+ }+ return getAudioSizeError(size);+}+export function fileToBase64(file: File): Promise<string> {return new Promise((resolve, reject) => {const reader = new FileReader();diff --git a/studio/frontend/tests/chat-attachment-display.test.ts b/studio/frontend/tests/chat-attachment-display.test.tsindex 5235bd34c8d..721c234fc80 100644--- a/studio/frontend/tests/chat-attachment-display.test.ts+++ b/studio/frontend/tests/chat-attachment-display.test.ts@@ -6,6 +6,12 @@ import assert from "node:assert/strict";import test from "node:test";import { attachedMediaUnavailableReason } from "../src/features/chat/lib/attached-media-gate.ts";+import {+ getAudioAddError,+ MAX_AUDIO_FILES,+ MAX_AUDIO_SIZE,+ maxAudioFilesFor,+} from "../src/lib/audio-utils.ts";import {ATTACHMENT_KIND_ICON_CLASS,attachmentFileKind,@@ -184,6 +190,44 @@ test("audio or video attached before a model loaded is checked when sent", () =>);});+test("several clips are sent to GGUF and transformers, but refused on MLX", () => {+ const base = { checkpoint: "unsloth/gemma-4-E4B-it", modelLabel: "Gemma" };+ const gguf = { hasAudioInput: true, isMlx: false };+ const mlx = { hasAudioInput: true, isMlx: true };+ assert.equal(+ attachedMediaUnavailableReason({ ...base, activeModel: gguf, audio: true, audioCount: 3, video: false }),+ null,+ );+ assert.equal(+ attachedMediaUnavailableReason({ ...base, activeModel: mlx, audio: true, audioCount: 1, video: false }),+ null,+ );+ assert.match(+ attachedMediaUnavailableReason({ ...base, activeModel: mlx, audio: true, audioCount: 2, video: false }) ?? "",+ /^Gemma takes one audio file per message\./,+ );+});++test("audio caps cover a message's clips together", () => {+ assert.equal(getAudioAddError(0, 0, 1024), null);+ assert.equal(getAudioAddError(3, 10 * 1024 * 1024, 10 * 1024 * 1024), null);+ assert.match(+ getAudioAddError(1, 20 * 1024 * 1024, 10 * 1024 * 1024) ?? "",+ /together exceed/,+ );+ assert.match(getAudioAddError(0, 0, MAX_AUDIO_SIZE + 1) ?? "", /exceeds/);+ assert.match(+ getAudioAddError(MAX_AUDIO_FILES, 0, 1) ?? "",+ new RegExp(`Up to ${MAX_AUDIO_FILES} audio files`),+ );+ assert.equal(maxAudioFilesFor({ isMlx: true }), 1);+ assert.equal(maxAudioFilesFor(undefined), MAX_AUDIO_FILES);+ assert.match(+ getAudioAddError(1, 0, 1, maxAudioFilesFor({ isMlx: true })) ?? "",+ /one audio file per message/,+ );+});+test("attaching no longer needs a loaded model, only a capable one when one is loaded", async () => {const audio = await readSrcAsync("features/chat/audio-attachment-adapter.ts");const video = await readSrcAsync("features/chat/video-attachment-adapter.ts");diff --git a/studio/frontend/tests/native-drop-paths.test.ts b/studio/frontend/tests/native-drop-paths.test.tsindex 27c715077ff..88ff8e20004 100644--- a/studio/frontend/tests/native-drop-paths.test.ts+++ b/studio/frontend/tests/native-drop-paths.test.ts@@ -53,6 +53,7 @@ import {} from "../src/features/native-intents/drop-paths.ts";import type { NativeIntent } from "../src/features/native-intents/types.ts";import { RAG_UPLOAD_ACCEPT } from "../src/features/rag/types/rag.ts";+import { MAX_AUDIO_FILES } from "../src/lib/audio-utils.ts";import { MAX_REFERENCE_BYTES } from "../src/features/video/reference-budget.ts";import {VIDEO_ACCEPT,@@ -535,22 +536,39 @@ test("a single audio file routes to chat audio attachments", () => {]);});-// One clip per message, so a larger batch is turned away before it is read.-test("multi-audio drops are rejected before they are routed", () => {+// Models like Gemma 4 take several clips in one message, so a batch routes whole.+test("multi-audio drops route to chat audio attachments", () => {const dropped = classifyDropPaths(["/clips/take.WAV","/clips/note.mp3","/clips/voice.flac",]);- assert.equal(dropped.kind, "unsupported");+ assert.equal(dropped.kind, "audio");+ assert.deepEqual(dropped.kind === "audio" ? dropped.paths : [], [+ "/clips/take.WAV",+ "/clips/note.mp3",+ "/clips/voice.flac",+ ]);});-test("a second clip alongside other attachments is rejected too", () => {+test("several clips alongside other attachments route together", () => {const dropped = classifyDropPaths(["/docs/a.pdf","/clips/note.mp3","/clips/voice.flac",]);+ assert.equal(dropped.kind, "attach");+ assert.deepEqual(dropped.kind === "attach" ? dropped.audio : [], [+ "/clips/note.mp3",+ "/clips/voice.flac",+ ]);+});++// Past the per-message cap a batch would attach only partly, so it is turned away before it is read.+test("an audio batch over the per-message cap is rejected", () => {+ const dropped = classifyDropPaths(+ Array.from({ length: MAX_AUDIO_FILES + 1 }, (_, i) => `/clips/${i}.wav`),+ );assert.equal(dropped.kind, "unsupported");});diff --git a/studio/frontend/tests/reload-snapshot.test.ts b/studio/frontend/tests/reload-snapshot.test.tsindex 8ac8a198bc8..117688ae680 100644--- a/studio/frontend/tests/reload-snapshot.test.ts+++ b/studio/frontend/tests/reload-snapshot.test.ts@@ -1391,7 +1391,7 @@ test("carries live form state, except what sensitive fields hide", () => {);assert.match(sharedComposerSource,- /data-reload-snapshot-sensitive[\s\S]*?pendingAudio\.name/,+ /pendingAudio\.map\([\s\S]*?data-reload-snapshot-sensitive[\s\S]*?\{clip\.name\}/,);// Both carriers: the tooltip on the name, and the accessible name on the// remove button, a sibling no ancestor marker would reach.