Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: batch MLX tool-protocol, response_format and video replies; several clips and Gemma 4 video on MLX (#15920)
Long Yixingcb07e19f0dc4
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex 355bc9f8c27..ee6252119c6 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -2591,7 +2591,7 @@ def messages_with_attached_image(fallback_user_text: str = "",structured_content: bool = False,image: int = 1,- video: bool = False,+ video: int = 0,audio: Any = None,extra_audio: Sequence[Any] = (),) -> list:diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.pyindex 44e634061c0..08e9a6f7608 100644--- a/studio/backend/core/inference/managed_engine.py+++ b/studio/backend/core/inference/managed_engine.py@@ -1199,7 +1199,7 @@ class ManagedEngine:raise ValueError("Tools and adapter comparisons are not supported by this optional engine profile.")- if params.get("video"):+ if params.get("videos"):raise ValueError("Video input is not yet supported by the Unsloth managed engine transport.")diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 3502c480db7..2ee53103b4e 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -13,7 +13,7 @@ import shutilimport sysimport threadingimport time-from collections import OrderedDict+from collections import Counter, OrderedDictfrom contextlib import ExitStack, closing, contextmanager, nullcontextfrom typing import Optional, Generatorfrom core.inference.worker import RowRefused@@ -1589,6 +1589,43 @@ def _video_placeholder(processor):return None+def _marks_video(target, processor) -> bool:+ """Whether ``target`` renders a video part as a marker: a render that merely differs from the+ text one can still be prose, as in the image probe."""+ from core.inference.chat_template_helpers import apply_chat_template_for_generation++ marked = apply_chat_template_for_generation(target, _VIDEO_PROBE_MESSAGES)+ if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):+ return False+ if _vlm_prompt_issue(marked, _VIDEO_PROBE_MESSAGES):+ return False+ placeholder = _video_placeholder(processor)+ return placeholder is None or placeholder in marked+++def _video_render_target(processor):+ """What renders a turn with video. mlx-vlm's Gemma 4 processor rebuilds each message from media+ counts and drops a video part it holds no clip for; its tokenizer carries the same template and+ renders the part as written. A processor that keeps the marker keeps rendering, since some+ deliberately differ from their tokenizer (MiniCPM-V's non-thinking prefix)."""+ from core.inference.chat_template_helpers import chat_render_target++ target = chat_render_target(processor)+ tokenizer = getattr(processor, "tokenizer", None)+ template = getattr(processor, "chat_template", None)+ if (+ target is not processor+ or not template+ or getattr(tokenizer, "chat_template", None) != template+ ):+ return target+ try:+ drops = not _marks_video(processor, processor)+ except Exception:+ drops = True+ return tokenizer if drops else processor++def _mlx_lm_reads_images(model) -> bool:"""An image model unsloth-zoo loaded through mlx-lm, with an unsloth-zoo that prefills its images."""if getattr(model, "_unsloth_supports_images", None) is not True:@@ -1601,26 +1638,13 @@ def _mlx_lm_reads_images(model) -> bool:def _mlx_reads_video(processor) -> bool:- """Processor carries a video component AND the template places a video marker: a render that- merely differs from the text one can still be prose, as in the image probe."""+ """Processor carries a video component AND the template places a video marker."""if processor is None or getattr(processor, "video_processor", None) is None:return Falseif not _mlx_vlm_decodes_video():return False- from core.inference.chat_template_helpers import (- apply_chat_template_for_generation,- chat_render_target,- )-try:- target = chat_render_target(processor)- marked = apply_chat_template_for_generation(target, _VIDEO_PROBE_MESSAGES)- if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):- return False- if _vlm_prompt_issue(marked, _VIDEO_PROBE_MESSAGES):- return False- placeholder = _video_placeholder(processor)- return placeholder is None or placeholder in marked+ return _marks_video(_video_render_target(processor), processor)except BaseException:# A load must never fail on a probe, matching _image_marker_survives.return False@@ -1697,9 +1721,17 @@ def _video_sampling_target(processor) -> tuple[float, int, Optional[int]]:)-def _video_frame_rate(clip_path: str, processor) -> Optional[float]:- """The sampling ``fps`` keeping the decoded stack within the budget; None when the clip has no- readable frame geometry or count, which mlx-vlm then refuses itself."""+class _ClipsTooLarge(RuntimeError):+ pass+++def _video_frame_rate(+ clip_path: str,+ processor,+ clips: int = 1,+) -> Optional[float]:+ """The sampling ``fps`` keeping the decoded stack within this clip's share of the budget; None+ when the clip has no readable frame geometry or count, which mlx-vlm then refuses itself."""import cv2capture = cv2.VideoCapture(clip_path)@@ -1718,16 +1750,31 @@ def _video_frame_rate(clip_path: str, processor) -> Optional[float]:# The one measurement mlx-vlm substitutes rather than refusing; match its substitution.native_fps = 1.0wanted_fps, min_frames, nframes = _video_sampling_target(processor)- affordable = _VIDEO_DECODE_BUDGET_BYTES // (2 * 3 * width * height)+ # mlx-vlm holds every decoded clip until the processor runs, so the clips share one budget.+ budget = _VIDEO_DECODE_BUDGET_BYTES // clips+ affordable = budget // (2 * 3 * width * height)needed = min_frames if nframes is None else nframesif affordable < needed:- raise RuntimeError(+ raise _ClipsTooLarge(f"The video's {width}x{height} frames are too large to decode: {needed} of them "- f"exceed the {_VIDEO_DECODE_BUDGET_BYTES >> 20} MiB frame budget."+ f"exceed the {budget >> 20} MiB frame budget.")return min(wanted_fps, affordable * native_fps / total_frames)+def _video_sampling_fps(+ clip_paths,+ processor,+ rows = 1,+) -> Optional[float]:+ """One rate for every clip, since mlx-vlm before 0.7.3 applies a single ``fps`` to all.++ ``rows`` copies of the clips share the decode budget when a batch decodes them together.+ """+ rates = [_video_frame_rate(path, processor, len(clip_paths) * rows) for path in clip_paths]+ return min((rate for rate in rates if rate is not None), default = None)++def _discard_video_clip(path: str) -> None:try:os.remove(path)@@ -1935,7 +1982,10 @@ class _MlxStreamedText:token_id = getattr(response, "token", None)if token_id is None:return ""- token_id = int(token_id)+ return self.feed_token(int(token_id))++ def feed_token(self, token_id: int) -> str:+ """One sampled id, for a runtime that reports ids one at a time."""if token_id in self._stop_ids:self._held_stops.append(token_id)return ""@@ -1976,6 +2026,25 @@ class _MlxStreamedText:return closing+def _require_row_detokenizer(streamed):+ """A batch row feeds ids one at a time, so unlike the single path it has no runtime text to fall back on."""+ if streamed is not None and streamed._detokenizer is None:+ raise RowRefused("no independent detokenizer to decode this row's ids")+++def _native_reply_text(decoder, token_ids, stop_ids, prefix) -> str:+ """A native-protocol reply re-decoded from its ids. A runtime may stop on an allowlisted+ control (<|end_message|>): the trailing stop id stays only when it closes an open envelope,+ judged with the prefill, since a restored <think> lives in the prompt."""+ ids = list(token_ids)+ if ids and ids[-1] in stop_ids:+ whole = decoder.decode(ids)+ closer = whole[len(decoder.decode(ids[:-1])) :]+ if not closes_an_open_envelope(prefix + whole, closer):+ ids = ids[:-1]+ return decoder.decode(ids)++def _mlx_stop_cut(text: str, stops) -> tuple[int, bool]:"""Holds back text that could grow into a stop; a rewriting caller must withhold until turn ends"""resolved = text.rstrip("\ufffd")@@ -3572,6 +3641,14 @@ def _make_mlx_logit_bias_processor(logit_bias: dict):return _processor+_DIFFUSION_RESPONSE_FORMAT = (+ "response_format is not supported on this model: it generates by "+ "diffusion rather than one token at a time, so no grammar can "+ "constrain the next token. Load an autoregressive model to use "+ "guided decoding."+)++def _vlm_generation_is_diffusion(model) -> bool:try:from mlx_vlm.generate.diffusion import is_diffusion_model@@ -3933,6 +4010,15 @@ class _RowStream:return [self._prefix + self.sampled[: self._cut]]return []+ def flush(self, text):+ """Text a decoder held until the reply ended; ``settle`` delivers it for normalized or re-decoded rows."""+ if not self._sequences or (self._normalizer is None and self._decode is None):+ # feed releases up to a stop it finds here, which settle no longer would once stopped.+ return self.feed(text = text)+ self.sampled += text+ self._cut, self.stopped = _mlx_stop_cut(self.sampled, self._sequences)+ return []+def settle(self):if not self._sequences:if self._decode is not None and self.sampled.endswith("\ufffd"):@@ -3997,21 +4083,59 @@ def _row_processor_gap(engine, requests):return unavailable() if callable(unavailable) else None+def _constrained(request):+ return request.get("response_format") not in (None, {"type": "text"}) or (+ request.get("grammar") is not None+ )++def _request_batch_gap(request):"""What a reply asks for that only the one-at-a-time decode serves, or None."""if request.get("images") or request.get("images_base64"):return "a reply replays tool images"- if request.get("video") is not None or request.get("video_base64"):- return "a reply carries a video clip"- if request.get("tools") or request.get("tool_protocol_active"):- return "a reply runs the tool protocol"- if request.get("response_format") is not None or request.get("grammar") is not None:- return "a reply is constrained to a response format"if request.get("n_probs"):return "a reply reports token probabilities"return None+def _carries_video(request):+ return bool(request.get("videos") or request.get("videos_base64"))+++def _batch_clips(stack, plans, processor):+ """Each row's clips as files mlx-vlm reads, removed when ``stack`` closes.++ Rows with the same clips share their files, so unsloth-zoo batches them as one group, and that+ group decodes its clips together, so its rows share one decode budget. Rows whose clips are too+ large to share it get files of their own, so each decodes alone with the whole budget.+ """+ files, shared = [], {}+ stack.callback(lambda: [_discard_video_clip(path) for path in files])++ def write(clip):+ files.append(_write_video_clip(clip))+ return files[-1]++ rows = Counter(tuple(plan.videos or ()) for plan in plans)+ clips = []+ for plan in plans:+ videos = tuple(plan.videos or ())+ paths = [+ shared[clip] if clip in shared else shared.setdefault(clip, write(clip))+ for clip in videos+ ]+ if not paths:+ clips.append({})+ continue+ try:+ rate = _video_sampling_fps(paths, processor, rows[videos])+ except _ClipsTooLarge:+ paths = [write(clip) for clip in videos]+ rate = _video_sampling_fps(paths, processor)+ clips.append({"videos": paths, "fps": rate})+ return clips++def _batched_vision_engine_gap(engine):def fields(name):return _engine_fields(engine, name)@@ -4035,7 +4159,7 @@ class _TextRowPlan:"sampler","processors","max_tokens",- "detokenizes",+ "streamed",)def __init__(@@ -4047,12 +4171,12 @@ class _TextRowPlan:sampler,processors,max_tokens,- detokenizes = False,+ streamed = None,):self.prompt = promptself.think_prefix = think_prefixself.stream = stream- self.detokenizes = detokenizes+ self.streamed = streamedself.sampler = samplerself.processors = processorsself.max_tokens = max_tokens@@ -4067,14 +4191,14 @@ class _TextBatchRow:"all_tokens","prompt_len","cached",- "detokenizer","generated","reason","processing_at","ready_at",+ "failure",)- def __init__(self, *, handle, plan, cache_key, all_tokens, prompt_len, cached, detokenizer):+ def __init__(self, *, handle, plan, cache_key, all_tokens, prompt_len, cached):self.handle = handleself.plan = planself.uid = None@@ -4082,11 +4206,41 @@ class _TextBatchRow:self.all_tokens = all_tokensself.prompt_len = prompt_lenself.cached = cached- self.detokenizer = detokenizerself.generated = 0self.reason = Noneself.processing_at = Noneself.ready_at = None+ self.failure = None+++def _failing_alone(row, processors):+ """A row's processors, whose failure is kept on the row rather than raised through the batch."""++ def guarded(processor):+ def _processor(tokens, logits):+ if row.failure is None:+ try:+ return processor(tokens, logits)+ except Exception as exc:+ row.failure = exc+ return logits++ # A drafted batch drafts only for rows whose processors all read just the history.+ _processor.history_only = getattr(processor, "history_only", False)+ return _processor++ return [guarded(processor) for processor in processors]+++def _streamed_tail(row):+ """What a row decoded from ids still holds when it ends, released as the single path's finish."""+ stream = row.plan.stream+ if row.plan.streamed is None or stream.stopped:+ return []+ tail = stream.flush(row.plan.streamed.finish(stream.sampled))+ if stream.stopped:+ row.reason = "stop"+ return taildef _row_kwargs(request):@@ -4179,6 +4333,10 @@ class _TextBatchSession:_messages_with_system_prompt(messages, request.get("system_prompt", ""))if image is Noneelse backend._conversation_to_render(messages, request.get("system_prompt", ""), image),+ tool_protocol_active = request.get("tool_protocol_active"),+ response_format = request.get("response_format"),+ reasoning_is_extracted = request.get("reasoning_is_extracted", False),+ grammar = request.get("grammar"),**row_kwargs,)began = time.perf_counter()@@ -4205,7 +4363,6 @@ class _TextBatchSession:all_tokens = all_tokens,prompt_len = len(remainder) if image is None else len(all_tokens),cached = len(prefix) if image is None else 0,- detokenizer = backend._tokenizer.detokenizer if plan.detokenizes else None,)if image is not None:row.processing_at = began@@ -4215,7 +4372,7 @@ class _TextBatchSession:caches = [cache],all_tokens = [prefix],samplers = [plan.sampler],- logits_processors = [_row_logits_processors(plan.processors)],+ logits_processors = [_failing_alone(row, _row_logits_processors(plan.processors))],)if len(uids) != 1:self.generator.remove(list(uids))@@ -4245,6 +4402,13 @@ class _TextBatchSession:row = self._by_uid.get(event.uid)if row is not None and row.processing_at is None:row.processing_at = began+ # A processor runs a step ahead, so the single path raises before yielding this token;+ # reported first, so a sibling ending in this step cannot answer the request as done.+ for row in [row for row in self._rows.values() if row.failure is not None]:+ self.generator.remove([row.uid])+ self._rows.pop(row.handle, None)+ self._by_uid.pop(row.uid, None)+ yield row.handle, row.failurefor event in events:row = self._by_uid.get(event.uid)if row is None:@@ -4258,18 +4422,9 @@ class _TextBatchSession:def _feed(self, row, event):stream = row.plan.stream- text = None- if event.finish_reason is None:- if row.detokenizer is not None:- row.detokenizer.add_token(event.token)- text = row.detokenizer.last_segment- else:- text = ""- if row.detokenizer is not None:- if event.finish_reason == "length":- row.detokenizer.add_token(event.token)- row.detokenizer.finalize()- text = row.detokenizer.last_segment+ # A stop id is held, not written: the end-of-reply flush decides whether it closes markup.+ streamed = row.plan.streamed+ text = "" if streamed is None else streamed.feed_token(int(event.token))row.generated += 1yield from ((row.handle, snapshot) for snapshot in stream.feed(token = event.token, text = text)@@ -4318,6 +4473,7 @@ class _TextBatchSession:self._rows.pop(row.handle, None)self._by_uid.pop(row.uid, None)stream = row.plan.stream+ yield from ((row.handle, snap) for snap in _streamed_tail(row))yield from ((row.handle, snap) for snap in stream.settle())yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))ready_at = row.ready_at or time.perf_counter()@@ -4351,12 +4507,15 @@ class _VLMRowPlan:__slots__ = ("prompt","images",+ "videos","think_prefix","stream","kwargs","sampling","max_tokens","processors",+ "streamed",+ "speculates",)def __init__(@@ -4370,15 +4529,21 @@ class _VLMRowPlan:sampling,max_tokens,processors = None,+ streamed = None,+ speculates = True,+ videos = None,):self.prompt = promptself.images = images+ self.videos = videosself.think_prefix = think_prefixself.stream = streamself.kwargs = kwargsself.sampling = samplingself.max_tokens = max_tokensself.processors = processors+ self.streamed = streamed+ self.speculates = speculatesclass _VisionBatchRow:@@ -4394,9 +4559,16 @@ class _VisionBatchRow:"ready_at","cancelled","drafts",+ "failure",)- def __init__(self, *, handle, plan, row):+ def __init__(+ self,+ *,+ handle,+ plan,+ row = None,+ ):self.handle = handleself.plan = planself.row = row@@ -4408,6 +4580,14 @@ class _VisionBatchRow:self.ready_at = Noneself.cancelled = Falseself.drafts = None+ self.failure = None+++def _vision_batch_reports_ids(engine, speculative):+ """Whether vision rows can be decoded from their sampled ids, as the one-at-a-time decode does."""+ return "token_ids" in _engine_fields(engine, "GenerationEvent") and (+ speculative is None or "drafts" in _engine_fields(engine, "GenerationRequest")+ )def _row_prompt_cache_gap():@@ -4435,6 +4615,7 @@ class _VisionBatchSession:adapter_state = None,owns_model = False,):+ from unsloth_zoo.mlx import generate as enginefrom unsloth_zoo.mlx.generate import BatchStream, GenerationDefaultsself.backend = backend@@ -4452,6 +4633,7 @@ class _VisionBatchSession:)self._held.enter_context(backend._int8_prefill_scope())self._speculative = getattr(backend, "_speculative_draft", None)+ self._decodes_ids = _vision_batch_reports_ids(engine, self._speculative)self.stream = BatchStream(backend._model,backend._processor,@@ -4491,7 +4673,8 @@ class _VisionBatchSession:)backend = self.backend- plan = backend._plan_vlm_request(request)+ plan = backend._plan_vlm_request(request, decodes_ids = self._decodes_ids)+ row = _VisionBatchRow(handle = handle, plan = plan)state = (backend._vlm_batch_row_cache(self._adapter_state, plan.images)if self._resumes_rows@@ -4511,16 +4694,16 @@ class _VisionBatchSession:image = plan.images[0] if plan.images else None,max_tokens = plan.max_tokens,sampling = SamplingParams(**plan.sampling),- logits_processors = plan.processors,+ logits_processors = plan.processors and _failing_alone(row, plan.processors),**resume,+ **({} if self._speculative is None or plan.speculates else {"drafts": False}),))except BatchRowRefused as refusal:raise RowRefused(str(refusal)) from refusal- row = _VisionBatchRow(handle = handle, plan = plan, row = row_number)- if self._speculative is not None:- # A speculative batch prefills the row inside add(), so its clock starts before it.- row.admitted_at = admitted_at+ row.row = row_number+ # A speculative batch prefills the row inside add(), so its clock starts before it.+ row.admitted_at = admitted_at if self._speculative is not None else time.perf_counter()self._rows[handle] = rowself._by_row[row_number] = rowlogger.info(@@ -4536,22 +4719,36 @@ class _VisionBatchSession:now, stopped = None, {}# Only a speculative step spans many tokens, reported as they are decoded; only its unsloth-zoo has the call.# The batch cannot change mid-step, so a row that stopped itself ends the step and leaves after it.- for event in (- self.stream.iter_step(lambda: bool(stopped) or (waiting is not None and waiting()))+ failing = {}+ events = (+ self.stream.iter_step(+ lambda: bool(stopped or failing) or (waiting is not None and waiting())+ )if self._speculative is not Noneelse self.stream.step()- ):+ )+ # As in _TextBatchSession.step: reported before the step's tokens, which it never yields.+ yield from self._fail([row for row in self._rows.values() if row.failure is not None])+ for event in events:row = self._by_row.get(event.index)if row is None:continue+ if row.failure is not None:+ # Only a drafted step decodes as it is read, so a row can fail mid-step and leave after it.+ failing[row.handle] = row+ continueif now is None:now = time.perf_counter()if row.ready_at is None:row.ready_at = now- if event.delta and not row.cancelled:- yield from (- (row.handle, snapshot) for snapshot in row.plan.stream.feed(text = event.delta)- )+ streamed = row.plan.streamed+ text = (+ event.delta+ if streamed is None+ else "".join(streamed.feed_token(int(token)) for token in event.token_ids)+ )+ if text and not row.cancelled:+ yield from ((row.handle, snapshot) for snapshot in row.plan.stream.feed(text = text))if row.plan.stream.stopped:row.cancelled = Truerow.reason = "stop"@@ -4563,10 +4760,18 @@ class _VisionBatchSession:yield from self._retire(row, cancelled = row.cancelled)elif row.cancelled:stopped[row.handle] = row+ yield from self._fail(list(failing.values()))for row in stopped.values():if row.handle in self._rows:yield from self._retake(row)+ def _fail(self, rows):+ for row in rows:+ self.stream.withdraw(row.row)+ self._rows.pop(row.handle, None)+ self._by_row.pop(row.row, None)+ yield row.handle, row.failure+def withdraw(self, handles):for handle in list(handles):row = self._rows.get(handle)@@ -4592,6 +4797,7 @@ class _VisionBatchSession:self._rows.pop(row.handle, None)self._by_row.pop(row.row, None)stream = row.plan.stream+ yield from ((row.handle, snap) for snap in _streamed_tail(row))yield from ((row.handle, snap) for snap in stream.settle())yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))ready_at = row.ready_at or time.perf_counter()@@ -5197,7 +5403,7 @@ class MLXInferenceBackend:if videos:# Same fps as generation, not the processor's default rate.kwargs["videos"] = videos- fps = _video_frame_rate(videos[0], self._processor)+ fps = _video_sampling_fps(videos, self._processor)if fps is not None:kwargs["fps"] = fpsinputs = prepare_inputs(@@ -6019,9 +6225,15 @@ class MLXInferenceBackend:full_messages.extend(messages)return full_messages- def _conversation_to_render(self, messages, system_prompt, image):+ def _conversation_to_render(+ self,+ messages,+ system_prompt,+ image,+ video = 0,+ ):"""The turns this request renders, whichever way its reply is decoded."""- if (self._is_vlm or self._reads_vision_input()) and image is not None:+ if (self._is_vlm or self._reads_vision_input()) and (image is not None or video):from core.inference.chat_template_helpers import (chat_render_target as _chat_render_target,)@@ -6032,6 +6244,8 @@ class MLXInferenceBackend:self._processor is not Noneand _chat_render_target(self._processor) is self._processor),+ image = int(image is not None),+ video = video,)else:full_messages = self._with_system_prompt(messages, system_prompt)@@ -6104,11 +6318,11 @@ class MLXInferenceBackend:_adapter_state = None,# Unrestricted mode uses an EMPTY tools list, so bool(tools) cannot detect the protocoltool_protocol_active = None,- video = None,+ videos = None,) -> Generator[str, None, None]:if self._model is None:raise RuntimeError("No model loaded")- if video is not None:+ if videos:if not _mlx_vlm_decodes_video():raise RuntimeError("The installed mlx-vlm does not read video.")if not self._reads_vision_input():@@ -6122,7 +6336,7 @@ class MLXInferenceBackend:# History first, the attachment last: MLX binds pixels to markers in order.attached = list(images or []) + ([image] if image is not None else [])# Shared with the transformers vision path so both render the same turns (#10092).- if self._reads_vision_input() and (attached or video is not None):+ if self._reads_vision_input() and (attached or videos):# Processor templates want part lists, the tokenizer fallback wants strings.from core.inference.chat_template_helpers import (chat_render_target as _chat_render_target,@@ -6137,7 +6351,7 @@ class MLXInferenceBackend:system_prompt = system_prompt,structured_content = _renders_via_processor,image = len(attached),- video = video is not None,+ video = len(videos or ()),)# That helper skips convos with markers, but replayed MCP picture markers belong to other pictures, so the render would be one short_prior_markers = image_marker_parts(full_messages)@@ -6188,7 +6402,7 @@ class MLXInferenceBackend:tool_protocol_active = tool_protocol_active,response_format = response_format,reasoning_is_extracted = reasoning_is_extracted,- video = video,+ videos = videos,)else:stream = self._generate_text(@@ -6555,18 +6769,9 @@ class MLXInferenceBackend:# Re-decoding every id rebuilds rather than extends, so an invalid byte sequence can revise# characters already shown. Predates stop handling and affects plain replies too.if native_token_decoder is not None:- # A runtime may stop on an allowlisted control (<|end_message|>): drop only the trailing id, so a real tool closer survives- _ids = list(token_ids)- if _ids and _ids[-1] in _mlx_stop_token_ids(- self._tokenizer, self._model- ):- # Only without tool markup: the same marker can be the closer that strict parsing needs.- _whole = native_token_decoder.decode(_ids)- _closer = _whole[len(native_token_decoder.decode(_ids[:-1])) :]- # Include the prefill: a restored <think> lives in the prompt, and judging the closer on generated ids alone left the block open- if not closes_an_open_envelope(think_prefix + _whole, _closer):- _ids = _ids[:-1]- sampled = native_token_decoder.decode(_ids)+ sampled = _native_reply_text(+ native_token_decoder, token_ids, stop_token_ids, think_prefix+ )else:sampled = self._tokenizer.decode(token_ids,@@ -6676,12 +6881,17 @@ class MLXInferenceBackend:tool_protocol_active = None,prompt_ids = None,penalty_window = None,+ response_format = None,+ reasoning_is_extracted = False,+ grammar = None,) -> "_TextRowPlan":from mlx_lm.sample_utils import make_samplerif prompt_ids is not None:prompt, reasoning_channel_markers, think_prefix = list(prompt_ids), None, ""_resumed_partial = False+ preserve_native_channels = False+ constraint = _raw_prompt_constraint(grammar, self._tokenizer)else:render_result = self._render_text_prompt(messages,@@ -6695,18 +6905,30 @@ class MLXInferenceBackend:reasoning_channel_markers = render_result.reasoning_channel_markers_resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))+ preserve_native_channels = reasoning_channel_markers is not None+ think_close_survives = (+ bool(tools) or tool_protocol_active or preserve_native_channels+ ) and decoder_preserves_token(+ self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)+ )think_prefix = _think_prefix(prompt,getattr(self._tokenizer, "all_special_tokens", None),messages,continue_final_message,- preserves_think_close = (- bool(tools) or tool_protocol_active or reasoning_channel_markers is not None- )- and decoder_preserves_token(- self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)- ),+ preserves_think_close = think_close_survives,+ )+ constraint = _build_grammar_constraint(+ response_format,+ self._tokenizer,+ prompt,+ reasoning_markers = reasoning_channel_markers,+ reasoning_is_extracted = reasoning_is_extracted,+ reply_keeps_special_tokens = preserve_native_channels or think_close_survives,)+ if constraint is not None and not constraint.allows_reasoning:+ think_prefix = ""+ preserve_native_channels = Falseif seed is None:sampler = make_sampler(temp = temperature,@@ -6728,6 +6950,7 @@ class MLXInferenceBackend:presence_penalty = presence_penalty,frequency_penalty = frequency_penalty,logit_bias = logit_bias,+ grammar_constraint = constraint,penalty_window = penalty_window,penalty_prompt = prompt_ids,)@@ -6739,28 +6962,65 @@ class MLXInferenceBackend:prompt, reasoning_channel_markers, _resumed_partial),)- if reasoning_channel_markers is not None+ if preserve_native_channels+ else None+ )+ # As _generate_text decodes the reply, so a batched turn shows the same native controls.+ native_token_decoder = (+ NativeToolTokenDecoder(+ self._tokenizer,+ preserved_tokens = reasoning_control_tokens(reasoning_channel_markers),+ )+ if tools or preserve_native_channels or tool_protocol_active+ else None+ )+ if constraint is not None:+ constraint.decoded_dropping(+ native_token_decoder.dropped_ids() if native_token_decoder is not None else None+ )+ stop_token_ids = (+ _mlx_stop_token_ids(self._tokenizer, self._model)+ if native_token_decoder is not None+ else ()+ )+ streamed = (+ _MlxStreamedText(+ _mlx_stream_detokenizer(self._tokenizer),+ native_token_decoder,+ self._tokenizer,+ stop_token_ids,+ )+ if preserve_native_channelselse None)+ _require_row_detokenizer(streamed)+ if streamed is not None:+ decode = None+ elif native_token_decoder is not None:+ decode = lambda ids: _native_reply_text(+ native_token_decoder, ids, stop_token_ids, think_prefix+ )+ else:+ decode = lambda ids: self._tokenizer.decode(+ ids,+ skip_special_tokens = True,+ **(+ {"clean_up_tokenization_spaces": False}+ if constraint is not None or prompt_ids is not None+ else {}+ ),+ )row = _RowStream(prefix = think_prefix,sequences = _mlx_stop_sequences(stop),normalizer = normalizer,- decode = (- None- if normalizer is not None- else lambda ids: self._tokenizer.decode(- ids,- skip_special_tokens = True,- **({"clean_up_tokenization_spaces": False} if prompt_ids is not None else {}),- )- ),+ decode = decode,)return _TextRowPlan(prompt = prompt,think_prefix = think_prefix,stream = row,- detokenizes = normalizer is not None,+ streamed = streamed,sampler = sampler,processors = logits_processors,max_tokens = (@@ -6791,7 +7051,9 @@ class MLXInferenceBackend:)# Same render target as the healing catalog the route builds (#7066)- chat_target = chat_render_target(self._processor)+ chat_target = (+ _video_render_target(self._processor) if videos else chat_render_target(self._processor)+ )attached_images = 0 if images is None else len(images)structured_images = sum(@@ -6924,15 +7186,51 @@ class MLXInferenceBackend:prompt, markers = recovered.prompt, recovered.reasoning_channel_markersreturn prompt, chat_target, markers- def _plan_vlm_request(self, request) -> "_VLMRowPlan":+ def _media_generation_limit(self, prompt, max_new_tokens, prompt_ids, images, videos):+ # Counted from a temp file (base64 is unreadable to the processor), discarded here.+ counted_clips = []+ try:+ if getattr(self, "_kv_context_budget", None):+ for clip in videos or ():+ counted_clips.append(_write_video_clip(clip))+ return self._generation_limit(+ prompt,+ max_new_tokens,+ prompt_ids,+ images = images,+ videos = counted_clips or None,+ cap = UNSET_GENERATION_BUDGET if images or videos else None,+ )+ finally:+ for counted_clip in counted_clips:+ _discard_video_clip(counted_clip)++ def _plan_vlm_request(+ self,+ request,+ *,+ decodes_ids = False,+ ) -> "_VLMRowPlan":image = request.get("image")+ videos = request.get("videos") or Nonemessages = request.get("messages") or []conversation = ([]if request.get("prompt_ids") is not None- else self._conversation_to_render(messages, request.get("system_prompt", ""), image)+ else self._conversation_to_render(+ messages, request.get("system_prompt", ""), image, video = len(videos or ())+ )+ )+ return self._plan_vlm_row(+ conversation,+ image,+ tool_protocol_active = request.get("tool_protocol_active"),+ response_format = request.get("response_format"),+ reasoning_is_extracted = request.get("reasoning_is_extracted", False),+ decodes_ids = decodes_ids,+ videos = videos,+ **_row_kwargs(request),)- return self._plan_vlm_row(conversation, image, **_row_kwargs(request))def _plan_vlm_row(self,@@ -6957,14 +7255,23 @@ class MLXInferenceBackend:stop = None,prompt_ids = None,penalty_window = None,+ tool_protocol_active = None,+ response_format = None,+ reasoning_is_extracted = False,+ decodes_ids = False,+ videos = None,) -> "_VLMRowPlan":+ """``decodes_ids``: the batch reports each row's ids, so the reply is decoded from them as+ ``_generate_vlm`` decodes it; otherwise from the batch's text, which keeps no native controls."""images = [image] if image is not None else None+ constraint = token_decoder = Noneif prompt_ids is not None:prompt, markers, think_prefix = list(prompt_ids), None, ""else:prompt, chat_target, markers = self._render_vlm_prompt(messages,images,+ videos = videos,tools = tools,enable_thinking = enable_thinking,reasoning_effort = reasoning_effort,@@ -6972,12 +7279,38 @@ class MLXInferenceBackend:continue_final_message = continue_final_message,)+ native = decodes_ids and bool(tools or tool_protocol_active or markers is not None)think_prefix = _think_prefix(prompt,getattr(chat_target, "all_special_tokens", None),messages,continue_final_message,+ preserves_think_close = native+ and decoder_preserves_token(self._tokenizer, "</think>"),)+ if decodes_ids:+ constraint = _build_grammar_constraint(+ response_format,+ chat_target,+ prompt,+ reasoning_markers = markers,+ tools = tools,+ reasoning_is_extracted = reasoning_is_extracted,+ reply_keeps_special_tokens = True,+ )+ if constraint is not None and not constraint.allows_reasoning:+ think_prefix = ""+ elif constraint is not None:+ from core.inference.chat_template_helpers import detect_think_prefill+ think_prefix = detect_think_prefill(prompt, preserves_think_close = True)+ if constraint is not None and _vlm_generation_is_diffusion(self._model):+ from core.inference.grammar_constraint import ResponseFormatError+ raise ResponseFormatError(_DIFFUSION_RESPONSE_FORMAT)+ if native and self._tokenizer:+ token_decoder = NativeToolTokenDecoder(+ self._tokenizer,+ preserved_tokens = reasoning_control_tokens(markers),+ )normalizer = (make_reasoning_normalizer(markers,@@ -6987,7 +7320,8 @@ class MLXInferenceBackend:bool(continue_final_message and trailing_assistant_text(messages)),),)- if markers is not None+ # The normalizer would rewrite marker text inside the document.+ if markers is not None and (constraint is None or constraint.allows_reasoning)else None)row = _RowStream(@@ -6995,12 +7329,8 @@ class MLXInferenceBackend:sequences = _mlx_stop_sequences(stop),normalizer = normalizer,)- max_new_tokens = self._generation_limit(- prompt,- max_new_tokens,- prompt_ids,- images = images,- cap = UNSET_GENERATION_BUDGET if images else None,+ max_new_tokens = self._media_generation_limit(+ prompt, max_new_tokens, prompt_ids, images, videos)vlm_kwargs = dict(max_tokens = max_new_tokens,@@ -7023,16 +7353,33 @@ class MLXInferenceBackend:presence_penalty = presence_penalty,frequency_penalty = frequency_penalty,logit_bias = logit_bias,+ grammar_constraint = constraint,penalty_window = penalty_window,penalty_prompt = prompt_ids,)if processors:# Not the repetition_penalty shortcut: mlx-vlm would build it outside our narrowingvlm_kwargs["logits_processors"] = _row_logits_processors(processors)+ streamed = None+ if constraint is not None:+ # mlx-vlm stops on the config's ids, which some repos set apart from the tokenizer's.+ constraint.stops_on(_mlx_stop_token_ids(self._tokenizer, self._model))+ if token_decoder is not None:+ constraint.decoded_dropping(token_decoder.dropped_ids())+ if token_decoder is not None:+ streamed = _MlxStreamedText(+ _mlx_stream_detokenizer(self._processor)+ or _mlx_stream_detokenizer(self._tokenizer),+ token_decoder,+ self._tokenizer,+ _mlx_stop_token_ids(self._tokenizer, self._model),+ )+ _require_row_detokenizer(streamed)return _VLMRowPlan(prompt = prompt,images = images,+ videos = videos,think_prefix = think_prefix,stream = row,kwargs = vlm_kwargs,@@ -7045,6 +7392,9 @@ class MLXInferenceBackend:},max_tokens = vlm_kwargs["max_tokens"],processors = processors,+ streamed = streamed,+ # As _generate_vlm, which drops the draft for tool turns.+ speculates = not (tools or tool_protocol_active),)def _generate_vlm(@@ -7071,7 +7421,7 @@ class MLXInferenceBackend:_adapter_state = None,tool_protocol_active = None,stop = None,- video = None,+ videos = None,response_format = None,reasoning_is_extracted = False,prompt_ids = None,@@ -7090,7 +7440,7 @@ class MLXInferenceBackend:prompt, chat_target, vlm_reasoning_markers = self._render_vlm_prompt(messages,images,- videos = [video] if video is not None else None,+ videos = videos or None,tools = tools,enable_thinking = enable_thinking,reasoning_effort = reasoning_effort,@@ -7116,29 +7466,14 @@ class MLXInferenceBackend:# Matched on the sampled text, for the reason _generate_text gives.sequences = _mlx_stop_sequences(stop)stopped = False- # Counted from a temp file (base64 is unreadable to the processor), discarded here.- counted_clip = (- _write_video_clip(video)- if video is not None and getattr(self, "_kv_context_budget", None)- else None+ max_new_tokens = self._media_generation_limit(+ prompt, max_new_tokens, prompt_ids, images, videos)- try:- max_new_tokens = self._generation_limit(- prompt,- max_new_tokens,- prompt_ids,- images = images,- videos = [counted_clip] if counted_clip is not None else None,- cap = UNSET_GENERATION_BUDGET if images or video is not None else None,- )- finally:- if counted_clip is not None:- _discard_video_clip(counted_clip)logger.info(- "VLM generating: prompt_len=%d, images=%d, has_video=%s",+ "VLM generating: prompt_len=%d, images=%d, videos=%d",len(prompt),len(images or ()),- video is not None,+ len(videos or ()),)# generate_step expects temperature=; temp= is silently ignored (stuck at greedy)vlm_kwargs = dict(@@ -7183,12 +7518,7 @@ class MLXInferenceBackend:prefill = detect_think_prefill(prompt, preserves_think_close = True)if constraint is not None and _vlm_generation_is_diffusion(self._model):from core.inference.grammar_constraint import ResponseFormatError- raise ResponseFormatError(- "response_format is not supported on this model: it generates by "- "diffusion rather than one token at a time, so no grammar can "- "constrain the next token. Load an autoregressive model to use "- "guided decoding."- )+ raise ResponseFormatError(_DIFFUSION_RESPONSE_FORMAT)recorded = token_logprobs = Noneif n_probs:if _vlm_generation_is_diffusion(self._model):@@ -7270,7 +7600,7 @@ class MLXInferenceBackend:import mlx.core as mxvlm_kwargs["input_ids"] = mx.array([self._encode_prompt(prompt)])session = self._vlm_prompt_cache_session(- _adapter_state, images, prompt, has_video = video is not None+ _adapter_state, images, prompt, has_video = bool(videos))vlm_kwargs.update(self._kv_runtime_quant_kwargs())if session is not None:@@ -7291,7 +7621,7 @@ class MLXInferenceBackend:draft is Noneor toolsor tool_protocol_active- or video is not None+ or videosor not all(getattr(processor, "history_only", False) for processor in processors)or _vlm_generation_is_diffusion(self._model)):@@ -7315,14 +7645,14 @@ class MLXInferenceBackend:ExitStack() as generation_scope,session_scope,):- if session is None and (images or video is not None):+ if session is None and (images or videos):# Free headroom for the vision pass under the lock; a video turn resumes no snapshot, so retained ones are pure occupancyself._release_vlm_snapshots()generation_scope.enter_context(self._generation_scopes("vision", *(() if draft is None else ("speculative",))))final_response = None- clip_path = None+ clip_paths = []if draft is not None:from unsloth_zoo.mlx.generate import SamplingParams@@ -7340,10 +7670,11 @@ class MLXInferenceBackend:)vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)try:- if video is not None:- clip_path = _write_video_clip(video)- vlm_kwargs["video"] = [clip_path]- frame_rate = _video_frame_rate(clip_path, self._processor)+ if videos:+ for clip in videos:+ clip_paths.append(_write_video_clip(clip))+ vlm_kwargs["video"] = clip_paths+ frame_rate = _video_sampling_fps(clip_paths, self._processor)if frame_rate is not None:vlm_kwargs["fps"] = frame_rate# Emit prefilled <think> first; inside the adapter so unsupported requests raise before any output.@@ -7431,7 +7762,7 @@ class MLXInferenceBackend:if sequences and not stopped and released < len(sampled):yield prefill + sampledfinally:- if clip_path is not None:+ for clip_path in clip_paths:_discard_video_clip(clip_path)cached_n = produced_n = 0produced_s = 0.0@@ -7506,6 +7837,9 @@ class MLXInferenceBackend:reason = _request_batch_gap(request)if reason is not None:return reason+ reason = self._video_batch_gap(requests)+ if reason is not None:+ return reasonreason = self._kv_policy_batch_reason()if reason is not None:return reason@@ -7526,6 +7860,9 @@ class MLXInferenceBackend:if self._model is None:return "no model is loaded"reason = _request_batch_gap(request)+ if reason is None and _carries_video(request):+ # Whether a clip's row fits the open batch is known only once its frames are decoded.+ reason = "a reply carries a video clip"if reason is not None:return reasonreason = self._kv_policy_batch_reason(resident = True)@@ -7535,6 +7872,22 @@ class MLXInferenceBackend:return self._vlm_resident_unavailable_reason(request)return self._text_batch_unavailable_reason()+ def _video_batch_gap(self, requests):+ if not any(map(_carries_video, requests)):+ return None+ # The one-at-a-time decode refuses these clips by name.+ if not self._is_vlm or not self._reads_vision_input() or not _mlx_vlm_decodes_video():+ return "a reply carries a video clip this load does not read"+ if self._speculative_draft is not None:+ return "a video clip decodes without the draft"+ try:+ from unsloth_zoo.mlx import generate as engine+ except ImportError:+ return "the installed unsloth-zoo has no MLX generation"+ if "videos" not in _engine_fields(engine, "GenerationRequest"):+ return "the installed unsloth-zoo cannot batch a video clip"+ return None+def _text_batch_unavailable_reason(self):"""Why the installed mlx-lm cannot batch this load's replies, or None."""try:@@ -7544,7 +7897,7 @@ class MLXInferenceBackend:return stream_unavailable_reason(self._model, self._tokenizer)def _vlm_resident_unavailable_reason(self, request):- reason = self._vlm_batch_unavailable_reason([request])+ reason = self._vlm_batch_unavailable_reason([request], resident = True)if reason is not None:return reasontry:@@ -7579,7 +7932,12 @@ class MLXInferenceBackend:or self.resident_unavailable_reason({}) is None)- def _vlm_batch_unavailable_reason(self, requests):+ def _vlm_batch_unavailable_reason(+ self,+ requests,+ *,+ resident = False,+ ):try:from unsloth_zoo.mlx import generate as engineexcept ImportError:@@ -7590,6 +7948,14 @@ class MLXInferenceBackend:gap = _row_processor_gap(engine, requests)if gap is not None:return gap+ # Only the resident session decodes rows from their ids, as the one-at-a-time decode does.+ decodes_ids = resident and _vision_batch_reports_ids(engine, self._speculative_draft)+ if not decodes_ids and any(+ request.get("tools") or request.get("tool_protocol_active") for request in requests+ ):+ return "a reply runs the tool protocol"+ if not decodes_ids and any(_constrained(request) for request in requests):+ return "a reply is constrained to a response format"if any(request.get("prompt_ids") is not None for request in requests):# The batch detokenizer keeps control tokens and trims the first leading space.return "a raw prompt on a vision model decodes on its own"@@ -7629,6 +7995,9 @@ class MLXInferenceBackend:):def report(events):for row, snapshot in events:+ if isinstance(snapshot, BaseException):+ # Every row answers one command, which fails as a single reply would.+ raise snapshotif snapshot is None:self.last_batch_generation_stats[row] = session.take_stats(row)yield row, snapshot@@ -7686,16 +8055,6 @@ class MLXInferenceBackend:)plans = [self._plan_vlm_request(request) for request in requests]- batch = [- GenerationRequest(- **self._vlm_batch_prompt(plan.prompt),- image = plan.images[0] if plan.images else None,- max_tokens = plan.max_tokens,- sampling = SamplingParams(**plan.sampling),- logits_processors = plan.processors,- )- for plan in plans- ]defaults = GenerationDefaults(max_tokens = max(plan.max_tokens for plan in plans),prefill_batch_size = len(plans),@@ -7707,13 +8066,26 @@ class MLXInferenceBackend:self._generation_lock,_temporary_mlx_adapter_state(self._model, _adapter_state),self._int8_prefill_scope(),+ ExitStack() as clips,):- if any(plan.images for plan in plans):+ batch = [+ GenerationRequest(+ **self._vlm_batch_prompt(plan.prompt),+ image = plan.images[0] if plan.images else None,+ **row_clips,+ max_tokens = plan.max_tokens,+ sampling = SamplingParams(**plan.sampling),+ logits_processors = plan.processors,+ )+ for plan, row_clips in zip(plans, _batch_clips(clips, plans, self._processor))+ ]+ if any(plan.images or plan.videos for plan in plans):self._release_vlm_snapshots()logger.info(- "Generating %d vision replies as one batch: images=%d, max_tokens=%s, model=%s",+ "Generating %d vision replies as one batch: images=%d, videos=%d, max_tokens=%s, model=%s",len(plans),sum(1 for plan in plans if plan.images),+ sum(len(plan.videos or ()) for plan in plans),[plan.max_tokens for plan in plans],type(self._model).__name__,)diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 65e602f5f44..964abfd0c15 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -1391,7 +1391,7 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,rows: Optional[list] = None,- video_b64: Optional[str] = None,+ videos_b64: Optional[list[str]] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,diffusion_frames: bool = False,@@ -1427,8 +1427,8 @@ class InferenceOrchestrator:if response_format is not None:cmd["response_format"] = response_formatcmd["reasoning_is_extracted"] = bool(reasoning_is_extracted)- if video_b64:- cmd["video_base64"] = video_b64+ if videos_b64:+ cmd["videos_base64"] = videos_b64if use_adapter is not None:cmd["use_adapter"] = use_adapterif tools is not None:@@ -1699,7 +1699,7 @@ class InferenceOrchestrator:stop: Optional[list] = None,rows: Optional[list] = None,expected_model: Optional[str] = None,- video: Optional[str] = None,+ videos: Optional[list[str]] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,diffusion_frames: bool = False,@@ -1757,7 +1757,7 @@ class InferenceOrchestrator:tool_protocol_active = tool_protocol_active,seed = seed,rows = rows,- video_b64 = video,+ videos_b64 = videos,diffusion_frames = diffusion_frames,prompt_ids = prompt_ids,penalty_window = penalty_window,@@ -3197,7 +3197,7 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,- video: Optional[str] = None,+ videos: Optional[list[str]] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,diffusion_frames: bool = False,@@ -3231,7 +3231,7 @@ class InferenceOrchestrator:stop = stop,response_format = response_format,reasoning_is_extracted = reasoning_is_extracted,- video = video,+ videos = videos,diffusion_frames = diffusion_frames,)@@ -3262,7 +3262,7 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,- video: Optional[str] = None,+ videos: Optional[list[str]] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,) -> Generator[Any, None, None]:@@ -3294,7 +3294,7 @@ class InferenceOrchestrator:logit_bias = logit_bias,stop = stop,rows = rows,- video = video,+ videos = videos,response_format = response_format,reasoning_is_extracted = reasoning_is_extracted,)@@ -3598,7 +3598,7 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,rows: Optional[list] = None,- video: Optional[str] = None,+ videos: Optional[list[str]] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,diffusion_frames: bool = False,@@ -3667,7 +3667,7 @@ class InferenceOrchestrator:stop = stop,rows = rows,expected_model = expected_model,- video = video,+ videos = videos,response_format = response_format,reasoning_is_extracted = reasoning_is_extracted,diffusion_frames = diffusion_frames,@@ -3715,7 +3715,7 @@ class InferenceOrchestrator:tool_protocol_active = tool_protocol_active,seed = seed,rows = rows,- video_b64 = video,+ videos_b64 = videos,diffusion_frames = diffusion_frames,prompt_ids = prompt_ids,penalty_window = penalty_window,diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex ea05f4fd72c..f15e63cbba6 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1154,10 +1154,10 @@ def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:):if gated in cmd and _backend_declares(backend, gated):gen_kwargs[gated] = cmd[gated]- if cmd.get("video_base64"):- if not _backend_declares(backend, "video"):+ if cmd.get("videos_base64"):+ if not _backend_declares(backend, "videos"):raise RuntimeError("The loaded backend does not read video.")- gen_kwargs["video"] = cmd["video_base64"]+ gen_kwargs["videos"] = cmd["videos_base64"]# Only a raw completion carries these; its batch rows are planned from them.for raw in ("prompt_ids", "penalty_window", "grammar", "n_probs"):if raw in cmd:@@ -1615,13 +1615,31 @@ class _ResidentBatch:returntry:for handle, snapshot in self.session.step(waiting):- self._report(handle, snapshot)+ if not isinstance(snapshot, BaseException):+ self._report(handle, snapshot)+ elif not self._fail_row(handle[0], snapshot):+ returnexcept Exception as exc:logger.error("Batched generation error: %s", exc, exc_info = True)self._fail_all(exc)returnself._close_if_empty()+ def _fail_row(self, request_id: str, exc: BaseException) -> bool:+ """Answer a request whose own decode failed, the batch decoding on. False where it gave up the batch."""+ if request_id not in self._owed:+ return True+ logger.error("A batched reply failed: %s", exc, exc_info = exc)+ stack = "".join(traceback.format_exception(exc, limit = 20))+ taken_back = self._forget(request_id, withdraw = True)+ # _forget already dropped it from _owed, so _fail_all would not answer it.+ _send_response(+ self.resp_queue, {**_generation_error_payload(request_id, exc), "stack": stack}+ )+ if not taken_back:+ self._fail_all(exc)+ return taken_back+def _fail_all(self, exc: BaseException) -> None:"""Give up on the batch: every request in it is owed an error, not silence."""stack = traceback.format_exc(limit = 20)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 2c857832e35..d05de4044a1 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -26686,36 +26686,31 @@ _VIDEO_INPUT_REFUSAL = ()-def _local_video_clip(payload, model_info) -> str:- """Serves a video_url part the same as the legacy field; refuses a model with no video input."""+def _local_video_clips(payload, model_info) -> list[str]:+ """Serves video_url parts the same as the legacy field, in request order; refuses a model with+ no video input."""if not model_info.get("has_video_input"):raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)clips = _request_video_clips(payload)if not clips:raise HTTPException(status_code = 400, detail = "Could not read the provided video file.")- # Generation takes one video kwarg; extra clips would be silently dropped.- if len(clips) > 1:- raise HTTPException(- status_code = 400,- detail = "Only one video is supported per request on this model.",- )# MLX would move an older-turn clip onto the newest turn.if _video_is_on_an_older_turn(payload):raise HTTPException(status_code = 400,detail = "A video must be attached to the latest message on this model.",)- # Refused everywhere, so the message must not send the caller to GGUF for it.- if _is_remote_video(clips[0]):- raise HTTPException(status_code = _REMOTE_VIDEO_REFUSAL[0], detail = _REMOTE_VIDEO_REFUSAL[1])- # _request_video_rejection may not have run; reject unsupported schemes here too.- scheme_rejection = _video_scheme_rejection(clips[0])- if scheme_rejection is not None:- raise HTTPException(status_code = scheme_rejection[0], detail = scheme_rejection[1])- video_b64, rejection = _video_b64_rejection(clips[0])+ # The pre-switch validation may not have run; count, scheme and size are refused here too.+ rejection = _request_video_rejection(payload)if rejection is not None:raise HTTPException(status_code = rejection[0], detail = rejection[1])- return video_b64+ served = []+ for clip in clips:+ video_b64, rejection = _video_b64_rejection(clip)+ if rejection is not None:+ raise HTTPException(status_code = rejection[0], detail = rejection[1])+ served.append(video_b64)+ return serveddef _video_payload_chars(clip: str) -> int:@@ -30666,9 +30661,9 @@ async def produce_openai_chat_completions(model_name = _orchestrator_public_model_id(backend) or payload.modelmodel_info = backend.models.get(backend.active_model_name, {})# Before the speech and audio-input dispatches, which return before the clip is attached.- _video_clip = None+ _video_clips: list[str] = []if _request_has_video(payload):- _video_clip = _local_video_clip(payload, model_info)+ _video_clips = _local_video_clips(payload, model_info)# Settled here: a model without audio input never enters the audio-input path.if payload.audio_base64:raise HTTPException(status_code = 400, detail = _AUDIO_VIDEO_INPUT_DETAIL)@@ -33464,7 +33459,7 @@ async def produce_openai_chat_completions(_sf_image_tpl = ((_sf_model_info.get("chat_template_info") or {}).get("processor_template")# A clip renders through the processor as an image does.- if _sf_has_any_image or _video_clip is not None+ if _sf_has_any_image or _video_clipselse None)# Differs from processor_template: a template-less processor still places the image.@@ -33531,7 +33526,7 @@ async def produce_openai_chat_completions(and (not _sf_has_any_image or (bool(_sf_model_info.get("is_vision")) and not payload.tools))# A caller's several images and a video clip stay out of the loop, as on main.and not images- and _video_clip is None+ and not _video_clipsand not _sf_is_gptossand _sf_tool_budget > 0)@@ -33616,7 +33611,7 @@ async def produce_openai_chat_completions(request,include_api_key = True,template_default = False,- image = _sf_has_any_image or _video_clip is not None,+ image = _sf_has_any_image or bool(_video_clips),tools = True,controls = _date_controls,)@@ -34135,8 +34130,8 @@ async def produce_openai_chat_completions(stop = normalized_stop,response_format = _extract_response_format(payload),)- if _video_clip is not None:- gen_kwargs["video"] = _video_clip+ if _video_clips:+ gen_kwargs["videos"] = _video_clips# The worker/template wrapper peels off kwargs the template doesn't accept.if payload.enable_thinking is not None:gen_kwargs["enable_thinking"] = payload.enable_thinking@@ -34170,7 +34165,7 @@ async def produce_openai_chat_completions(# Gate on _sf_use_tools, not mcp_enabled: an empty MCP registry must not drop client tools._sf_client_tools = (# Read the resolved value: recomputing would undo the launcher-default withdrawal above.- (not _sf_tools_on or ((_sf_has_any_image or _video_clip is not None) and not _sf_use_tools))+ (not _sf_tools_on or ((_sf_has_any_image or bool(_video_clips)) and not _sf_use_tools))and not _sf_use_toolsand not _sf_is_gptossand _sf_supports_tools@@ -34248,7 +34243,7 @@ async def produce_openai_chat_completions(_sf_client_system_prompt = _apply_current_date_prompt(_user_system_prompt,request,- image = _sf_has_any_image or _video_clip is not None,+ image = _sf_has_any_image or bool(_video_clips),tools = bool(_sf_client_catalog),controls = _date_controls,)@@ -34381,7 +34376,7 @@ async def produce_openai_chat_completions(fitted_kwargs, events = gen_kwargs, ()# The count cannot price pictures or video, so those prompts are left alone.if _sf_fit_overflow and all(- gen_kwargs.get(key) is None for key in ("image", "images", "video")+ gen_kwargs.get(key) is None for key in ("image", "images", "videos")):fitted = backend.compact_chat_context(gen_kwargs.get("messages") or [],@@ -34828,14 +34823,14 @@ async def produce_openai_chat_completions(# or the reverse scan attaches the picture to it (#10092).# The fitted prompt: the retry extends it and is not refitted._nudge_base = _sf_fit_request()[0]["messages"]- if _sf_renders_image or _video_clip is not None:+ if _sf_renders_image or _video_clips:from core.inference.chat_template_helpers import (messages_with_attached_image as _nudge_attach,)_nudge_base = _nudge_attach(_nudge_base,image = _sf_renders_image,- video = _video_clip is not None,+ video = len(_video_clips),)retry_messages = [*_nudge_base,diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 7d324651b06..7a4f9924d6b 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -2696,6 +2696,166 @@ def test_mlx_reasoning_tool_turn_keeps_provenance_across_buffered_segments(monkeassert [call["function"]["name"] for call in calls] == ["terminal"]+def _decode_alone_and_batched(+ monkeypatch,+ turn,+ markers,+ cancel_after = None,+ prompt = "prompt",+ **request,+):+ """*turn* answered by the single path and by a text batch row: every snapshot each emits."""+ import threading++ _install_fake_mlx(monkeypatch)+ from core.inference import mlx_inference++ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda *_args, **_kwargs: prompt,+ )+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.render_with_native_template_fallback",+ lambda formatted_prompt, **_kwargs: SimpleNamespace(+ prompt = formatted_prompt, reasoning_channel_markers = markers+ ),+ )+ rows, last = {}, "stop" if turn._ends == "stop" else "length"++ def step():+ events = [(uid, ids.pop(0)) for uid, ids in rows.items() if ids]+ return [], [+ SimpleNamespace(uid = uid, token = token, finish_reason = None if rows[uid] else last)+ for uid, token in events+ ]++ generator = SimpleNamespace(+ insert = lambda *_a, **_k: [rows.setdefault(len(rows), list(turn.ids)) and len(rows) - 1],+ next = step,+ remove = lambda uids, **_k: [rows[uid].clear() for uid in uids] and {},+ close = lambda: None,+ )+ for name, attrs in (+ ("mlx_lm", {"stream_generate": lambda *_a, **_k: turn.stream()}),+ (+ "mlx_lm.sample_utils",+ {"make_sampler": lambda **_k: None, "make_logits_processors": None},+ ),+ ("mlx_lm.generate", {"BatchGenerator": lambda *_a, **_k: generator}),+ ):+ module = types.ModuleType(name)+ module.__dict__.update(attrs)+ monkeypatch.setitem(sys.modules, name, module)++ backend = mlx_inference.MLXInferenceBackend()+ backend._model, backend._tokenizer, backend._is_vlm = object(), turn, False+ request = {"messages": [{"role": "user", "content": "run it"}], "max_new_tokens": 99, **request}+ cancel, alone = threading.Event(), []+ for snapshot in backend.generate_chat_response(**request, cancel_event = cancel):+ alone.append(snapshot)+ if len(alone) == cancel_after:+ cancel.set()++ backend._batch_prompt_row = lambda _prompt, _state: ([], [0], None, None, [0])+ session = mlx_inference._TextBatchSession(backend, width = 2)+ # Two rows decoding together, so no state a row keeps can be shared or misrouted.+ batched = {row: [prefix] if (prefix := session.admit(request, row)) else [] for row in (0, 1)}+ while session.rows_in_flight:+ cut = cancel_after is not None and len(batched[0]) >= cancel_after+ events = session.withdraw([0, 1]) if cut else session.step()+ for row, snapshot in events:+ if snapshot is not None:+ batched[row].append(snapshot)+ session.close()+ return alone, batched+++_TOOL = [{"type": "function", "function": {"name": "terminal"}}]+_CHANNELS = ("<|channel>thought", "<channel|>")+_GEMMA_SPECIALS = ("<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>", "<pad>", "<eos>")+_GEMMA_TOOL_TURN = (+ *("<|channel>", "thought", "\n", "weighing", "<pad>", "▁it", "<channel|>"),+ *("<|tool_call>", "call:terminal{command:id}", "<tool_call|>", "<eos>"),+)+++ "markers, request_, last, ends, cancel_after",+ [+ (_CHANNELS, {"tools": _TOOL}, "<eos>", "stop", None),+ (None, {"tools": _TOOL}, "<eos>", "stop", None),+ (None, {"tools": [], "tool_protocol_active": True}, "<eos>", "stop", None),+ # Stopping on the call's closer keeps it; stopping on an opener closes nothing.+ (None, {"tools": _TOOL}, "<tool_call|>", "stop", None),+ (None, {"tools": _TOOL}, "<|tool_call>", "stop", None),+ (_CHANNELS, {"tools": _TOOL}, "call:terminal{command:id}", "length", None),+ (_CHANNELS, {"tools": _TOOL, "stop": ["{command"]}, "<eos>", "stop", None),+ # Text held to the end, its tail a stop sequence's first characters.+ (_CHANNELS, {"tools": _TOOL, "stop": ["<tool_call|>!"]}, "<eos>", "stop", None),+ (_CHANNELS, {"tools": _TOOL}, "<eos>", "stop", 3),+ (_CHANNELS, {}, "<eos>", "stop", None),+ ],+)+def test_a_batched_tool_row_decodes_what_the_reply_alone_does(+ monkeypatch, markers, request_, last, ends, cancel_after+):+ pieces = _GEMMA_TOOL_TURN[: _GEMMA_TOOL_TURN.index(last) + 1]+ specials = tuple(special for special in _GEMMA_SPECIALS if special in pieces)+ eos = last if ends == "stop" else None+ # Built before the fake mlx modules land, so the real detokenizer still imports.+ turn = _SpmTurn(pieces, specials = specials, eos = eos, chat_template = "x", ends = ends)+ alone, batched = _decode_alone_and_batched(monkeypatch, turn, markers, cancel_after, **request_)+ assert alone and batched == {0: alone, 1: alone}+++ "markers, request_, allows_reasoning",+ [+ (_CHANNELS, {"tools": _TOOL}, False),+ (_CHANNELS, {"tools": _TOOL, "reasoning_is_extracted": True}, True),+ (None, {"tools": _TOOL}, True),+ (None, {}, False),+ ],+)+def test_a_batched_constrained_row_is_planned_as_the_reply_alone_is(+ monkeypatch, markers, request_, allows_reasoning+):+ from core.inference import mlx_inference++ turn = _SpmTurn(+ _GEMMA_TOOL_TURN, specials = _GEMMA_SPECIALS, eos = "<eos>", chat_template = "x", ends = "stop"+ )+ seen, decodes, sampling = [], [], mlx_inference._mlx_sampling_processors+ decode = turn.decode+ monkeypatch.setattr(turn, "decode", lambda ids, **kw: decodes.append(kw) or decode(ids, **kw))+ # A kept </think> is what tells the grammar a closer survives the decode.+ monkeypatch.setattr(mlx_inference, "decoder_preserves_token", lambda *_a: True)+ monkeypatch.setattr(+ mlx_inference,+ "_build_grammar_constraint",+ lambda response_format, *_a, **kw: response_format+ and SimpleNamespace(+ allows_reasoning = allows_reasoning,+ decoded_dropping = lambda ids: seen.append((_a, sorted(kw.items()), ids)),+ ),+ )+ monkeypatch.setattr(+ mlx_inference,+ "_mlx_sampling_processors",+ lambda **kw: seen.append(kw["grammar_constraint"] is not None) or sampling(**kw),+ )+ request_ = {**request_, "response_format": {"type": "json_object"}}+ # The template opens <think>, which a document-only reply must not re-emit.+ alone, batched = _decode_alone_and_batched(+ monkeypatch, turn, markers, prompt = "P<think>", **request_+ )+ assert alone and batched == {0: alone, 1: alone}+ third = len(seen) // 3+ assert seen and seen == seen[:third] * 3 and True in seen+ plain = [kw for kw in decodes if kw.get("skip_special_tokens")]+ assert all(kw["clean_up_tokenization_spaces"] is False for kw in plain)++def test_mlx_text_normalizes_native_reasoning_and_close_releases_lock(monkeypatch):from core.inference.mlx_inference import MLXInferenceBackend@@ -3293,7 +3453,7 @@ def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypatbackend._make_row_cache = lambda: ["cache"]backend._batch_prompt_row = lambda prompt, _state: ([], [9], ["text cache"], "key", [9])plan = SimpleNamespace(- prompt = "P", max_tokens = 4, sampler = None, processors = (), detokenizes = False, think_prefix = ""+ prompt = "P", max_tokens = 4, sampler = None, processors = (), streamed = None, think_prefix = "")backend._plan_text_row = lambda messages, **_kw: rendered.append(messages) or planwith monkeypatch.context() as patch:@@ -3313,6 +3473,80 @@ def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypatsession.close()[email protected]("ends", [None, "length", "stop"])+def test_a_row_whose_own_processor_fails_leaves_and_its_neighbour_decodes_on(monkeypatch, ends):+ import numpy++ from core.inference import mlx_inference++ rows, removed, steps = {}, [], []++ def insert(_prompts, logits_processors, **_kwargs):+ rows[len(rows)] = logits_processors[0]+ return [len(rows) - 1]++ def step():+ steps.append(None)+ for processors in rows.values():+ for processor in processors:+ processor(numpy.array([1]), "logits")+ # The healthy row runs out of budget every third step, so a swallowed failure ends too.+ healthy = "length" if len(steps) % 3 == 0 else None+ return [], [+ SimpleNamespace(uid = uid, token = 5, finish_reason = healthy if uid else ends)+ for uid in rows+ ]++ generator = SimpleNamespace(+ insert = insert,+ next = step,+ remove = lambda uids, **_k: [removed.append(uid) or rows.pop(uid) for uid in uids] and {},+ close = lambda: None,+ )+ generate = types.ModuleType("mlx_lm.generate")+ generate.BatchGenerator = lambda *_a, **_k: generator+ monkeypatch.setitem(sys.modules, "mlx_lm.generate", generate)+ backend = mlx_inference.MLXInferenceBackend()+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])+ backend._is_vlm = False+ backend._batch_prompt_row = lambda _prompt, _state: ([], [9], None, None, [9])++ def failing(_tokens, _logits):+ raise ValueError("schema")++ plans = iter([failing, None])+ backend._plan_text_row = lambda _messages, **_kw: SimpleNamespace(+ prompt = "P",+ think_prefix = "",+ stream = mlx_inference._RowStream(decode = lambda ids: "x" * len(ids)),+ streamed = None,+ sampler = None,+ processors = [processor] if (processor := next(plans)) else [],+ max_tokens = 8,+ )+ session = mlx_inference._TextBatchSession(backend, width = 2)+ session.admit({"messages": []}, "bad")+ session.admit({"messages": []}, "good")+ (handle, failure), *events = session.step()+ assert handle == "bad" and isinstance(failure, ValueError) and str(failure) == "schema"+ # Its token was sampled unconstrained, and the single path raises before yielding it.+ assert removed == [0] and session.handles == ["good"]+ assert events == [("good", "x")]+ assert list(session.step()) == [("good", "xx")]+ session.close()+ # A drafted batch drafts only for rows whose processors all read just the history.+ penalty = mlx_inference._mlx_sampling_processors(presence_penalty = 1.5)+ guarded = mlx_inference._failing_alone(None, [*penalty, lambda _tokens, logits: logits])+ assert [p.history_only for p in guarded] == [True, False]+ # The fixed multi-reply command fails as a single reply would.+ plans = iter([failing, None])+ backend.last_batch_generation_stats = [None, None]+ with pytest.raises(ValueError, match = "schema"):+ list(+ backend._generate_session_batch([{"messages": []}] * 2, mlx_inference._TextBatchSession)+ )++def test_mlx_text_without_lru_prompt_cache_prefills_the_full_prompt(monkeypatch):from core.inference import mlx_inference@@ -3620,14 +3854,16 @@ def test_mlx_batch_rows_plan_a_raw_completion_prompt_untemplated(monkeypatch):monkeypatch.setattr(backend, "_render_text_prompt", lambda *_a, **_k: pytest.fail("templated"))seen = {}monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **kw: seen.update(kw))+ monkeypatch.setattr(mlx_inference, "_raw_prompt_constraint", _RawConstraint)row = mlx_inference._row_kwargs({"prompt_ids": [1, 2, 3], "penalty_window": 2})- plan = backend._plan_text_row([], **row)+ plan = backend._plan_text_row([], grammar = "G", **row)assert (plan.prompt, plan.think_prefix, seen["penalty_prompt"], seen["penalty_window"]) == ([1, 2, 3],"",[1, 2, 3],2,)+ assert seen["grammar_constraint"].built == ("G", backend._tokenizer)backend._make_row_cache = lambda: Noneassert backend._batch_prompt_row(plan.prompt, None)[1] == [1, 2, 3]kwargs = []@@ -3641,7 +3877,6 @@ def test_mlx_batch_rows_plan_a_raw_completion_prompt_untemplated(monkeypatch):vision = backend._plan_vlm_request({"prompt_ids": [4, 5], "messages": []})assert (vision.prompt, vision.max_tokens) == ([4, 5], ([4, 5], [4, 5]))assert backend._vlm_batch_prompt([4, 5]) == {"prompt_token_ids": [4, 5]}- assert mlx_inference._request_batch_gap({"grammar": "G"})_COMPLETION_TOK = SimpleNamespace(@@ -3856,13 +4091,14 @@ def test_mlx_batch_releases_an_over_budget_cache_while_another_row_decodes(monkeuid = handle,cache_key = handle,all_tokens = [],- plan = SimpleNamespace(stream = stream),+ plan = SimpleNamespace(stream = stream, streamed = None),ready_at = None,processing_at = None,prompt_len = 1,generated = 1,cached = 0,reason = "stop",+ failure = None,)for handle in banked}@@ -7010,7 +7246,7 @@ def _drive_vlm_generation(image = None,max_new_tokens = 1,session = None,- video = None,+ videos = None,on_stream = None,**request,):@@ -7048,7 +7284,7 @@ def _drive_vlm_generation(backend.generate_chat_response(messages = [{"role": "user", "content": "hi"}],image = image,- video = video,+ videos = videos,max_new_tokens = max_new_tokens,**request,)@@ -7119,7 +7355,7 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):assert "draft_n" not in backend.last_generation_stats["timings"]# Only the drafted reply entered the verify scope: open from preparing the draft through decoding.assert len(prepared) == 1 and scopes == [["open"]] * 2 and verify_scope == ["open", "closed"]- opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r: None+ opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r, **_k: Nonebackend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)assert backend.batch_unavailable_reason([{}, {}]) == "no stream"monkeypatch.setattr(@@ -7221,7 +7457,7 @@ def test_the_context_budget_counts_the_request_in_tokens_the_processor_would_prodef test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monkeypatch):- """A video is counted from a readable file, not the route's base64."""+ """Every clip is counted from a readable file, not the route's base64."""import base64import os@@ -7235,15 +7471,17 @@ def test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monkedef _limit(*_a, **kwargs):seen.update(kwargs)seen["on_disk"] = [os.path.exists(clip) for clip in kwargs.get("videos") or ()]+ seen["contents"] = [Path(clip).read_bytes() for clip in kwargs.get("videos") or ()]return 1monkeypatch.setattr(backend, "_generation_limit", _limit)- clip = base64.b64encode(b"not a readable clip").decode()+ clips = [base64.b64encode(b"not a readable clip").decode(), "BBBB"]with contextlib.suppress(Exception):- _drive_vlm_generation(backend, monkeypatch, video = clip)- assert seen["videos"] != [clip]- assert seen["on_disk"] == [True]- assert not os.path.exists(seen["videos"][0])+ _drive_vlm_generation(backend, monkeypatch, videos = clips)+ assert seen["videos"] != clips+ assert seen["on_disk"] == [True, True]+ assert seen["contents"] == [base64.b64decode(clip) for clip in clips]+ assert not any(os.path.exists(path) for path in seen["videos"])def test_media_is_prepared_only_under_a_budget_and_only_once(monkeypatch):@@ -8099,7 +8337,7 @@ def _open_vision_session(rows):session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)session._rows = {row.handle: row for row in rows}session._by_row, session._settled = {row.row: row for row in rows}, {}- session._speculative = None+ session._speculative, session._decodes_ids = None, Falsereturn session@@ -8115,7 +8353,7 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto[mlx_inference._VisionBatchRow(handle = handle,- plan = SimpleNamespace(stream = _vlm_row(backend, stop = ["HALT"])),+ plan = SimpleNamespace(stream = _vlm_row(backend, stop = ["HALT"]), streamed = None),row = number,)for number, handle in enumerate(handles)@@ -8148,6 +8386,32 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto), "a row the engine has already retired is not handed back again"+def test_a_resident_vision_tool_row_decodes_its_ids_and_keeps_the_stop_that_closes_its_call(+ monkeypatch,+):+ from core.inference import mlx_inference++ pieces = ("<|tool_call>", "call:terminal{command:id}", "<tool_call|>")+ turn = _SpmTurn(+ pieces, specials = pieces[::2], eos = "<tool_call|>", chat_template = "x", ends = "stop"+ )+ backend = _vlm_backend(monkeypatch, markers = None)+ backend._tokenizer = backend._processor.tokenizer = turn+ plan = backend._plan_vlm_row(+ [{"role": "user", "content": "hi"}],+ None,+ **dict(temperature = 0, top_p = 1, top_k = 0, min_p = 0, max_new_tokens = 9),+ repetition_penalty = 1.0,+ tools = _TOOL,+ decodes_ids = True,+ )+ session = _open_vision_session([mlx_inference._VisionBatchRow(handle = "r", plan = plan, row = 0)])+ ended = SimpleNamespace(finish_reason = "stop", prompt_token_count = 3, token_ids = turn.ids[:2])+ event = SimpleNamespace(index = 0, delta = "", token_ids = tuple(turn.ids), result = ended)+ session.stream = SimpleNamespace(step = lambda: [event])+ assert list(session.step())[-2:] == [("r", "".join(pieces)), ("r", None)]++def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatch):engine = _batch_engine(monkeypatch)monkeypatch.setattr(engine, "GenerationRequest", SimpleNamespace, raising = False)@@ -8156,14 +8420,16 @@ def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatmonkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: True, raising = False)backend = _vlm_backend(monkeypatch, markers = None)backend._tokenizer.bos_token = "<s>"- backend._plan_vlm_request = lambda request: SimpleNamespace(+ backend._plan_vlm_request = lambda request, **_k: SimpleNamespace(prompt = "<s>hi",images = None,+ videos = None,max_tokens = 4,sampling = {},processors = None,think_prefix = "",stream = None,+ streamed = None,)session = _open_vision_session([])session.backend, session._adapter_state, session._resumes_rows = backend, None, False@@ -8226,8 +8492,10 @@ def test_a_vision_row_times_its_prefill_wherever_the_batch_runs_it(monkeypatch,processors = None,think_prefix = "",stream = _vlm_row(backend),+ streamed = None,+ speculates = True,)- backend._plan_vlm_request = lambda request: plan+ backend._plan_vlm_request = lambda request, **_k: planbackend._reads_vision = Falsesession = _open_vision_session([])session.backend, session._adapter_state, session._resumes_rows = backend, None, False@@ -8276,6 +8544,48 @@ def test_a_vision_row_times_its_prefill_wherever_the_batch_runs_it(monkeypatch,assert timings["predicted_per_second"] == pytest.approx(10.0)+def test_a_fixed_vision_batch_marks_each_row_s_clips_and_shares_their_decode_budget(monkeypatch):+ import base64++ from core.inference import mlx_inference++ engine, backend = _batch_engine(monkeypatch), _vlm_backend(monkeypatch, markers = None)+ for name in ("GenerationRequest", "GenerationDefaults"):+ monkeypatch.setattr(engine, name, SimpleNamespace, raising = False)+ monkeypatch.setattr(engine, "SamplingParams", dict, raising = False)+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: False, raising = False)+ too_large, refuse = (_ for _ in ()).throw, mlx_inference._ClipsTooLarge+ fps = lambda p, proc, rows = 1: rows / 4 if rows < 3 else too_large(refuse())+ monkeypatch.setattr(mlx_inference, "_video_sampling_fps", fps)+ seen, paths = [], []+ clips, other = ([base64.b64encode(data).decode()] for data in (b"one", b"two"))+ backend._render_vlm_prompt = lambda messages, images, videos, **k: (+ seen.append(([p["type"] for p in messages[-1]["content"]], videos)) or ("p", None, None)+ )+ backend._media_generation_limit = lambda p, n, ids, images, videos: seen.append(videos) or n+ backend._kv_window_generate_kwargs, backend._generation_lock = dict, threading.Lock()+ backend._release_vlm_snapshots = lambda: seen.append("freed")++ def stream_batch(model, processor, batch, defaults):+ seen.extend((row.fps, [Path(path).read_bytes() for path in row.videos]) for row in batch)+ paths.extend(path for row in batch for path in row.videos)+ return iter(())++ monkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)+ request = {"messages": [{"role": "user", "content": "hi"}], "videos": clips}+ with pytest.raises(RuntimeError, match = "unfinished"):+ list(backend._generate_vlm_batch([request, request, {**request, "videos": other}]))+ planned = [(["video", "text"], clips), clips] * 2 + [(["video", "text"], other), other]+ assert seen[:7] == planned + ["freed"]+ rates = [(0.5, [b"one"])] * 2 + [(0.25, [b"two"])]+ assert seen[7:] == rates and paths[0] == paths[1] != paths[2], "a group shares files and budget"+ seen.clear()+ with pytest.raises(RuntimeError, match = "unfinished"):+ list(backend._generate_vlm_batch([request] * 3))+ assert seen[-3:] == [(0.25, [b"one"])] * 3 and len(set(paths[3:])) == 3, "too large to share"+ assert not any(map(os.path.exists, paths)), "removed when the batch ends"++def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(monkeypatch):from core.inference import mlx_inference@@ -8294,7 +8604,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mobackend._vlm_is_diffusion_model = lambda model: Falsebackend._vlm_snapshot_scope = lambda state, images: (None, "m|img" if images else "m", list)- backend._plan_vlm_request = lambda request: SimpleNamespace(+ backend._plan_vlm_request = lambda request, **_k: SimpleNamespace(prompt = "p",images = request.get("images"),max_tokens = 4,@@ -8302,6 +8612,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(moprocessors = None,think_prefix = "",stream = _vlm_row(backend),+ streamed = None,)session = _open_vision_session([])session.backend, session._adapter_state = backend, None@@ -8380,15 +8691,12 @@ def test_a_text_load_whose_engine_cannot_batch_says_so_before_it_commits(monkeyp({"images": [object()]},{"images_base64": ["AAAA"]},- {"video": "AAAAGGZ0eXBtcDQy"},- {"video_base64": "AAAAGGZ0eXBtcDQy"},- {"tools": [{"type": "function", "function": {"name": "lookup"}}]},- {"tool_protocol_active": True},- {"response_format": {"type": "json_object"}},+ {"videos": ["AAAAGGZ0eXBtcDQy"]},+ {"videos_base64": ["AAAAGGZ0eXBtcDQy"]},),)def test_a_reply_the_batch_cannot_serve_decodes_alone(monkeypatch, extra):- """Replayed tool images, a clip and the native tool protocol live only on the"""+ """Replayed tool images and a clip live only on the single path."""from core.inference.mlx_inference import MLXInferenceBackendengine = _batch_engine(monkeypatch)@@ -8404,6 +8712,55 @@ def test_a_reply_the_batch_cannot_serve_decodes_alone(monkeypatch, extra):assert backend.batch_unavailable_reason([plain, {**plain, **extra}]) is not None+ "extra, refusal",+ (+ ({"tools": [{"type": "function"}]}, "a reply runs the tool protocol"),+ ({"tool_protocol_active": True}, "a reply runs the tool protocol"),+ (+ {"response_format": {"type": "json_object"}},+ "a reply is constrained to a response format",+ ),+ ({"grammar": 'root ::= "a"'}, "a reply is constrained to a response format"),+ ),+)+def test_a_tool_or_constrained_reply_joins_a_batch_that_decodes_it_from_its_ids(+ monkeypatch, extra, refusal+):+ from dataclasses import make_dataclass++ from core.inference import mlx_inference++ engine = _batch_engine(monkeypatch)+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)+ monkeypatch.setattr(mlx_inference, "_batched_vision_engine_gap", lambda _engine: None)+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False+ assert backend.batch_unavailable_reason([{}, extra]) is None+ backend._model, backend._is_vlm = SimpleNamespace(_is_vlm_model = True), True+ backend._processor, backend._speculative_draft = object(), None+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": None}, None+ backend._turboquant = False+ assert backend.resident_unavailable_reason({}) is None+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)+ monkeypatch.setattr(engine, "BatchStream", lambda *_a, **_k: None, raising = False)+ for fields, draft, joins in (+ (("token_ids",), None, True),+ ((), None, False),+ (("token_ids",), 1, False),+ (("drafts",), 1, False),+ (("token_ids", "drafts"), 1, True),+ ):+ for name in ("GenerationEvent", "GenerationRequest"):+ monkeypatch.setattr(engine, name, make_dataclass(name, fields), raising = False)+ backend._speculative_draft = draft+ assert backend.resident_unavailable_reason(extra) == (None if joins else refusal)+ assert mlx_inference._VisionBatchSession(backend, width = 2)._decodes_ids == joins+ backend._speculative_draft = None # The fixed multi-reply command decodes the batch's text.+ assert backend.batch_unavailable_reason([{}, extra]) == refusal+ assert backend.resident_unavailable_reason({"response_format": {"type": "text"}}) is None++def test_a_quantized_or_budgeted_kv_cache_does_not_batch(monkeypatch):from core.inference.mlx_inference import MLXInferenceBackend@@ -8429,7 +8786,7 @@ def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):backend._model, backend._processor, backend._is_vlm = object(), object(), Truebackend._kv_quant, backend._kv_context_budget = {"kv_bits": 4}, 4096backend._turboquant = False- backend._vlm_batch_unavailable_reason = lambda requests: None+ backend._vlm_batch_unavailable_reason = lambda requests, **_k: Nonemonkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)capable = lambda: Nonemonkeypatch.setattr(@@ -8446,6 +8803,24 @@ def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):assert backend.resident_unavailable_reason({}) == "old mlx-vlm"[email protected]("gap", (None, "speculative", "old zoo", "no decoder", "no vision"))+def test_a_video_reply_batches_only_as_a_fixed_command_of_a_load_that_reads_it(monkeypatch, gap):+ from core.inference import mlx_inference++ engine, backend = _batch_engine(monkeypatch), _vlm_backend(monkeypatch, markers = None)+ backend._speculative_draft = gap == "speculative" or None+ backend._reads_vision, available = gap != "no vision", lambda request: None+ backend._vlm_batch_unavailable_reason = backend._vlm_resident_unavailable_reason = available+ monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: gap != "no decoder")+ fields = SimpleNamespace(__dataclass_fields__ = ("image", "videos")[: 1 + (gap != "old zoo")])+ monkeypatch.setattr(engine, "GenerationRequest", fields, raising = False)+ clip, sent = {"videos": [_CLIP_B64]}, {"videos_base64": [_CLIP_B64]}+ for row in (clip, sent):+ assert (backend.batch_unavailable_reason([{}, row]) is None) == (gap is None)+ assert backend.resident_unavailable_reason(sent) == "a reply carries a video clip"+ assert backend.resident_unavailable_reason({}) is None++def test_a_turboquant_vision_batch_quantizes_rows_as_the_single_path_does(monkeypatch):from core.inference import mlx_inference@@ -10147,7 +10522,7 @@ def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch){"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}]args = (turn, None, 0, 1, 0, 0, 1, 1, None)- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]((_, stream_kwargs, _),) = streamsassert "prompt_cache" not in stream_kwargsassert "prompt_cache_state" not in stream_kwargs@@ -10176,7 +10551,7 @@ def test_mlx_vlm_a_video_turn_releases_retained_snapshots(monkeypatch):]args = (turn, None, 0, 1, 0, 0, 1, 1, None)- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]assert released == [True]@@ -10197,7 +10572,7 @@ def test_mlx_vlm_an_unset_budget_is_capped_for_a_video_turn(monkeypatch):]args = (turn, None, 0, 1, 0, 0, None, 1, None)- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]assert streams[-1][1]["max_tokens"] == UNSET_GENERATION_BUDGET@@ -10212,13 +10587,13 @@ def test_mlx_vlm_video_clip_lives_on_disk_only_for_the_stream(monkeypatch):]args = (turn, None, 0, 1, 0, 0, 1, 1, None)- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]((stream_args, stream_kwargs, contents),) = streamsassert stream_args[3] is None, "no image was attached"assert contents == [base64.b64decode(_CLIP_B64)]assert not any(Path(p).exists() for p in stream_kwargs["video"])- gen = backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)+ gen = backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])assert next(gen) == "ok"(path,) = streams[-1][1]["video"]assert Path(path).exists()@@ -10226,6 +10601,45 @@ def test_mlx_vlm_video_clip_lives_on_disk_only_for_the_stream(monkeypatch):assert not Path(path).exists()+def test_mlx_vlm_every_clip_reaches_mlx_vlm_in_order_at_one_rate(monkeypatch):+ """Releases before 0.7.3 apply one ``fps`` to every clip, so the slowest affordable rate+ is the one that keeps every clip inside its share of the decode budget."""+ import base64++ from core.inference import mlx_inference++ streams = []+ backend = _video_vlm_backend(monkeypatch, streams)+ # The slowest clip sits between faster ones, so neither the first nor the last rate passes.+ rates = {b"fast clip": 2.0, b"slow clip": 0.5, b"medium clip": 1.0}+ shares = []++ def _rate(+ path,+ _processor,+ clips = 1,+ ):+ shares.append(clips)+ return rates[Path(path).read_bytes()]++ monkeypatch.setattr(mlx_inference, "_video_frame_rate", _rate)+ clips = [base64.b64encode(content).decode() for content in rates]+ turn = [+ {+ "role": "user",+ "content": [{"type": "video"}] * 3 + [{"type": "text", "text": "which"}],+ }+ ]+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)++ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = clips)) == ["ok"]+ ((_, stream_kwargs, contents),) = streams+ assert contents == list(rates)+ assert stream_kwargs["fps"] == 0.5+ assert shares == [3, 3, 3]+ assert not any(Path(p).exists() for p in stream_kwargs["video"])++def _tiny_clip(path,width = 32,@@ -10264,6 +10678,11 @@ def test_mlx_vlm_the_decoded_frame_stack_is_bounded(monkeypatch, tmp_path):monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 4)assert mlx_inference._video_frame_rate(clip, plain) == pytest.approx(4 / 3)+ # mlx-vlm holds every decoded clip at once, so two clips split the budget.+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 8)+ assert mlx_inference._video_frame_rate(clip, plain) == 2.0+ assert mlx_inference._video_sampling_fps([clip, clip], plain) == pytest.approx(4 / 3)+monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 3)with pytest.raises(RuntimeError, match = "32x24 frames are too large"):mlx_inference._video_frame_rate(clip, plain)@@ -10350,14 +10769,14 @@ def test_mlx_vlm_the_frame_rate_rides_the_stream(monkeypatch, tmp_path):assert list(backend._generate_vlm(- turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [clip_b64])) == ["ok"]assert streams[0][1]["fps"] == pytest.approx(4 / 3)monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 3)gen = backend._generate_vlm(- turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [clip_b64])with pytest.raises(RuntimeError, match = "too large to decode"):next(gen)@@ -10376,7 +10795,7 @@ def test_mlx_vlm_a_video_turn_whose_render_is_unusable_is_refused_not_recovered(with pytest.raises(RuntimeError, match = "for a video turn"):list(backend._generate_vlm(- turn, [object()], 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = _CLIP_B64+ turn, [object()], 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [_CLIP_B64]))@@ -10387,11 +10806,11 @@ def test_mlx_vlm_an_undecodable_clip_leaves_no_file_behind(monkeypatch, tmp_pathmonkeypatch.setattr(tempfile, "tempdir", str(tmp_path))backend = _video_vlm_backend(monkeypatch, [])- turn = [{"role": "user", "content": [{"type": "video"}]}]+ turn = [{"role": "user", "content": [{"type": "video"}, {"type": "video"}]}]with pytest.raises(binascii.Error):list(backend._generate_vlm(- turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = "A"+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [_CLIP_B64, "A"]))assert list(tmp_path.iterdir()) == []@@ -10402,7 +10821,7 @@ def test_mlx_vlm_structured_video_items_must_match_attached_clips(monkeypatch):args = (None, 0, 1, 0, 0, 1, 1, None)without_part = [{"role": "user", "content": [{"type": "text", "text": "what moves"}]}]with pytest.raises(RuntimeError, match = "0 structured video item.*1 attached video"):- list(backend._generate_vlm(without_part, *args, _adapter_state = False, video = _CLIP_B64))+ list(backend._generate_vlm(without_part, *args, _adapter_state = False, videos = [_CLIP_B64]))with_part = [{"role": "user", "content": [{"type": "video"}]}]with pytest.raises(RuntimeError, match = "1 structured video item.*0 attached video"):list(backend._generate_vlm(with_part, *args, _adapter_state = False))@@ -10422,21 +10841,35 @@ def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_clicaptured.append((messages, kwargs)) or iter(()))list(- backend.generate_chat_response([{"role": "user", "content": "what moves"}], video = _CLIP_B64)+ backend.generate_chat_response(+ [{"role": "user", "content": "what moves"}], videos = [_CLIP_B64]+ ))messages, kwargs = captured[0]assert messages[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]- assert kwargs["video"] == _CLIP_B64+ assert kwargs["videos"] == [_CLIP_B64]+ list(+ backend.generate_chat_response(+ [{"role": "user", "content": "what moves"}], videos = [_CLIP_B64, "BBBB"]+ )+ )+ messages, kwargs = captured[1]+ assert messages[-1]["content"][:2] == [{"type": "video"}, {"type": "video"}]+ assert kwargs["videos"] == [_CLIP_B64, "BBBB"]backend._reads_vision = Falsewith pytest.raises(RuntimeError, match = "loaded model does not read video"):- list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))+ list(+ backend.generate_chat_response([{"role": "user", "content": "hi"}], videos = [_CLIP_B64])+ )backend._is_vlm = Truebackend._reads_vision = Truemonkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)with pytest.raises(RuntimeError, match = "installed mlx-vlm does not read video"):- list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))+ list(+ backend.generate_chat_response([{"role": "user", "content": "hi"}], videos = [_CLIP_B64])+ )def test_mlx_vlm_decodes_video_only_with_its_clip_decoder(monkeypatch):@@ -10517,6 +10950,61 @@ def test_mlx_reads_video_rejects_a_clip_rendered_as_prose(monkeypatch):assert _mlx_reads_video(unnamed) is True+def _same_template_processor(template = "T", drops_video = True):+ """A processor that would render, with its tokenizer holding ``template``. ``drops_video`` is+ mlx-vlm's Gemma 4 processor, which loses a bare video part its tokenizer renders as written."""+ return SimpleNamespace(+ apply_chat_template = lambda *_a, **_k: "unused",+ chat_template = "T",+ video_processor = object(),+ video_token = "<|video|>",+ drops_video = drops_video,+ tokenizer = SimpleNamespace(chat_template = template),+ )+++def _render_by_target(monkeypatch):+ def _render(target, messages, **_kwargs):+ has_video = any(+ isinstance(part, dict) and part.get("type") == "video"+ for message in messages+ for part in (message.get("content") or [])+ )+ marked = has_video and not getattr(target, "drops_video", False)+ return "<|video|> hi" if marked else "hi"++ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation", _render+ )+++def test_a_video_turn_renders_through_the_tokenizer_only_when_the_processor_drops_it(monkeypatch):+ from core.inference.mlx_inference import MLXInferenceBackend, _mlx_reads_video++ _require_video_stack()+ _render_by_target(monkeypatch)+ drops = _same_template_processor()+ assert _mlx_reads_video(drops) is True+ # A template of the processor's own is what it must render, as before.+ assert _mlx_reads_video(_same_template_processor(template = "other")) is False++ backend = MLXInferenceBackend()+ video_turn = [{"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "hi"}]}]+ image_turn = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]}]+ backend._processor = drops+ prompt, target, _ = backend._render_vlm_prompt(video_turn, None, videos = ["clip.mp4"])+ assert (prompt, target) == ("<|video|> hi", drops.tokenizer)+ _, target, _ = backend._render_vlm_prompt(image_turn, [object()])+ assert target is drops++ # Some processors deliberately differ from their tokenizer, so one that keeps the marker renders.+ keeps = _same_template_processor(drops_video = False)+ backend._processor = keeps+ prompt, target, _ = backend._render_vlm_prompt(video_turn, None, videos = ["clip.mp4"])+ assert (prompt, target) == ("<|video|> hi", keeps)+ assert _mlx_reads_video(keeps) is True++def _sampling_release(monkeypatch, **utils_fields):"""Stand mlx_vlm.utils up in one release's shape, so one processor can be read on each."""mlx_vlm = _require_video_stack_module()@@ -11279,3 +11767,30 @@ def test_mlx_precision_folder_loads_from_its_own_download(monkeypatch, tmp_path)assert calls[-1][0] == (repo,)assert backend.models[repo]["gguf_variant"] is Noneassert len(fetched) == 1+++def test_a_batch_row_without_an_independent_detokenizer_is_refused_not_crashed():+ from core.inference import mlx_inference+ from core.inference.worker import RowRefused++ decoder = SimpleNamespace(suppresses = lambda _id: False)+ mlx_inference._require_row_detokenizer(None)+ mlx_inference._require_row_detokenizer(+ mlx_inference._MlxStreamedText(SimpleNamespace(), decoder, None, ())+ )+ with pytest.raises(RowRefused, match = "detokenizer"):+ mlx_inference._require_row_detokenizer(+ mlx_inference._MlxStreamedText(None, decoder, None, ())+ )+++def test_a_flushed_tail_releases_its_text_before_a_stop_it_completes():+ from core.inference.mlx_inference import _RowStream++ stream = _RowStream(sequences = ("HALT",))+ assert stream.feed(text = "already ") == ["already "]+ assert stream.flush("remaining HALT") == ["already remaining "]+ assert stream.stopped and stream.settle() == []+ plain = _RowStream(sequences = ("HALT",))+ plain.feed(text = "all ")+ assert plain.flush("of it") == ["all of it"] and plain.settle() == []diff --git a/studio/backend/tests/test_mlx_resident_batch.py b/studio/backend/tests/test_mlx_resident_batch.pyindex 6a89df1c63b..855dddfe711 100644--- a/studio/backend/tests/test_mlx_resident_batch.py+++ b/studio/backend/tests/test_mlx_resident_batch.py@@ -161,6 +161,54 @@ def test_replies_a_command_asked_for_together_are_addressed_and_counted_by_row()assert resp.sent[-1]["stats"] is None+def test_a_reply_whose_own_decode_fails_is_answered_alone_and_the_batch_decodes_on():+ batch, resp = _batch()+ batch.admit(_cmd("r1", rows = [{"seed": 1}, {"seed": 2}], parallel_slots = 4), None)+ batch.admit(_cmd("r2", parallel_slots = 4), None)+ batch.admit(_cmd("r3", parallel_slots = 4), None)+ session = batch.session++ def step(_waiting):+ session.rows.remove(("r1", 0))+ session.rows.remove(("r3", None))+ yield ("r1", 0), ValueError("schema")+ yield ("r3", None), KeyError("bind")+ yield ("r2", None), "b"+ # The failed reply's sibling ending in the same step does not answer it as done.+ yield ("r1", 1), None++ session.step = step+ batch.step()+ sent = [(m["type"], m["request_id"], m.get("error")) for m in resp.sent]+ assert sent == [+ ("gen_error", "r1", "schema"),+ ("gen_error", "r3", "'bind'"),+ ("token", "r2", None),+ ]+ assert "ValueError" in resp.sent[0]["stack"] and "KeyError" in resp.sent[1]["stack"]+ assert session.rows == [("r2", None)] and not session.closed+++def test_a_failed_reply_whose_siblings_cannot_be_withdrawn_is_still_answered():+ batch, resp = _batch()+ batch.admit(_cmd("r1", rows = [{"seed": 1}, {"seed": 2}], parallel_slots = 4), None)+ batch.admit(_cmd("r2", parallel_slots = 4), None)+ session = batch.session++ def step(_waiting):+ session.rows.remove(("r1", 0))+ yield ("r1", 0), ValueError("schema")++ def withdraw(_handles):+ raise RuntimeError("metal")+ yield++ session.step, session.withdraw = step, withdraw+ batch.step()+ errors = {m["request_id"]: m["error"] for m in resp.sent if m["type"] == "gen_error"}+ assert errors == {"r1": "schema", "r2": "schema"} and batch.session is None++def test_raw_completion_rows_join_the_batch_and_decode_alone_through_the_completion_path():batch, _ = _batch()cmd = _cmd(diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.pyindex fa3eeeeb815..0a53e32e312 100644--- a/studio/backend/tests/test_presence_penalty.py+++ b/studio/backend/tests/test_presence_penalty.py@@ -351,7 +351,7 @@ def test_the_mlx_vlm_prefill_predicate_matches_its_decoder_gate():def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_it():- """The clip rides the command like an image; a backend without ``video`` fails the request."""+ """Clips ride the command like images; a backend without ``videos`` fails the request."""from pathlib import Pathfrom core.inference.orchestrator import InferenceOrchestrator, _mirrored_model_entry@@ -360,11 +360,11 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_io = InferenceOrchestrator.__new__(InferenceOrchestrator)o.models, o.active_model_name = {}, Noneo._stop_ledger = o._pending_teardowns = None- assert "video_base64" not in o._build_generate_cmd("r", None, messages = [])+ assert "videos_base64" not in o._build_generate_cmd("r", None, messages = [])cmd = o._build_generate_cmd(- "r", None, messages = [{"role": "user", "content": "hi"}], video_b64 = "AAAA"+ "r", None, messages = [{"role": "user", "content": "hi"}], videos_b64 = ["AAAA", "BBBB"])- assert cmd["video_base64"] == "AAAA"+ assert cmd["videos_base64"] == ["AAAA", "BBBB"]class _FakeQueue:def __init__(self):@@ -379,10 +379,10 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_idef generate_chat_response(self,- video = None,+ videos = None,**kwargs,):- self.received = video+ self.received = videosreturn iter(())class _TextBackend:@@ -395,7 +395,7 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_itakes = _VideoBackend()_handle_generate(takes, cmd, _FakeQueue(), threading.Event())- assert takes.received == "AAAA"+ assert takes.received == ["AAAA", "BBBB"]refuses, queue = _TextBackend(), _FakeQueue()_handle_generate(refuses, cmd, queue, threading.Event())assert refuses.called is False@@ -440,10 +440,10 @@ def test_both_orchestrator_entry_points_forward_the_clip_into_the_command():o._build_generate_cmd = _buildturn = [{"role": "user", "content": "hi"}]with pytest.raises(_Built):- list(o.generate_chat_response(messages = turn, video = "AAAA"))+ list(o.generate_chat_response(messages = turn, videos = ["AAAA"]))with pytest.raises(_Built):- list(o.generate_with_adapter_control(use_adapter = True, messages = turn, video = "AAAA"))- assert [kw["video_b64"] for kw in built] == ["AAAA", "AAAA"]+ list(o.generate_with_adapter_control(use_adapter = True, messages = turn, videos = ["AAAA"]))+ assert [kw["videos_b64"] for kw in built] == [["AAAA"], ["AAAA"]]def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_serves_it():diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex a251a7ab8fb..1c7a491c982 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -2268,7 +2268,7 @@ def test_video_turn_with_tools_enabled_keeps_the_client_tool_catalog(monkeypatchbody = _json_body(_call(payload, monkeypatch, backend))assert backend.calls[0]["tools"] == [LOOKUP_TOOL]- assert backend.calls[0]["video"] == clip+ assert backend.calls[0]["videos"] == [clip]assert body["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "lookup"@@ -2299,7 +2299,7 @@ def test_a_nudge_retry_keeps_the_video_on_the_question_turn(monkeypatch):assert len(backend.calls) == 2, "the nudge retry did not run"retry = backend.calls[1]["messages"]- assert backend.calls[1]["video"] == clip+ assert backend.calls[1]["videos"] == [clip]question = next(m for m in retry if m["role"] == "user")assert question["content"][0] == {"type": "video"}assert retry[-1]["role"] == "user" and isinstance(retry[-1]["content"], str)diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.pyindex f7d72db5ea5..ca886ae9cbc 100644--- a/studio/backend/tests/test_video_attachment_part.py+++ b/studio/backend/tests/test_video_attachment_part.py@@ -162,19 +162,19 @@ def _model_info(**fields):def test_a_video_backend_is_handed_the_bare_clip():from fastapi import HTTPExceptionfrom models.inference import ChatCompletionRequest- from routes.inference import _VIDEO_INPUT_REFUSAL, _local_video_clip+ from routes.inference import _VIDEO_INPUT_REFUSAL, _local_video_clipspayload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)- assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64+ assert _local_video_clips(payload, _model_info(has_video_input = True)) == [_CLIP_B64]payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _CLIP_B64)- assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64+ assert _local_video_clips(payload, _model_info(has_video_input = True)) == [_CLIP_B64]with pytest.raises(HTTPException) as exc:- _local_video_clip(payload, _model_info())+ _local_video_clips(payload, _model_info())assert exc.value.status_code == 400 and exc.value.detail == _VIDEO_INPUT_REFUSALpayload = ChatCompletionRequest(model = "m", messages = [], video_base64 = "data:video/mp4;base64,")with pytest.raises(HTTPException) as exc:- _local_video_clip(payload, _model_info(has_video_input = True))+ _local_video_clips(payload, _model_info(has_video_input = True))assert exc.value.status_code == 400@@ -182,15 +182,15 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera"""The gate precedes the early-returning dispatches, and the clip rides the generation kwargs."""source = _inference_source()handler = source.index("using_gguf = llama_backend.is_loaded")- gate = source.index("_video_clip = _local_video_clip(payload, model_info)", handler)+ gate = source.index("_video_clips = _local_video_clips(payload, model_info)", handler)speech = source.index("return await _monitored_generate_audio(model_name)", handler)audio_input = source.index("# ── Audio INPUT path", handler)assert gate < speech and gate < audio_input- assert 'gen_kwargs["video"] = _video_clip' in source+ assert 'gen_kwargs["videos"] = _video_clips' in sourceuse_tools = source.index("_sf_use_tools = (", handler)tools_block = source[use_tools : source.index("\n )", use_tools)]tools_block = "\n".join(line.split("#")[0] for line in tools_block.splitlines())- assert "and _video_clip is None" in tools_block+ assert "and not _video_clips" in tools_block# Structural, not literal: #10970 widened the image half to `_sf_has_image`.client_tools = source.index("_sf_client_tools = (", handler)block = source[client_tools : source.index("\n )", client_tools)]@@ -200,7 +200,7 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_generaassert "and not _sf_use_tools" in escape, escape# Image and clip must be ALTERNATIVES, each read positively, in either order.image = r"(?:\bimage is not None\b|\b_sf_has_(?:any_)?image\b)"- clip = r"\b_video_clip is not None\b"+ clip = r"\bbool\(_video_clips\)"assert re.search(rf"{image}\s+or\s+{clip}|{clip}\s+or\s+{image}", escape), escapeassert not re.search(r"\bimage is None\b|\bnot\s+_sf_has_(?:any_)?image\b", escape), escape# Settled at the gate: a model without audio input never enters the audio-input path.@@ -208,7 +208,9 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_generaassert conflict < speechassert "_AUDIO_VIDEO_INPUT_DETAIL" in source[conflict : conflict + 200]# The gate reads both spellings, so a video_url part reaches MLX exactly as the field does.- assert "if _request_has_video(payload):\n _video_clip = _local_video_clip(" in source+ assert (+ "if _request_has_video(payload):\n _video_clips = _local_video_clips(" in source+ )def test_token_counting_refuses_video_like_image_and_audio():@@ -222,10 +224,10 @@ def test_token_counting_refuses_video_like_image_and_audio():def test_both_video_checks_share_one_rule():- """Two size checks that drift let the pre-switch one pass what the post-load- one refuses, which is the model load this was meant to avoid."""+ """Size checks that drift let the pre-switch one pass what a post-load one+ refuses, which is the model load this was meant to avoid."""source = _inference_source()- assert source.count("= _request_video_rejection(payload)") == 2+ assert source.count("= _request_video_rejection(payload)") == 3def _video_url_request(diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.pyindex 49d77c6453d..dcd4fe1661f 100644--- a/studio/backend/tests/test_video_chat_completions_behaviour.py+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py@@ -238,7 +238,7 @@ def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(mondef _gate(payload, model_info):reached.append(model_info)- return _CLIP_B64+ return [_CLIP_B64]monkeypatch.setattr(inference_route,@@ -248,7 +248,7 @@ def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(monmodels = {"mlx-model": {"is_vision": True, "has_video_input": True}},),)- monkeypatch.setattr(inference_route, "_local_video_clip", _gate)+ monkeypatch.setattr(inference_route, "_local_video_clips", _gate)with _client(monkeypatch, None) as client:response = client.post("/v1/chat/completions", json = body)@@ -259,15 +259,15 @@ def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(mondef test_the_non_gguf_gate_reads_both_spellings(monkeypatch):- """_local_video_clip predates the part, so left reading payload.video_base64 alone it would+ """_local_video_clips predates the part, so left reading payload.video_base64 alone it wouldhand an MLX model nothing while the legacy field worked."""from models.inference import ChatCompletionRequestinfo = {"is_vision": True, "has_video_input": True}field = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)part = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))- assert inference_route._local_video_clip(field, info) == _CLIP_B64- assert inference_route._local_video_clip(part, info) == _CLIP_B64+ assert inference_route._local_video_clips(field, info) == [_CLIP_B64]+ assert inference_route._local_video_clips(part, info) == [_CLIP_B64]def test_a_non_gguf_backend_without_video_refuses_by_name():@@ -276,34 +276,42 @@ def test_a_non_gguf_backend_without_video_refuses_by_name():payload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))with pytest.raises(HTTPException) as exc:- inference_route._local_video_clip(payload, {"is_vision": True})+ inference_route._local_video_clips(payload, {"is_vision": True})assert exc.value.status_code == 400assert exc.value.detail == inference_route._VIDEO_INPUT_REFUSALassert "MLX" in inference_route._VIDEO_INPUT_REFUSAL-def test_a_second_clip_is_refused_rather_than_dropped_on_a_non_gguf_backend():- """Generation takes one video kwarg. Taking clips[0] silently changed the prompt by- backend: GGUF forwards every clip, so the same request meant two different things."""+def test_every_clip_is_served_in_request_order_on_a_non_gguf_backend():+ """GGUF forwards every clip, so the non-GGUF path must too rather than serve one; the+ request-wide count and size caps still apply, since the pre-switch check may not have run."""from fastapi import HTTPExceptionfrom models.inference import ChatCompletionRequestinfo = {"is_vision": True, "has_video_input": True}- two = ChatCompletionRequest.model_validate(_part_body(_DATA_URI, _DATA_URI))- with pytest.raises(HTTPException) as exc:- inference_route._local_video_clip(two, info)- assert exc.value.status_code == 400- assert "Only one video" in exc.value.detail+ second = "BBBBGGZ0eXBtcDQy"+ two = ChatCompletionRequest.model_validate(+ _part_body(_DATA_URI, f"data:video/mp4;base64,{second}")+ )+ assert inference_route._local_video_clips(two, info) == [_CLIP_B64, second]both = ChatCompletionRequest.model_validate(- _part_body(_DATA_URI, **{"video_base64": _DATA_URI})+ _part_body(f"data:video/mp4;base64,{second}", **{"video_base64": _DATA_URI})+ )+ assert inference_route._local_video_clips(both, info) == [_CLIP_B64, second]++ too_many = ChatCompletionRequest.model_validate(+ _part_body(*[_DATA_URI] * (inference_route._MAX_VIDEO_CLIPS_PER_REQUEST + 1)))with pytest.raises(HTTPException) as exc:- inference_route._local_video_clip(both, info)- assert exc.value.status_code == 400+ inference_route._local_video_clips(too_many, info)+ assert exc.value.status_code == 400 and "Too many videos" in exc.value.detail- one = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))- assert inference_route._local_video_clip(one, info) == _CLIP_B64+ half = "data:video/mp4;base64," + "A" * (inference_route._MAX_VIDEO_B64_CHARS // 2 + 10)+ oversized = ChatCompletionRequest.model_validate(_part_body(half, half))+ with pytest.raises(HTTPException) as exc:+ inference_route._local_video_clips(oversized, info)+ assert exc.value.status_code == 413def test_an_unsupported_scheme_is_refused_on_the_non_gguf_path_too():@@ -314,7 +322,7 @@ def test_an_unsupported_scheme_is_refused_on_the_non_gguf_path_too():payload = ChatCompletionRequest.model_validate(_part_body("file:///etc/passwd"))with pytest.raises(HTTPException) as exc:- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ inference_route._local_video_clips(payload, {"is_vision": True, "has_video_input": True})assert exc.value.status_code == 400assert "Unsupported video URL scheme" in exc.value.detail@@ -352,7 +360,7 @@ def test_a_remote_clip_is_refused_on_a_non_gguf_backend():payload = ChatCompletionRequest.model_validate(_part_body(_REMOTE))with pytest.raises(HTTPException) as exc:- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ inference_route._local_video_clips(payload, {"is_vision": True, "has_video_input": True})assert exc.value.status_code == 400assert "Remote video URLs are not supported" in exc.value.detail@@ -665,14 +673,14 @@ def _older_turn_body(clip = _DATA_URI):def test_a_clip_on_an_older_turn_is_refused_on_a_non_gguf_backend():"""GGUF keeps the part on the turn that carried it; the non-GGUF path flattens the parts and- hands generation one video kwarg, which MLX attaches to the newest turn. Refusing keeps the+ hands generation the clips alone, which MLX attaches to the newest turn. Refusing keeps thetwo backends from reading the same request as different conversations."""from fastapi import HTTPExceptionfrom models.inference import ChatCompletionRequestpayload = ChatCompletionRequest.model_validate(_older_turn_body())with pytest.raises(HTTPException) as exc:- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ inference_route._local_video_clips(payload, {"is_vision": True, "has_video_input": True})assert exc.value.status_code == 400assert "attached to the latest message" in exc.value.detail@@ -680,10 +688,9 @@ def test_a_clip_on_an_older_turn_is_refused_on_a_non_gguf_backend():def test_a_clip_on_the_latest_turn_is_still_served_on_a_non_gguf_backend():from models.inference import ChatCompletionRequestpayload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))- assert (- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})- == _CLIP_B64- )+ assert inference_route._local_video_clips(+ payload, {"is_vision": True, "has_video_input": True}+ ) == [_CLIP_B64]def test_the_legacy_field_is_not_treated_as_an_older_turn():@@ -692,10 +699,9 @@ def test_the_legacy_field_is_not_treated_as_an_older_turn():payload = ChatCompletionRequest.model_validate(_field_body())assert inference_route._video_is_on_an_older_turn(payload) is False- assert (- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})- == _CLIP_B64- )+ assert inference_route._local_video_clips(+ payload, {"is_vision": True, "has_video_input": True}+ ) == [_CLIP_B64]def test_gguf_still_keeps_a_clip_on_the_turn_that_carried_it(monkeypatch):diff --git a/studio/backend/tests/test_video_pr9057_simulation.py b/studio/backend/tests/test_video_pr9057_simulation.pyindex 965238eea59..2fc1a16985f 100644--- a/studio/backend/tests/test_video_pr9057_simulation.py+++ b/studio/backend/tests/test_video_pr9057_simulation.py@@ -371,7 +371,7 @@ def _routes_source() -> str:"needle",["raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)",- "_video_clip = _local_video_clip(payload, model_info)",+ "_video_clips = _local_video_clips(payload, model_info)",'if not getattr(llama_backend, "_has_video_input", False):','"Video input is not supported together with guided decoding or client-supplied tools yet."','"Cannot count tokens for messages containing video."',diff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.pyindex 850d12c5651..cc35722ccaa 100644--- a/studio/backend/tests/test_vision_client_tools.py+++ b/studio/backend/tests/test_vision_client_tools.py@@ -891,7 +891,7 @@ def test_video_tool_support_is_classified_from_the_processor_template():assert backend.calls, "generation never ran"assert backend.calls[0]["tools"] == [passthrough.LOOKUP_TOOL]- assert backend.calls[0]["video"] == clip+ assert backend.calls[0]["videos"] == [clip]def test_mlx_selects_structured_content_for_a_processor_render():