Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: batch MLX tool-protocol, response_format and video replies; several clips and Gemma 4 video on MLX (#15920)

Long Yixingcb07e19f0dc4
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 355bc9f8c27..ee6252119c6 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -2591,7 +2591,7 @@ def messages_with_attached_image(
fallback_user_text: str = "",
structured_content: bool = False,
image: int = 1,
- video: bool = False,
+ video: int = 0,
audio: Any = None,
extra_audio: Sequence[Any] = (),
) -> list:
diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.py
index 44e634061c0..08e9a6f7608 100644
--- a/studio/backend/core/inference/managed_engine.py
+++ b/studio/backend/core/inference/managed_engine.py
@@ -1199,7 +1199,7 @@ class ManagedEngine:
raise ValueError(
"Tools and adapter comparisons are not supported by this optional engine profile."
)
- if params.get("video"):
+ if params.get("videos"):
raise ValueError(
"Video input is not yet supported by the Unsloth managed engine transport."
)
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 3502c480db7..2ee53103b4e 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -13,7 +13,7 @@ import shutil
import sys
import threading
import time
-from collections import OrderedDict
+from collections import Counter, OrderedDict
from contextlib import ExitStack, closing, contextmanager, nullcontext
from typing import Optional, Generator
from core.inference.worker import RowRefused
@@ -1589,6 +1589,43 @@ def _video_placeholder(processor):
return None
+def _marks_video(target, processor) -> bool:
+ """Whether ``target`` renders a video part as a marker: a render that merely differs from the
+ text one can still be prose, as in the image probe."""
+ from core.inference.chat_template_helpers import apply_chat_template_for_generation
+
+ marked = apply_chat_template_for_generation(target, _VIDEO_PROBE_MESSAGES)
+ if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):
+ return False
+ if _vlm_prompt_issue(marked, _VIDEO_PROBE_MESSAGES):
+ return False
+ placeholder = _video_placeholder(processor)
+ return placeholder is None or placeholder in marked
+
+
+def _video_render_target(processor):
+ """What renders a turn with video. mlx-vlm's Gemma 4 processor rebuilds each message from media
+ counts and drops a video part it holds no clip for; its tokenizer carries the same template and
+ renders the part as written. A processor that keeps the marker keeps rendering, since some
+ deliberately differ from their tokenizer (MiniCPM-V's non-thinking prefix)."""
+ from core.inference.chat_template_helpers import chat_render_target
+
+ target = chat_render_target(processor)
+ tokenizer = getattr(processor, "tokenizer", None)
+ template = getattr(processor, "chat_template", None)
+ if (
+ target is not processor
+ or not template
+ or getattr(tokenizer, "chat_template", None) != template
+ ):
+ return target
+ try:
+ drops = not _marks_video(processor, processor)
+ except Exception:
+ drops = True
+ return tokenizer if drops else processor
+
+
def _mlx_lm_reads_images(model) -> bool:
"""An image model unsloth-zoo loaded through mlx-lm, with an unsloth-zoo that prefills its images."""
if getattr(model, "_unsloth_supports_images", None) is not True:
@@ -1601,26 +1638,13 @@ def _mlx_lm_reads_images(model) -> bool:
def _mlx_reads_video(processor) -> bool:
- """Processor carries a video component AND the template places a video marker: a render that
- merely differs from the text one can still be prose, as in the image probe."""
+ """Processor carries a video component AND the template places a video marker."""
if processor is None or getattr(processor, "video_processor", None) is None:
return False
if not _mlx_vlm_decodes_video():
return False
- from core.inference.chat_template_helpers import (
- apply_chat_template_for_generation,
- chat_render_target,
- )
-
try:
- target = chat_render_target(processor)
- marked = apply_chat_template_for_generation(target, _VIDEO_PROBE_MESSAGES)
- if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):
- return False
- if _vlm_prompt_issue(marked, _VIDEO_PROBE_MESSAGES):
- return False
- placeholder = _video_placeholder(processor)
- return placeholder is None or placeholder in marked
+ return _marks_video(_video_render_target(processor), processor)
except BaseException:
# A load must never fail on a probe, matching _image_marker_survives.
return False
@@ -1697,9 +1721,17 @@ def _video_sampling_target(processor) -> tuple[float, int, Optional[int]]:
)
-def _video_frame_rate(clip_path: str, processor) -> Optional[float]:
- """The sampling ``fps`` keeping the decoded stack within the budget; None when the clip has no
- readable frame geometry or count, which mlx-vlm then refuses itself."""
+class _ClipsTooLarge(RuntimeError):
+ pass
+
+
+def _video_frame_rate(
+ clip_path: str,
+ processor,
+ clips: int = 1,
+) -> Optional[float]:
+ """The sampling ``fps`` keeping the decoded stack within this clip's share of the budget; None
+ when the clip has no readable frame geometry or count, which mlx-vlm then refuses itself."""
import cv2
capture = cv2.VideoCapture(clip_path)
@@ -1718,16 +1750,31 @@ def _video_frame_rate(clip_path: str, processor) -> Optional[float]:
# The one measurement mlx-vlm substitutes rather than refusing; match its substitution.
native_fps = 1.0
wanted_fps, min_frames, nframes = _video_sampling_target(processor)
- affordable = _VIDEO_DECODE_BUDGET_BYTES // (2 * 3 * width * height)
+ # mlx-vlm holds every decoded clip until the processor runs, so the clips share one budget.
+ budget = _VIDEO_DECODE_BUDGET_BYTES // clips
+ affordable = budget // (2 * 3 * width * height)
needed = min_frames if nframes is None else nframes
if affordable < needed:
- raise RuntimeError(
+ raise _ClipsTooLarge(
f"The video's {width}x{height} frames are too large to decode: {needed} of them "
- f"exceed the {_VIDEO_DECODE_BUDGET_BYTES >> 20} MiB frame budget."
+ f"exceed the {budget >> 20} MiB frame budget."
)
return min(wanted_fps, affordable * native_fps / total_frames)
+def _video_sampling_fps(
+ clip_paths,
+ processor,
+ rows = 1,
+) -> Optional[float]:
+ """One rate for every clip, since mlx-vlm before 0.7.3 applies a single ``fps`` to all.
+
+ ``rows`` copies of the clips share the decode budget when a batch decodes them together.
+ """
+ rates = [_video_frame_rate(path, processor, len(clip_paths) * rows) for path in clip_paths]
+ return min((rate for rate in rates if rate is not None), default = None)
+
+
def _discard_video_clip(path: str) -> None:
try:
os.remove(path)
@@ -1935,7 +1982,10 @@ class _MlxStreamedText:
token_id = getattr(response, "token", None)
if token_id is None:
return ""
- token_id = int(token_id)
+ return self.feed_token(int(token_id))
+
+ def feed_token(self, token_id: int) -> str:
+ """One sampled id, for a runtime that reports ids one at a time."""
if token_id in self._stop_ids:
self._held_stops.append(token_id)
return ""
@@ -1976,6 +2026,25 @@ class _MlxStreamedText:
return closing
+def _require_row_detokenizer(streamed):
+ """A batch row feeds ids one at a time, so unlike the single path it has no runtime text to fall back on."""
+ if streamed is not None and streamed._detokenizer is None:
+ raise RowRefused("no independent detokenizer to decode this row's ids")
+
+
+def _native_reply_text(decoder, token_ids, stop_ids, prefix) -> str:
+ """A native-protocol reply re-decoded from its ids. A runtime may stop on an allowlisted
+ control (<|end_message|>): the trailing stop id stays only when it closes an open envelope,
+ judged with the prefill, since a restored <think> lives in the prompt."""
+ ids = list(token_ids)
+ if ids and ids[-1] in stop_ids:
+ whole = decoder.decode(ids)
+ closer = whole[len(decoder.decode(ids[:-1])) :]
+ if not closes_an_open_envelope(prefix + whole, closer):
+ ids = ids[:-1]
+ return decoder.decode(ids)
+
+
def _mlx_stop_cut(text: str, stops) -> tuple[int, bool]:
"""Holds back text that could grow into a stop; a rewriting caller must withhold until turn ends"""
resolved = text.rstrip("\ufffd")
@@ -3572,6 +3641,14 @@ def _make_mlx_logit_bias_processor(logit_bias: dict):
return _processor
+_DIFFUSION_RESPONSE_FORMAT = (
+ "response_format is not supported on this model: it generates by "
+ "diffusion rather than one token at a time, so no grammar can "
+ "constrain the next token. Load an autoregressive model to use "
+ "guided decoding."
+)
+
+
def _vlm_generation_is_diffusion(model) -> bool:
try:
from mlx_vlm.generate.diffusion import is_diffusion_model
@@ -3933,6 +4010,15 @@ class _RowStream:
return [self._prefix + self.sampled[: self._cut]]
return []
+ def flush(self, text):
+ """Text a decoder held until the reply ended; ``settle`` delivers it for normalized or re-decoded rows."""
+ if not self._sequences or (self._normalizer is None and self._decode is None):
+ # feed releases up to a stop it finds here, which settle no longer would once stopped.
+ return self.feed(text = text)
+ self.sampled += text
+ self._cut, self.stopped = _mlx_stop_cut(self.sampled, self._sequences)
+ return []
+
def settle(self):
if not self._sequences:
if self._decode is not None and self.sampled.endswith("\ufffd"):
@@ -3997,21 +4083,59 @@ def _row_processor_gap(engine, requests):
return unavailable() if callable(unavailable) else None
+def _constrained(request):
+ return request.get("response_format") not in (None, {"type": "text"}) or (
+ request.get("grammar") is not None
+ )
+
+
def _request_batch_gap(request):
"""What a reply asks for that only the one-at-a-time decode serves, or None."""
if request.get("images") or request.get("images_base64"):
return "a reply replays tool images"
- if request.get("video") is not None or request.get("video_base64"):
- return "a reply carries a video clip"
- if request.get("tools") or request.get("tool_protocol_active"):
- return "a reply runs the tool protocol"
- if request.get("response_format") is not None or request.get("grammar") is not None:
- return "a reply is constrained to a response format"
if request.get("n_probs"):
return "a reply reports token probabilities"
return None
+def _carries_video(request):
+ return bool(request.get("videos") or request.get("videos_base64"))
+
+
+def _batch_clips(stack, plans, processor):
+ """Each row's clips as files mlx-vlm reads, removed when ``stack`` closes.
+
+ Rows with the same clips share their files, so unsloth-zoo batches them as one group, and that
+ group decodes its clips together, so its rows share one decode budget. Rows whose clips are too
+ large to share it get files of their own, so each decodes alone with the whole budget.
+ """
+ files, shared = [], {}
+ stack.callback(lambda: [_discard_video_clip(path) for path in files])
+
+ def write(clip):
+ files.append(_write_video_clip(clip))
+ return files[-1]
+
+ rows = Counter(tuple(plan.videos or ()) for plan in plans)
+ clips = []
+ for plan in plans:
+ videos = tuple(plan.videos or ())
+ paths = [
+ shared[clip] if clip in shared else shared.setdefault(clip, write(clip))
+ for clip in videos
+ ]
+ if not paths:
+ clips.append({})
+ continue
+ try:
+ rate = _video_sampling_fps(paths, processor, rows[videos])
+ except _ClipsTooLarge:
+ paths = [write(clip) for clip in videos]
+ rate = _video_sampling_fps(paths, processor)
+ clips.append({"videos": paths, "fps": rate})
+ return clips
+
+
def _batched_vision_engine_gap(engine):
def fields(name):
return _engine_fields(engine, name)
@@ -4035,7 +4159,7 @@ class _TextRowPlan:
"sampler",
"processors",
"max_tokens",
- "detokenizes",
+ "streamed",
)
def __init__(
@@ -4047,12 +4171,12 @@ class _TextRowPlan:
sampler,
processors,
max_tokens,
- detokenizes = False,
+ streamed = None,
):
self.prompt = prompt
self.think_prefix = think_prefix
self.stream = stream
- self.detokenizes = detokenizes
+ self.streamed = streamed
self.sampler = sampler
self.processors = processors
self.max_tokens = max_tokens
@@ -4067,14 +4191,14 @@ class _TextBatchRow:
"all_tokens",
"prompt_len",
"cached",
- "detokenizer",
"generated",
"reason",
"processing_at",
"ready_at",
+ "failure",
)
- def __init__(self, *, handle, plan, cache_key, all_tokens, prompt_len, cached, detokenizer):
+ def __init__(self, *, handle, plan, cache_key, all_tokens, prompt_len, cached):
self.handle = handle
self.plan = plan
self.uid = None
@@ -4082,11 +4206,41 @@ class _TextBatchRow:
self.all_tokens = all_tokens
self.prompt_len = prompt_len
self.cached = cached
- self.detokenizer = detokenizer
self.generated = 0
self.reason = None
self.processing_at = None
self.ready_at = None
+ self.failure = None
+
+
+def _failing_alone(row, processors):
+ """A row's processors, whose failure is kept on the row rather than raised through the batch."""
+
+ def guarded(processor):
+ def _processor(tokens, logits):
+ if row.failure is None:
+ try:
+ return processor(tokens, logits)
+ except Exception as exc:
+ row.failure = exc
+ return logits
+
+ # A drafted batch drafts only for rows whose processors all read just the history.
+ _processor.history_only = getattr(processor, "history_only", False)
+ return _processor
+
+ return [guarded(processor) for processor in processors]
+
+
+def _streamed_tail(row):
+ """What a row decoded from ids still holds when it ends, released as the single path's finish."""
+ stream = row.plan.stream
+ if row.plan.streamed is None or stream.stopped:
+ return []
+ tail = stream.flush(row.plan.streamed.finish(stream.sampled))
+ if stream.stopped:
+ row.reason = "stop"
+ return tail
def _row_kwargs(request):
@@ -4179,6 +4333,10 @@ class _TextBatchSession:
_messages_with_system_prompt(messages, request.get("system_prompt", ""))
if image is None
else backend._conversation_to_render(messages, request.get("system_prompt", ""), image),
+ tool_protocol_active = request.get("tool_protocol_active"),
+ response_format = request.get("response_format"),
+ reasoning_is_extracted = request.get("reasoning_is_extracted", False),
+ grammar = request.get("grammar"),
**row_kwargs,
)
began = time.perf_counter()
@@ -4205,7 +4363,6 @@ class _TextBatchSession:
all_tokens = all_tokens,
prompt_len = len(remainder) if image is None else len(all_tokens),
cached = len(prefix) if image is None else 0,
- detokenizer = backend._tokenizer.detokenizer if plan.detokenizes else None,
)
if image is not None:
row.processing_at = began
@@ -4215,7 +4372,7 @@ class _TextBatchSession:
caches = [cache],
all_tokens = [prefix],
samplers = [plan.sampler],
- logits_processors = [_row_logits_processors(plan.processors)],
+ logits_processors = [_failing_alone(row, _row_logits_processors(plan.processors))],
)
if len(uids) != 1:
self.generator.remove(list(uids))
@@ -4245,6 +4402,13 @@ class _TextBatchSession:
row = self._by_uid.get(event.uid)
if row is not None and row.processing_at is None:
row.processing_at = began
+ # A processor runs a step ahead, so the single path raises before yielding this token;
+ # reported first, so a sibling ending in this step cannot answer the request as done.
+ for row in [row for row in self._rows.values() if row.failure is not None]:
+ self.generator.remove([row.uid])
+ self._rows.pop(row.handle, None)
+ self._by_uid.pop(row.uid, None)
+ yield row.handle, row.failure
for event in events:
row = self._by_uid.get(event.uid)
if row is None:
@@ -4258,18 +4422,9 @@ class _TextBatchSession:
def _feed(self, row, event):
stream = row.plan.stream
- text = None
- if event.finish_reason is None:
- if row.detokenizer is not None:
- row.detokenizer.add_token(event.token)
- text = row.detokenizer.last_segment
- else:
- text = ""
- if row.detokenizer is not None:
- if event.finish_reason == "length":
- row.detokenizer.add_token(event.token)
- row.detokenizer.finalize()
- text = row.detokenizer.last_segment
+ # A stop id is held, not written: the end-of-reply flush decides whether it closes markup.
+ streamed = row.plan.streamed
+ text = "" if streamed is None else streamed.feed_token(int(event.token))
row.generated += 1
yield from (
(row.handle, snapshot) for snapshot in stream.feed(token = event.token, text = text)
@@ -4318,6 +4473,7 @@ class _TextBatchSession:
self._rows.pop(row.handle, None)
self._by_uid.pop(row.uid, None)
stream = row.plan.stream
+ yield from ((row.handle, snap) for snap in _streamed_tail(row))
yield from ((row.handle, snap) for snap in stream.settle())
yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))
ready_at = row.ready_at or time.perf_counter()
@@ -4351,12 +4507,15 @@ class _VLMRowPlan:
__slots__ = (
"prompt",
"images",
+ "videos",
"think_prefix",
"stream",
"kwargs",
"sampling",
"max_tokens",
"processors",
+ "streamed",
+ "speculates",
)
def __init__(
@@ -4370,15 +4529,21 @@ class _VLMRowPlan:
sampling,
max_tokens,
processors = None,
+ streamed = None,
+ speculates = True,
+ videos = None,
):
self.prompt = prompt
self.images = images
+ self.videos = videos
self.think_prefix = think_prefix
self.stream = stream
self.kwargs = kwargs
self.sampling = sampling
self.max_tokens = max_tokens
self.processors = processors
+ self.streamed = streamed
+ self.speculates = speculates
class _VisionBatchRow:
@@ -4394,9 +4559,16 @@ class _VisionBatchRow:
"ready_at",
"cancelled",
"drafts",
+ "failure",
)
- def __init__(self, *, handle, plan, row):
+ def __init__(
+ self,
+ *,
+ handle,
+ plan,
+ row = None,
+ ):
self.handle = handle
self.plan = plan
self.row = row
@@ -4408,6 +4580,14 @@ class _VisionBatchRow:
self.ready_at = None
self.cancelled = False
self.drafts = None
+ self.failure = None
+
+
+def _vision_batch_reports_ids(engine, speculative):
+ """Whether vision rows can be decoded from their sampled ids, as the one-at-a-time decode does."""
+ return "token_ids" in _engine_fields(engine, "GenerationEvent") and (
+ speculative is None or "drafts" in _engine_fields(engine, "GenerationRequest")
+ )
def _row_prompt_cache_gap():
@@ -4435,6 +4615,7 @@ class _VisionBatchSession:
adapter_state = None,
owns_model = False,
):
+ from unsloth_zoo.mlx import generate as engine
from unsloth_zoo.mlx.generate import BatchStream, GenerationDefaults
self.backend = backend
@@ -4452,6 +4633,7 @@ class _VisionBatchSession:
)
self._held.enter_context(backend._int8_prefill_scope())
self._speculative = getattr(backend, "_speculative_draft", None)
+ self._decodes_ids = _vision_batch_reports_ids(engine, self._speculative)
self.stream = BatchStream(
backend._model,
backend._processor,
@@ -4491,7 +4673,8 @@ class _VisionBatchSession:
)
backend = self.backend
- plan = backend._plan_vlm_request(request)
+ plan = backend._plan_vlm_request(request, decodes_ids = self._decodes_ids)
+ row = _VisionBatchRow(handle = handle, plan = plan)
state = (
backend._vlm_batch_row_cache(self._adapter_state, plan.images)
if self._resumes_rows
@@ -4511,16 +4694,16 @@ class _VisionBatchSession:
image = plan.images[0] if plan.images else None,
max_tokens = plan.max_tokens,
sampling = SamplingParams(**plan.sampling),
- logits_processors = plan.processors,
+ logits_processors = plan.processors and _failing_alone(row, plan.processors),
**resume,
+ **({} if self._speculative is None or plan.speculates else {"drafts": False}),
)
)
except BatchRowRefused as refusal:
raise RowRefused(str(refusal)) from refusal
- row = _VisionBatchRow(handle = handle, plan = plan, row = row_number)
- if self._speculative is not None:
- # A speculative batch prefills the row inside add(), so its clock starts before it.
- row.admitted_at = admitted_at
+ row.row = row_number
+ # A speculative batch prefills the row inside add(), so its clock starts before it.
+ row.admitted_at = admitted_at if self._speculative is not None else time.perf_counter()
self._rows[handle] = row
self._by_row[row_number] = row
logger.info(
@@ -4536,22 +4719,36 @@ class _VisionBatchSession:
now, stopped = None, {}
# Only a speculative step spans many tokens, reported as they are decoded; only its unsloth-zoo has the call.
# The batch cannot change mid-step, so a row that stopped itself ends the step and leaves after it.
- for event in (
- self.stream.iter_step(lambda: bool(stopped) or (waiting is not None and waiting()))
+ failing = {}
+ events = (
+ self.stream.iter_step(
+ lambda: bool(stopped or failing) or (waiting is not None and waiting())
+ )
if self._speculative is not None
else self.stream.step()
- ):
+ )
+ # As in _TextBatchSession.step: reported before the step's tokens, which it never yields.
+ yield from self._fail([row for row in self._rows.values() if row.failure is not None])
+ for event in events:
row = self._by_row.get(event.index)
if row is None:
continue
+ if row.failure is not None:
+ # Only a drafted step decodes as it is read, so a row can fail mid-step and leave after it.
+ failing[row.handle] = row
+ continue
if now is None:
now = time.perf_counter()
if row.ready_at is None:
row.ready_at = now
- if event.delta and not row.cancelled:
- yield from (
- (row.handle, snapshot) for snapshot in row.plan.stream.feed(text = event.delta)
- )
+ streamed = row.plan.streamed
+ text = (
+ event.delta
+ if streamed is None
+ else "".join(streamed.feed_token(int(token)) for token in event.token_ids)
+ )
+ if text and not row.cancelled:
+ yield from ((row.handle, snapshot) for snapshot in row.plan.stream.feed(text = text))
if row.plan.stream.stopped:
row.cancelled = True
row.reason = "stop"
@@ -4563,10 +4760,18 @@ class _VisionBatchSession:
yield from self._retire(row, cancelled = row.cancelled)
elif row.cancelled:
stopped[row.handle] = row
+ yield from self._fail(list(failing.values()))
for row in stopped.values():
if row.handle in self._rows:
yield from self._retake(row)
+ def _fail(self, rows):
+ for row in rows:
+ self.stream.withdraw(row.row)
+ self._rows.pop(row.handle, None)
+ self._by_row.pop(row.row, None)
+ yield row.handle, row.failure
+
def withdraw(self, handles):
for handle in list(handles):
row = self._rows.get(handle)
@@ -4592,6 +4797,7 @@ class _VisionBatchSession:
self._rows.pop(row.handle, None)
self._by_row.pop(row.row, None)
stream = row.plan.stream
+ yield from ((row.handle, snap) for snap in _streamed_tail(row))
yield from ((row.handle, snap) for snap in stream.settle())
yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))
ready_at = row.ready_at or time.perf_counter()
@@ -5197,7 +5403,7 @@ class MLXInferenceBackend:
if videos:
# Same fps as generation, not the processor's default rate.
kwargs["videos"] = videos
- fps = _video_frame_rate(videos[0], self._processor)
+ fps = _video_sampling_fps(videos, self._processor)
if fps is not None:
kwargs["fps"] = fps
inputs = prepare_inputs(
@@ -6019,9 +6225,15 @@ class MLXInferenceBackend:
full_messages.extend(messages)
return full_messages
- def _conversation_to_render(self, messages, system_prompt, image):
+ def _conversation_to_render(
+ self,
+ messages,
+ system_prompt,
+ image,
+ video = 0,
+ ):
"""The turns this request renders, whichever way its reply is decoded."""
- if (self._is_vlm or self._reads_vision_input()) and image is not None:
+ if (self._is_vlm or self._reads_vision_input()) and (image is not None or video):
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
)
@@ -6032,6 +6244,8 @@ class MLXInferenceBackend:
self._processor is not None
and _chat_render_target(self._processor) is self._processor
),
+ image = int(image is not None),
+ video = video,
)
else:
full_messages = self._with_system_prompt(messages, system_prompt)
@@ -6104,11 +6318,11 @@ class MLXInferenceBackend:
_adapter_state = None,
# Unrestricted mode uses an EMPTY tools list, so bool(tools) cannot detect the protocol
tool_protocol_active = None,
- video = None,
+ videos = None,
) -> Generator[str, None, None]:
if self._model is None:
raise RuntimeError("No model loaded")
- if video is not None:
+ if videos:
if not _mlx_vlm_decodes_video():
raise RuntimeError("The installed mlx-vlm does not read video.")
if not self._reads_vision_input():
@@ -6122,7 +6336,7 @@ class MLXInferenceBackend:
# History first, the attachment last: MLX binds pixels to markers in order.
attached = list(images or []) + ([image] if image is not None else [])
# Shared with the transformers vision path so both render the same turns (#10092).
- if self._reads_vision_input() and (attached or video is not None):
+ if self._reads_vision_input() and (attached or videos):
# Processor templates want part lists, the tokenizer fallback wants strings.
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
@@ -6137,7 +6351,7 @@ class MLXInferenceBackend:
system_prompt = system_prompt,
structured_content = _renders_via_processor,
image = len(attached),
- video = video is not None,
+ video = len(videos or ()),
)
# That helper skips convos with markers, but replayed MCP picture markers belong to other pictures, so the render would be one short
_prior_markers = image_marker_parts(full_messages)
@@ -6188,7 +6402,7 @@ class MLXInferenceBackend:
tool_protocol_active = tool_protocol_active,
response_format = response_format,
reasoning_is_extracted = reasoning_is_extracted,
- video = video,
+ videos = videos,
)
else:
stream = self._generate_text(
@@ -6555,18 +6769,9 @@ class MLXInferenceBackend:
# Re-decoding every id rebuilds rather than extends, so an invalid byte sequence can revise
# characters already shown. Predates stop handling and affects plain replies too.
if native_token_decoder is not None:
- # A runtime may stop on an allowlisted control (<|end_message|>): drop only the trailing id, so a real tool closer survives
- _ids = list(token_ids)
- if _ids and _ids[-1] in _mlx_stop_token_ids(
- self._tokenizer, self._model
- ):
- # Only without tool markup: the same marker can be the closer that strict parsing needs.
- _whole = native_token_decoder.decode(_ids)
- _closer = _whole[len(native_token_decoder.decode(_ids[:-1])) :]
- # Include the prefill: a restored <think> lives in the prompt, and judging the closer on generated ids alone left the block open
- if not closes_an_open_envelope(think_prefix + _whole, _closer):
- _ids = _ids[:-1]
- sampled = native_token_decoder.decode(_ids)
+ sampled = _native_reply_text(
+ native_token_decoder, token_ids, stop_token_ids, think_prefix
+ )
else:
sampled = self._tokenizer.decode(
token_ids,
@@ -6676,12 +6881,17 @@ class MLXInferenceBackend:
tool_protocol_active = None,
prompt_ids = None,
penalty_window = None,
+ response_format = None,
+ reasoning_is_extracted = False,
+ grammar = None,
) -> "_TextRowPlan":
from mlx_lm.sample_utils import make_sampler
if prompt_ids is not None:
prompt, reasoning_channel_markers, think_prefix = list(prompt_ids), None, ""
_resumed_partial = False
+ preserve_native_channels = False
+ constraint = _raw_prompt_constraint(grammar, self._tokenizer)
else:
render_result = self._render_text_prompt(
messages,
@@ -6695,18 +6905,30 @@ class MLXInferenceBackend:
reasoning_channel_markers = render_result.reasoning_channel_markers
_resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))
+ preserve_native_channels = reasoning_channel_markers is not None
+ think_close_survives = (
+ bool(tools) or tool_protocol_active or preserve_native_channels
+ ) and decoder_preserves_token(
+ self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
+ )
think_prefix = _think_prefix(
prompt,
getattr(self._tokenizer, "all_special_tokens", None),
messages,
continue_final_message,
- preserves_think_close = (
- bool(tools) or tool_protocol_active or reasoning_channel_markers is not None
- )
- and decoder_preserves_token(
- self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
- ),
+ preserves_think_close = think_close_survives,
+ )
+ constraint = _build_grammar_constraint(
+ response_format,
+ self._tokenizer,
+ prompt,
+ reasoning_markers = reasoning_channel_markers,
+ reasoning_is_extracted = reasoning_is_extracted,
+ reply_keeps_special_tokens = preserve_native_channels or think_close_survives,
)
+ if constraint is not None and not constraint.allows_reasoning:
+ think_prefix = ""
+ preserve_native_channels = False
if seed is None:
sampler = make_sampler(
temp = temperature,
@@ -6728,6 +6950,7 @@ class MLXInferenceBackend:
presence_penalty = presence_penalty,
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
+ grammar_constraint = constraint,
penalty_window = penalty_window,
penalty_prompt = prompt_ids,
)
@@ -6739,28 +6962,65 @@ class MLXInferenceBackend:
prompt, reasoning_channel_markers, _resumed_partial
),
)
- if reasoning_channel_markers is not None
+ if preserve_native_channels
+ else None
+ )
+ # As _generate_text decodes the reply, so a batched turn shows the same native controls.
+ native_token_decoder = (
+ NativeToolTokenDecoder(
+ self._tokenizer,
+ preserved_tokens = reasoning_control_tokens(reasoning_channel_markers),
+ )
+ if tools or preserve_native_channels or tool_protocol_active
+ else None
+ )
+ if constraint is not None:
+ constraint.decoded_dropping(
+ native_token_decoder.dropped_ids() if native_token_decoder is not None else None
+ )
+ stop_token_ids = (
+ _mlx_stop_token_ids(self._tokenizer, self._model)
+ if native_token_decoder is not None
+ else ()
+ )
+ streamed = (
+ _MlxStreamedText(
+ _mlx_stream_detokenizer(self._tokenizer),
+ native_token_decoder,
+ self._tokenizer,
+ stop_token_ids,
+ )
+ if preserve_native_channels
else None
)
+ _require_row_detokenizer(streamed)
+ if streamed is not None:
+ decode = None
+ elif native_token_decoder is not None:
+ decode = lambda ids: _native_reply_text(
+ native_token_decoder, ids, stop_token_ids, think_prefix
+ )
+ else:
+ decode = lambda ids: self._tokenizer.decode(
+ ids,
+ skip_special_tokens = True,
+ **(
+ {"clean_up_tokenization_spaces": False}
+ if constraint is not None or prompt_ids is not None
+ else {}
+ ),
+ )
row = _RowStream(
prefix = think_prefix,
sequences = _mlx_stop_sequences(stop),
normalizer = normalizer,
- decode = (
- None
- if normalizer is not None
- else lambda ids: self._tokenizer.decode(
- ids,
- skip_special_tokens = True,
- **({"clean_up_tokenization_spaces": False} if prompt_ids is not None else {}),
- )
- ),
+ decode = decode,
)
return _TextRowPlan(
prompt = prompt,
think_prefix = think_prefix,
stream = row,
- detokenizes = normalizer is not None,
+ streamed = streamed,
sampler = sampler,
processors = logits_processors,
max_tokens = (
@@ -6791,7 +7051,9 @@ class MLXInferenceBackend:
)
# Same render target as the healing catalog the route builds (#7066)
- chat_target = chat_render_target(self._processor)
+ chat_target = (
+ _video_render_target(self._processor) if videos else chat_render_target(self._processor)
+ )
attached_images = 0 if images is None else len(images)
structured_images = sum(
@@ -6924,15 +7186,51 @@ class MLXInferenceBackend:
prompt, markers = recovered.prompt, recovered.reasoning_channel_markers
return prompt, chat_target, markers
- def _plan_vlm_request(self, request) -> "_VLMRowPlan":
+ def _media_generation_limit(self, prompt, max_new_tokens, prompt_ids, images, videos):
+ # Counted from a temp file (base64 is unreadable to the processor), discarded here.
+ counted_clips = []
+ try:
+ if getattr(self, "_kv_context_budget", None):
+ for clip in videos or ():
+ counted_clips.append(_write_video_clip(clip))
+ return self._generation_limit(
+ prompt,
+ max_new_tokens,
+ prompt_ids,
+ images = images,
+ videos = counted_clips or None,
+ cap = UNSET_GENERATION_BUDGET if images or videos else None,
+ )
+ finally:
+ for counted_clip in counted_clips:
+ _discard_video_clip(counted_clip)
+
+ def _plan_vlm_request(
+ self,
+ request,
+ *,
+ decodes_ids = False,
+ ) -> "_VLMRowPlan":
image = request.get("image")
+ videos = request.get("videos") or None
messages = request.get("messages") or []
conversation = (
[]
if request.get("prompt_ids") is not None
- else self._conversation_to_render(messages, request.get("system_prompt", ""), image)
+ else self._conversation_to_render(
+ messages, request.get("system_prompt", ""), image, video = len(videos or ())
+ )
+ )
+ return self._plan_vlm_row(
+ conversation,
+ image,
+ tool_protocol_active = request.get("tool_protocol_active"),
+ response_format = request.get("response_format"),
+ reasoning_is_extracted = request.get("reasoning_is_extracted", False),
+ decodes_ids = decodes_ids,
+ videos = videos,
+ **_row_kwargs(request),
)
- return self._plan_vlm_row(conversation, image, **_row_kwargs(request))
def _plan_vlm_row(
self,
@@ -6957,14 +7255,23 @@ class MLXInferenceBackend:
stop = None,
prompt_ids = None,
penalty_window = None,
+ tool_protocol_active = None,
+ response_format = None,
+ reasoning_is_extracted = False,
+ decodes_ids = False,
+ videos = None,
) -> "_VLMRowPlan":
+ """``decodes_ids``: the batch reports each row's ids, so the reply is decoded from them as
+ ``_generate_vlm`` decodes it; otherwise from the batch's text, which keeps no native controls."""
images = [image] if image is not None else None
+ constraint = token_decoder = None
if prompt_ids is not None:
prompt, markers, think_prefix = list(prompt_ids), None, ""
else:
prompt, chat_target, markers = self._render_vlm_prompt(
messages,
images,
+ videos = videos,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
@@ -6972,12 +7279,38 @@ class MLXInferenceBackend:
continue_final_message = continue_final_message,
)
+ native = decodes_ids and bool(tools or tool_protocol_active or markers is not None)
think_prefix = _think_prefix(
prompt,
getattr(chat_target, "all_special_tokens", None),
messages,
continue_final_message,
+ preserves_think_close = native
+ and decoder_preserves_token(self._tokenizer, "</think>"),
)
+ if decodes_ids:
+ constraint = _build_grammar_constraint(
+ response_format,
+ chat_target,
+ prompt,
+ reasoning_markers = markers,
+ tools = tools,
+ reasoning_is_extracted = reasoning_is_extracted,
+ reply_keeps_special_tokens = True,
+ )
+ if constraint is not None and not constraint.allows_reasoning:
+ think_prefix = ""
+ elif constraint is not None:
+ from core.inference.chat_template_helpers import detect_think_prefill
+ think_prefix = detect_think_prefill(prompt, preserves_think_close = True)
+ if constraint is not None and _vlm_generation_is_diffusion(self._model):
+ from core.inference.grammar_constraint import ResponseFormatError
+ raise ResponseFormatError(_DIFFUSION_RESPONSE_FORMAT)
+ if native and self._tokenizer:
+ token_decoder = NativeToolTokenDecoder(
+ self._tokenizer,
+ preserved_tokens = reasoning_control_tokens(markers),
+ )
normalizer = (
make_reasoning_normalizer(
markers,
@@ -6987,7 +7320,8 @@ class MLXInferenceBackend:
bool(continue_final_message and trailing_assistant_text(messages)),
),
)
- if markers is not None
+ # The normalizer would rewrite marker text inside the document.
+ if markers is not None and (constraint is None or constraint.allows_reasoning)
else None
)
row = _RowStream(
@@ -6995,12 +7329,8 @@ class MLXInferenceBackend:
sequences = _mlx_stop_sequences(stop),
normalizer = normalizer,
)
- max_new_tokens = self._generation_limit(
- prompt,
- max_new_tokens,
- prompt_ids,
- images = images,
- cap = UNSET_GENERATION_BUDGET if images else None,
+ max_new_tokens = self._media_generation_limit(
+ prompt, max_new_tokens, prompt_ids, images, videos
)
vlm_kwargs = dict(
max_tokens = max_new_tokens,
@@ -7023,16 +7353,33 @@ class MLXInferenceBackend:
presence_penalty = presence_penalty,
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
+ grammar_constraint = constraint,
penalty_window = penalty_window,
penalty_prompt = prompt_ids,
)
if processors:
# Not the repetition_penalty shortcut: mlx-vlm would build it outside our narrowing
vlm_kwargs["logits_processors"] = _row_logits_processors(processors)
+ streamed = None
+ if constraint is not None:
+ # mlx-vlm stops on the config's ids, which some repos set apart from the tokenizer's.
+ constraint.stops_on(_mlx_stop_token_ids(self._tokenizer, self._model))
+ if token_decoder is not None:
+ constraint.decoded_dropping(token_decoder.dropped_ids())
+ if token_decoder is not None:
+ streamed = _MlxStreamedText(
+ _mlx_stream_detokenizer(self._processor)
+ or _mlx_stream_detokenizer(self._tokenizer),
+ token_decoder,
+ self._tokenizer,
+ _mlx_stop_token_ids(self._tokenizer, self._model),
+ )
+ _require_row_detokenizer(streamed)
return _VLMRowPlan(
prompt = prompt,
images = images,
+ videos = videos,
think_prefix = think_prefix,
stream = row,
kwargs = vlm_kwargs,
@@ -7045,6 +7392,9 @@ class MLXInferenceBackend:
},
max_tokens = vlm_kwargs["max_tokens"],
processors = processors,
+ streamed = streamed,
+ # As _generate_vlm, which drops the draft for tool turns.
+ speculates = not (tools or tool_protocol_active),
)
def _generate_vlm(
@@ -7071,7 +7421,7 @@ class MLXInferenceBackend:
_adapter_state = None,
tool_protocol_active = None,
stop = None,
- video = None,
+ videos = None,
response_format = None,
reasoning_is_extracted = False,
prompt_ids = None,
@@ -7090,7 +7440,7 @@ class MLXInferenceBackend:
prompt, chat_target, vlm_reasoning_markers = self._render_vlm_prompt(
messages,
images,
- videos = [video] if video is not None else None,
+ videos = videos or None,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
@@ -7116,29 +7466,14 @@ class MLXInferenceBackend:
# Matched on the sampled text, for the reason _generate_text gives.
sequences = _mlx_stop_sequences(stop)
stopped = False
- # Counted from a temp file (base64 is unreadable to the processor), discarded here.
- counted_clip = (
- _write_video_clip(video)
- if video is not None and getattr(self, "_kv_context_budget", None)
- else None
+ max_new_tokens = self._media_generation_limit(
+ prompt, max_new_tokens, prompt_ids, images, videos
)
- try:
- max_new_tokens = self._generation_limit(
- prompt,
- max_new_tokens,
- prompt_ids,
- images = images,
- videos = [counted_clip] if counted_clip is not None else None,
- cap = UNSET_GENERATION_BUDGET if images or video is not None else None,
- )
- finally:
- if counted_clip is not None:
- _discard_video_clip(counted_clip)
logger.info(
- "VLM generating: prompt_len=%d, images=%d, has_video=%s",
+ "VLM generating: prompt_len=%d, images=%d, videos=%d",
len(prompt),
len(images or ()),
- video is not None,
+ len(videos or ()),
)
# generate_step expects temperature=; temp= is silently ignored (stuck at greedy)
vlm_kwargs = dict(
@@ -7183,12 +7518,7 @@ class MLXInferenceBackend:
prefill = detect_think_prefill(prompt, preserves_think_close = True)
if constraint is not None and _vlm_generation_is_diffusion(self._model):
from core.inference.grammar_constraint import ResponseFormatError
- raise ResponseFormatError(
- "response_format is not supported on this model: it generates by "
- "diffusion rather than one token at a time, so no grammar can "
- "constrain the next token. Load an autoregressive model to use "
- "guided decoding."
- )
+ raise ResponseFormatError(_DIFFUSION_RESPONSE_FORMAT)
recorded = token_logprobs = None
if n_probs:
if _vlm_generation_is_diffusion(self._model):
@@ -7270,7 +7600,7 @@ class MLXInferenceBackend:
import mlx.core as mx
vlm_kwargs["input_ids"] = mx.array([self._encode_prompt(prompt)])
session = self._vlm_prompt_cache_session(
- _adapter_state, images, prompt, has_video = video is not None
+ _adapter_state, images, prompt, has_video = bool(videos)
)
vlm_kwargs.update(self._kv_runtime_quant_kwargs())
if session is not None:
@@ -7291,7 +7621,7 @@ class MLXInferenceBackend:
draft is None
or tools
or tool_protocol_active
- or video is not None
+ or videos
or not all(getattr(processor, "history_only", False) for processor in processors)
or _vlm_generation_is_diffusion(self._model)
):
@@ -7315,14 +7645,14 @@ class MLXInferenceBackend:
ExitStack() as generation_scope,
session_scope,
):
- if session is None and (images or video is not None):
+ if session is None and (images or videos):
# Free headroom for the vision pass under the lock; a video turn resumes no snapshot, so retained ones are pure occupancy
self._release_vlm_snapshots()
generation_scope.enter_context(
self._generation_scopes("vision", *(() if draft is None else ("speculative",)))
)
final_response = None
- clip_path = None
+ clip_paths = []
if draft is not None:
from unsloth_zoo.mlx.generate import SamplingParams
@@ -7340,10 +7670,11 @@ class MLXInferenceBackend:
)
vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)
try:
- if video is not None:
- clip_path = _write_video_clip(video)
- vlm_kwargs["video"] = [clip_path]
- frame_rate = _video_frame_rate(clip_path, self._processor)
+ if videos:
+ for clip in videos:
+ clip_paths.append(_write_video_clip(clip))
+ vlm_kwargs["video"] = clip_paths
+ frame_rate = _video_sampling_fps(clip_paths, self._processor)
if frame_rate is not None:
vlm_kwargs["fps"] = frame_rate
# Emit prefilled <think> first; inside the adapter so unsupported requests raise before any output.
@@ -7431,7 +7762,7 @@ class MLXInferenceBackend:
if sequences and not stopped and released < len(sampled):
yield prefill + sampled
finally:
- if clip_path is not None:
+ for clip_path in clip_paths:
_discard_video_clip(clip_path)
cached_n = produced_n = 0
produced_s = 0.0
@@ -7506,6 +7837,9 @@ class MLXInferenceBackend:
reason = _request_batch_gap(request)
if reason is not None:
return reason
+ reason = self._video_batch_gap(requests)
+ if reason is not None:
+ return reason
reason = self._kv_policy_batch_reason()
if reason is not None:
return reason
@@ -7526,6 +7860,9 @@ class MLXInferenceBackend:
if self._model is None:
return "no model is loaded"
reason = _request_batch_gap(request)
+ if reason is None and _carries_video(request):
+ # Whether a clip's row fits the open batch is known only once its frames are decoded.
+ reason = "a reply carries a video clip"
if reason is not None:
return reason
reason = self._kv_policy_batch_reason(resident = True)
@@ -7535,6 +7872,22 @@ class MLXInferenceBackend:
return self._vlm_resident_unavailable_reason(request)
return self._text_batch_unavailable_reason()
+ def _video_batch_gap(self, requests):
+ if not any(map(_carries_video, requests)):
+ return None
+ # The one-at-a-time decode refuses these clips by name.
+ if not self._is_vlm or not self._reads_vision_input() or not _mlx_vlm_decodes_video():
+ return "a reply carries a video clip this load does not read"
+ if self._speculative_draft is not None:
+ return "a video clip decodes without the draft"
+ try:
+ from unsloth_zoo.mlx import generate as engine
+ except ImportError:
+ return "the installed unsloth-zoo has no MLX generation"
+ if "videos" not in _engine_fields(engine, "GenerationRequest"):
+ return "the installed unsloth-zoo cannot batch a video clip"
+ return None
+
def _text_batch_unavailable_reason(self):
"""Why the installed mlx-lm cannot batch this load's replies, or None."""
try:
@@ -7544,7 +7897,7 @@ class MLXInferenceBackend:
return stream_unavailable_reason(self._model, self._tokenizer)
def _vlm_resident_unavailable_reason(self, request):
- reason = self._vlm_batch_unavailable_reason([request])
+ reason = self._vlm_batch_unavailable_reason([request], resident = True)
if reason is not None:
return reason
try:
@@ -7579,7 +7932,12 @@ class MLXInferenceBackend:
or self.resident_unavailable_reason({}) is None
)
- def _vlm_batch_unavailable_reason(self, requests):
+ def _vlm_batch_unavailable_reason(
+ self,
+ requests,
+ *,
+ resident = False,
+ ):
try:
from unsloth_zoo.mlx import generate as engine
except ImportError:
@@ -7590,6 +7948,14 @@ class MLXInferenceBackend:
gap = _row_processor_gap(engine, requests)
if gap is not None:
return gap
+ # Only the resident session decodes rows from their ids, as the one-at-a-time decode does.
+ decodes_ids = resident and _vision_batch_reports_ids(engine, self._speculative_draft)
+ if not decodes_ids and any(
+ request.get("tools") or request.get("tool_protocol_active") for request in requests
+ ):
+ return "a reply runs the tool protocol"
+ if not decodes_ids and any(_constrained(request) for request in requests):
+ return "a reply is constrained to a response format"
if any(request.get("prompt_ids") is not None for request in requests):
# The batch detokenizer keeps control tokens and trims the first leading space.
return "a raw prompt on a vision model decodes on its own"
@@ -7629,6 +7995,9 @@ class MLXInferenceBackend:
):
def report(events):
for row, snapshot in events:
+ if isinstance(snapshot, BaseException):
+ # Every row answers one command, which fails as a single reply would.
+ raise snapshot
if snapshot is None:
self.last_batch_generation_stats[row] = session.take_stats(row)
yield row, snapshot
@@ -7686,16 +8055,6 @@ class MLXInferenceBackend:
)
plans = [self._plan_vlm_request(request) for request in requests]
- batch = [
- GenerationRequest(
- **self._vlm_batch_prompt(plan.prompt),
- image = plan.images[0] if plan.images else None,
- max_tokens = plan.max_tokens,
- sampling = SamplingParams(**plan.sampling),
- logits_processors = plan.processors,
- )
- for plan in plans
- ]
defaults = GenerationDefaults(
max_tokens = max(plan.max_tokens for plan in plans),
prefill_batch_size = len(plans),
@@ -7707,13 +8066,26 @@ class MLXInferenceBackend:
self._generation_lock,
_temporary_mlx_adapter_state(self._model, _adapter_state),
self._int8_prefill_scope(),
+ ExitStack() as clips,
):
- if any(plan.images for plan in plans):
+ batch = [
+ GenerationRequest(
+ **self._vlm_batch_prompt(plan.prompt),
+ image = plan.images[0] if plan.images else None,
+ **row_clips,
+ max_tokens = plan.max_tokens,
+ sampling = SamplingParams(**plan.sampling),
+ logits_processors = plan.processors,
+ )
+ for plan, row_clips in zip(plans, _batch_clips(clips, plans, self._processor))
+ ]
+ if any(plan.images or plan.videos for plan in plans):
self._release_vlm_snapshots()
logger.info(
- "Generating %d vision replies as one batch: images=%d, max_tokens=%s, model=%s",
+ "Generating %d vision replies as one batch: images=%d, videos=%d, max_tokens=%s, model=%s",
len(plans),
sum(1 for plan in plans if plan.images),
+ sum(len(plan.videos or ()) for plan in plans),
[plan.max_tokens for plan in plans],
type(self._model).__name__,
)
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 65e602f5f44..964abfd0c15 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -1391,7 +1391,7 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
rows: Optional[list] = None,
- video_b64: Optional[str] = None,
+ videos_b64: Optional[list[str]] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
diffusion_frames: bool = False,
@@ -1427,8 +1427,8 @@ class InferenceOrchestrator:
if response_format is not None:
cmd["response_format"] = response_format
cmd["reasoning_is_extracted"] = bool(reasoning_is_extracted)
- if video_b64:
- cmd["video_base64"] = video_b64
+ if videos_b64:
+ cmd["videos_base64"] = videos_b64
if use_adapter is not None:
cmd["use_adapter"] = use_adapter
if tools is not None:
@@ -1699,7 +1699,7 @@ class InferenceOrchestrator:
stop: Optional[list] = None,
rows: Optional[list] = None,
expected_model: Optional[str] = None,
- video: Optional[str] = None,
+ videos: Optional[list[str]] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
diffusion_frames: bool = False,
@@ -1757,7 +1757,7 @@ class InferenceOrchestrator:
tool_protocol_active = tool_protocol_active,
seed = seed,
rows = rows,
- video_b64 = video,
+ videos_b64 = videos,
diffusion_frames = diffusion_frames,
prompt_ids = prompt_ids,
penalty_window = penalty_window,
@@ -3197,7 +3197,7 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
- video: Optional[str] = None,
+ videos: Optional[list[str]] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
diffusion_frames: bool = False,
@@ -3231,7 +3231,7 @@ class InferenceOrchestrator:
stop = stop,
response_format = response_format,
reasoning_is_extracted = reasoning_is_extracted,
- video = video,
+ videos = videos,
diffusion_frames = diffusion_frames,
)
@@ -3262,7 +3262,7 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
- video: Optional[str] = None,
+ videos: Optional[list[str]] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
) -> Generator[Any, None, None]:
@@ -3294,7 +3294,7 @@ class InferenceOrchestrator:
logit_bias = logit_bias,
stop = stop,
rows = rows,
- video = video,
+ videos = videos,
response_format = response_format,
reasoning_is_extracted = reasoning_is_extracted,
)
@@ -3598,7 +3598,7 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
rows: Optional[list] = None,
- video: Optional[str] = None,
+ videos: Optional[list[str]] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
diffusion_frames: bool = False,
@@ -3667,7 +3667,7 @@ class InferenceOrchestrator:
stop = stop,
rows = rows,
expected_model = expected_model,
- video = video,
+ videos = videos,
response_format = response_format,
reasoning_is_extracted = reasoning_is_extracted,
diffusion_frames = diffusion_frames,
@@ -3715,7 +3715,7 @@ class InferenceOrchestrator:
tool_protocol_active = tool_protocol_active,
seed = seed,
rows = rows,
- video_b64 = video,
+ videos_b64 = videos,
diffusion_frames = diffusion_frames,
prompt_ids = prompt_ids,
penalty_window = penalty_window,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index ea05f4fd72c..f15e63cbba6 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1154,10 +1154,10 @@ def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:
):
if gated in cmd and _backend_declares(backend, gated):
gen_kwargs[gated] = cmd[gated]
- if cmd.get("video_base64"):
- if not _backend_declares(backend, "video"):
+ if cmd.get("videos_base64"):
+ if not _backend_declares(backend, "videos"):
raise RuntimeError("The loaded backend does not read video.")
- gen_kwargs["video"] = cmd["video_base64"]
+ gen_kwargs["videos"] = cmd["videos_base64"]
# Only a raw completion carries these; its batch rows are planned from them.
for raw in ("prompt_ids", "penalty_window", "grammar", "n_probs"):
if raw in cmd:
@@ -1615,13 +1615,31 @@ class _ResidentBatch:
return
try:
for handle, snapshot in self.session.step(waiting):
- self._report(handle, snapshot)
+ if not isinstance(snapshot, BaseException):
+ self._report(handle, snapshot)
+ elif not self._fail_row(handle[0], snapshot):
+ return
except Exception as exc:
logger.error("Batched generation error: %s", exc, exc_info = True)
self._fail_all(exc)
return
self._close_if_empty()
+ def _fail_row(self, request_id: str, exc: BaseException) -> bool:
+ """Answer a request whose own decode failed, the batch decoding on. False where it gave up the batch."""
+ if request_id not in self._owed:
+ return True
+ logger.error("A batched reply failed: %s", exc, exc_info = exc)
+ stack = "".join(traceback.format_exception(exc, limit = 20))
+ taken_back = self._forget(request_id, withdraw = True)
+ # _forget already dropped it from _owed, so _fail_all would not answer it.
+ _send_response(
+ self.resp_queue, {**_generation_error_payload(request_id, exc), "stack": stack}
+ )
+ if not taken_back:
+ self._fail_all(exc)
+ return taken_back
+
def _fail_all(self, exc: BaseException) -> None:
"""Give up on the batch: every request in it is owed an error, not silence."""
stack = traceback.format_exc(limit = 20)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 2c857832e35..d05de4044a1 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -26686,36 +26686,31 @@ _VIDEO_INPUT_REFUSAL = (
)
-def _local_video_clip(payload, model_info) -> str:
- """Serves a video_url part the same as the legacy field; refuses a model with no video input."""
+def _local_video_clips(payload, model_info) -> list[str]:
+ """Serves video_url parts the same as the legacy field, in request order; refuses a model with
+ no video input."""
if not model_info.get("has_video_input"):
raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)
clips = _request_video_clips(payload)
if not clips:
raise HTTPException(status_code = 400, detail = "Could not read the provided video file.")
- # Generation takes one video kwarg; extra clips would be silently dropped.
- if len(clips) > 1:
- raise HTTPException(
- status_code = 400,
- detail = "Only one video is supported per request on this model.",
- )
# MLX would move an older-turn clip onto the newest turn.
if _video_is_on_an_older_turn(payload):
raise HTTPException(
status_code = 400,
detail = "A video must be attached to the latest message on this model.",
)
- # Refused everywhere, so the message must not send the caller to GGUF for it.
- if _is_remote_video(clips[0]):
- raise HTTPException(status_code = _REMOTE_VIDEO_REFUSAL[0], detail = _REMOTE_VIDEO_REFUSAL[1])
- # _request_video_rejection may not have run; reject unsupported schemes here too.
- scheme_rejection = _video_scheme_rejection(clips[0])
- if scheme_rejection is not None:
- raise HTTPException(status_code = scheme_rejection[0], detail = scheme_rejection[1])
- video_b64, rejection = _video_b64_rejection(clips[0])
+ # The pre-switch validation may not have run; count, scheme and size are refused here too.
+ rejection = _request_video_rejection(payload)
if rejection is not None:
raise HTTPException(status_code = rejection[0], detail = rejection[1])
- return video_b64
+ served = []
+ for clip in clips:
+ video_b64, rejection = _video_b64_rejection(clip)
+ if rejection is not None:
+ raise HTTPException(status_code = rejection[0], detail = rejection[1])
+ served.append(video_b64)
+ return served
def _video_payload_chars(clip: str) -> int:
@@ -30666,9 +30661,9 @@ async def produce_openai_chat_completions(
model_name = _orchestrator_public_model_id(backend) or payload.model
model_info = backend.models.get(backend.active_model_name, {})
# Before the speech and audio-input dispatches, which return before the clip is attached.
- _video_clip = None
+ _video_clips: list[str] = []
if _request_has_video(payload):
- _video_clip = _local_video_clip(payload, model_info)
+ _video_clips = _local_video_clips(payload, model_info)
# Settled here: a model without audio input never enters the audio-input path.
if payload.audio_base64:
raise HTTPException(status_code = 400, detail = _AUDIO_VIDEO_INPUT_DETAIL)
@@ -33464,7 +33459,7 @@ async def produce_openai_chat_completions(
_sf_image_tpl = (
(_sf_model_info.get("chat_template_info") or {}).get("processor_template")
# A clip renders through the processor as an image does.
- if _sf_has_any_image or _video_clip is not None
+ if _sf_has_any_image or _video_clips
else None
)
# Differs from processor_template: a template-less processor still places the image.
@@ -33531,7 +33526,7 @@ async def produce_openai_chat_completions(
and (not _sf_has_any_image or (bool(_sf_model_info.get("is_vision")) and not payload.tools))
# A caller's several images and a video clip stay out of the loop, as on main.
and not images
- and _video_clip is None
+ and not _video_clips
and not _sf_is_gptoss
and _sf_tool_budget > 0
)
@@ -33616,7 +33611,7 @@ async def produce_openai_chat_completions(
request,
include_api_key = True,
template_default = False,
- image = _sf_has_any_image or _video_clip is not None,
+ image = _sf_has_any_image or bool(_video_clips),
tools = True,
controls = _date_controls,
)
@@ -34135,8 +34130,8 @@ async def produce_openai_chat_completions(
stop = normalized_stop,
response_format = _extract_response_format(payload),
)
- if _video_clip is not None:
- gen_kwargs["video"] = _video_clip
+ if _video_clips:
+ gen_kwargs["videos"] = _video_clips
# The worker/template wrapper peels off kwargs the template doesn't accept.
if payload.enable_thinking is not None:
gen_kwargs["enable_thinking"] = payload.enable_thinking
@@ -34170,7 +34165,7 @@ async def produce_openai_chat_completions(
# Gate on _sf_use_tools, not mcp_enabled: an empty MCP registry must not drop client tools.
_sf_client_tools = (
# Read the resolved value: recomputing would undo the launcher-default withdrawal above.
- (not _sf_tools_on or ((_sf_has_any_image or _video_clip is not None) and not _sf_use_tools))
+ (not _sf_tools_on or ((_sf_has_any_image or bool(_video_clips)) and not _sf_use_tools))
and not _sf_use_tools
and not _sf_is_gptoss
and _sf_supports_tools
@@ -34248,7 +34243,7 @@ async def produce_openai_chat_completions(
_sf_client_system_prompt = _apply_current_date_prompt(
_user_system_prompt,
request,
- image = _sf_has_any_image or _video_clip is not None,
+ image = _sf_has_any_image or bool(_video_clips),
tools = bool(_sf_client_catalog),
controls = _date_controls,
)
@@ -34381,7 +34376,7 @@ async def produce_openai_chat_completions(
fitted_kwargs, events = gen_kwargs, ()
# The count cannot price pictures or video, so those prompts are left alone.
if _sf_fit_overflow and all(
- gen_kwargs.get(key) is None for key in ("image", "images", "video")
+ gen_kwargs.get(key) is None for key in ("image", "images", "videos")
):
fitted = backend.compact_chat_context(
gen_kwargs.get("messages") or [],
@@ -34828,14 +34823,14 @@ async def produce_openai_chat_completions(
# or the reverse scan attaches the picture to it (#10092).
# The fitted prompt: the retry extends it and is not refitted.
_nudge_base = _sf_fit_request()[0]["messages"]
- if _sf_renders_image or _video_clip is not None:
+ if _sf_renders_image or _video_clips:
from core.inference.chat_template_helpers import (
messages_with_attached_image as _nudge_attach,
)
_nudge_base = _nudge_attach(
_nudge_base,
image = _sf_renders_image,
- video = _video_clip is not None,
+ video = len(_video_clips),
)
retry_messages = [
*_nudge_base,
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 7d324651b06..7a4f9924d6b 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -2696,6 +2696,166 @@ def test_mlx_reasoning_tool_turn_keeps_provenance_across_buffered_segments(monke
assert [call["function"]["name"] for call in calls] == ["terminal"]
+def _decode_alone_and_batched(
+ monkeypatch,
+ turn,
+ markers,
+ cancel_after = None,
+ prompt = "prompt",
+ **request,
+):
+ """*turn* answered by the single path and by a text batch row: every snapshot each emits."""
+ import threading
+
+ _install_fake_mlx(monkeypatch)
+ from core.inference import mlx_inference
+
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda *_args, **_kwargs: prompt,
+ )
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.render_with_native_template_fallback",
+ lambda formatted_prompt, **_kwargs: SimpleNamespace(
+ prompt = formatted_prompt, reasoning_channel_markers = markers
+ ),
+ )
+ rows, last = {}, "stop" if turn._ends == "stop" else "length"
+
+ def step():
+ events = [(uid, ids.pop(0)) for uid, ids in rows.items() if ids]
+ return [], [
+ SimpleNamespace(uid = uid, token = token, finish_reason = None if rows[uid] else last)
+ for uid, token in events
+ ]
+
+ generator = SimpleNamespace(
+ insert = lambda *_a, **_k: [rows.setdefault(len(rows), list(turn.ids)) and len(rows) - 1],
+ next = step,
+ remove = lambda uids, **_k: [rows[uid].clear() for uid in uids] and {},
+ close = lambda: None,
+ )
+ for name, attrs in (
+ ("mlx_lm", {"stream_generate": lambda *_a, **_k: turn.stream()}),
+ (
+ "mlx_lm.sample_utils",
+ {"make_sampler": lambda **_k: None, "make_logits_processors": None},
+ ),
+ ("mlx_lm.generate", {"BatchGenerator": lambda *_a, **_k: generator}),
+ ):
+ module = types.ModuleType(name)
+ module.__dict__.update(attrs)
+ monkeypatch.setitem(sys.modules, name, module)
+
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model, backend._tokenizer, backend._is_vlm = object(), turn, False
+ request = {"messages": [{"role": "user", "content": "run it"}], "max_new_tokens": 99, **request}
+ cancel, alone = threading.Event(), []
+ for snapshot in backend.generate_chat_response(**request, cancel_event = cancel):
+ alone.append(snapshot)
+ if len(alone) == cancel_after:
+ cancel.set()
+
+ backend._batch_prompt_row = lambda _prompt, _state: ([], [0], None, None, [0])
+ session = mlx_inference._TextBatchSession(backend, width = 2)
+ # Two rows decoding together, so no state a row keeps can be shared or misrouted.
+ batched = {row: [prefix] if (prefix := session.admit(request, row)) else [] for row in (0, 1)}
+ while session.rows_in_flight:
+ cut = cancel_after is not None and len(batched[0]) >= cancel_after
+ events = session.withdraw([0, 1]) if cut else session.step()
+ for row, snapshot in events:
+ if snapshot is not None:
+ batched[row].append(snapshot)
+ session.close()
+ return alone, batched
+
+
+_TOOL = [{"type": "function", "function": {"name": "terminal"}}]
+_CHANNELS = ("<|channel>thought", "<channel|>")
+_GEMMA_SPECIALS = ("<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>", "<pad>", "<eos>")
+_GEMMA_TOOL_TURN = (
+ *("<|channel>", "thought", "\n", "weighing", "<pad>", "▁it", "<channel|>"),
+ *("<|tool_call>", "call:terminal{command:id}", "<tool_call|>", "<eos>"),
+)
+
+
+ "markers, request_, last, ends, cancel_after",
+ [
+ (_CHANNELS, {"tools": _TOOL}, "<eos>", "stop", None),
+ (None, {"tools": _TOOL}, "<eos>", "stop", None),
+ (None, {"tools": [], "tool_protocol_active": True}, "<eos>", "stop", None),
+ # Stopping on the call's closer keeps it; stopping on an opener closes nothing.
+ (None, {"tools": _TOOL}, "<tool_call|>", "stop", None),
+ (None, {"tools": _TOOL}, "<|tool_call>", "stop", None),
+ (_CHANNELS, {"tools": _TOOL}, "call:terminal{command:id}", "length", None),
+ (_CHANNELS, {"tools": _TOOL, "stop": ["{command"]}, "<eos>", "stop", None),
+ # Text held to the end, its tail a stop sequence's first characters.
+ (_CHANNELS, {"tools": _TOOL, "stop": ["<tool_call|>!"]}, "<eos>", "stop", None),
+ (_CHANNELS, {"tools": _TOOL}, "<eos>", "stop", 3),
+ (_CHANNELS, {}, "<eos>", "stop", None),
+ ],
+)
+def test_a_batched_tool_row_decodes_what_the_reply_alone_does(
+ monkeypatch, markers, request_, last, ends, cancel_after
+):
+ pieces = _GEMMA_TOOL_TURN[: _GEMMA_TOOL_TURN.index(last) + 1]
+ specials = tuple(special for special in _GEMMA_SPECIALS if special in pieces)
+ eos = last if ends == "stop" else None
+ # Built before the fake mlx modules land, so the real detokenizer still imports.
+ turn = _SpmTurn(pieces, specials = specials, eos = eos, chat_template = "x", ends = ends)
+ alone, batched = _decode_alone_and_batched(monkeypatch, turn, markers, cancel_after, **request_)
+ assert alone and batched == {0: alone, 1: alone}
+
+
+ "markers, request_, allows_reasoning",
+ [
+ (_CHANNELS, {"tools": _TOOL}, False),
+ (_CHANNELS, {"tools": _TOOL, "reasoning_is_extracted": True}, True),
+ (None, {"tools": _TOOL}, True),
+ (None, {}, False),
+ ],
+)
+def test_a_batched_constrained_row_is_planned_as_the_reply_alone_is(
+ monkeypatch, markers, request_, allows_reasoning
+):
+ from core.inference import mlx_inference
+
+ turn = _SpmTurn(
+ _GEMMA_TOOL_TURN, specials = _GEMMA_SPECIALS, eos = "<eos>", chat_template = "x", ends = "stop"
+ )
+ seen, decodes, sampling = [], [], mlx_inference._mlx_sampling_processors
+ decode = turn.decode
+ monkeypatch.setattr(turn, "decode", lambda ids, **kw: decodes.append(kw) or decode(ids, **kw))
+ # A kept </think> is what tells the grammar a closer survives the decode.
+ monkeypatch.setattr(mlx_inference, "decoder_preserves_token", lambda *_a: True)
+ monkeypatch.setattr(
+ mlx_inference,
+ "_build_grammar_constraint",
+ lambda response_format, *_a, **kw: response_format
+ and SimpleNamespace(
+ allows_reasoning = allows_reasoning,
+ decoded_dropping = lambda ids: seen.append((_a, sorted(kw.items()), ids)),
+ ),
+ )
+ monkeypatch.setattr(
+ mlx_inference,
+ "_mlx_sampling_processors",
+ lambda **kw: seen.append(kw["grammar_constraint"] is not None) or sampling(**kw),
+ )
+ request_ = {**request_, "response_format": {"type": "json_object"}}
+ # The template opens <think>, which a document-only reply must not re-emit.
+ alone, batched = _decode_alone_and_batched(
+ monkeypatch, turn, markers, prompt = "P<think>", **request_
+ )
+ assert alone and batched == {0: alone, 1: alone}
+ third = len(seen) // 3
+ assert seen and seen == seen[:third] * 3 and True in seen
+ plain = [kw for kw in decodes if kw.get("skip_special_tokens")]
+ assert all(kw["clean_up_tokenization_spaces"] is False for kw in plain)
+
+
def test_mlx_text_normalizes_native_reasoning_and_close_releases_lock(monkeypatch):
from core.inference.mlx_inference import MLXInferenceBackend
@@ -3293,7 +3453,7 @@ def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypat
backend._make_row_cache = lambda: ["cache"]
backend._batch_prompt_row = lambda prompt, _state: ([], [9], ["text cache"], "key", [9])
plan = SimpleNamespace(
- prompt = "P", max_tokens = 4, sampler = None, processors = (), detokenizes = False, think_prefix = ""
+ prompt = "P", max_tokens = 4, sampler = None, processors = (), streamed = None, think_prefix = ""
)
backend._plan_text_row = lambda messages, **_kw: rendered.append(messages) or plan
with monkeypatch.context() as patch:
@@ -3313,6 +3473,80 @@ def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypat
session.close()
[email protected]("ends", [None, "length", "stop"])
+def test_a_row_whose_own_processor_fails_leaves_and_its_neighbour_decodes_on(monkeypatch, ends):
+ import numpy
+
+ from core.inference import mlx_inference
+
+ rows, removed, steps = {}, [], []
+
+ def insert(_prompts, logits_processors, **_kwargs):
+ rows[len(rows)] = logits_processors[0]
+ return [len(rows) - 1]
+
+ def step():
+ steps.append(None)
+ for processors in rows.values():
+ for processor in processors:
+ processor(numpy.array([1]), "logits")
+ # The healthy row runs out of budget every third step, so a swallowed failure ends too.
+ healthy = "length" if len(steps) % 3 == 0 else None
+ return [], [
+ SimpleNamespace(uid = uid, token = 5, finish_reason = healthy if uid else ends)
+ for uid in rows
+ ]
+
+ generator = SimpleNamespace(
+ insert = insert,
+ next = step,
+ remove = lambda uids, **_k: [removed.append(uid) or rows.pop(uid) for uid in uids] and {},
+ close = lambda: None,
+ )
+ generate = types.ModuleType("mlx_lm.generate")
+ generate.BatchGenerator = lambda *_a, **_k: generator
+ monkeypatch.setitem(sys.modules, "mlx_lm.generate", generate)
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])
+ backend._is_vlm = False
+ backend._batch_prompt_row = lambda _prompt, _state: ([], [9], None, None, [9])
+
+ def failing(_tokens, _logits):
+ raise ValueError("schema")
+
+ plans = iter([failing, None])
+ backend._plan_text_row = lambda _messages, **_kw: SimpleNamespace(
+ prompt = "P",
+ think_prefix = "",
+ stream = mlx_inference._RowStream(decode = lambda ids: "x" * len(ids)),
+ streamed = None,
+ sampler = None,
+ processors = [processor] if (processor := next(plans)) else [],
+ max_tokens = 8,
+ )
+ session = mlx_inference._TextBatchSession(backend, width = 2)
+ session.admit({"messages": []}, "bad")
+ session.admit({"messages": []}, "good")
+ (handle, failure), *events = session.step()
+ assert handle == "bad" and isinstance(failure, ValueError) and str(failure) == "schema"
+ # Its token was sampled unconstrained, and the single path raises before yielding it.
+ assert removed == [0] and session.handles == ["good"]
+ assert events == [("good", "x")]
+ assert list(session.step()) == [("good", "xx")]
+ session.close()
+ # A drafted batch drafts only for rows whose processors all read just the history.
+ penalty = mlx_inference._mlx_sampling_processors(presence_penalty = 1.5)
+ guarded = mlx_inference._failing_alone(None, [*penalty, lambda _tokens, logits: logits])
+ assert [p.history_only for p in guarded] == [True, False]
+ # The fixed multi-reply command fails as a single reply would.
+ plans = iter([failing, None])
+ backend.last_batch_generation_stats = [None, None]
+ with pytest.raises(ValueError, match = "schema"):
+ list(
+ backend._generate_session_batch([{"messages": []}] * 2, mlx_inference._TextBatchSession)
+ )
+
+
def test_mlx_text_without_lru_prompt_cache_prefills_the_full_prompt(monkeypatch):
from core.inference import mlx_inference
@@ -3620,14 +3854,16 @@ def test_mlx_batch_rows_plan_a_raw_completion_prompt_untemplated(monkeypatch):
monkeypatch.setattr(backend, "_render_text_prompt", lambda *_a, **_k: pytest.fail("templated"))
seen = {}
monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **kw: seen.update(kw))
+ monkeypatch.setattr(mlx_inference, "_raw_prompt_constraint", _RawConstraint)
row = mlx_inference._row_kwargs({"prompt_ids": [1, 2, 3], "penalty_window": 2})
- plan = backend._plan_text_row([], **row)
+ plan = backend._plan_text_row([], grammar = "G", **row)
assert (plan.prompt, plan.think_prefix, seen["penalty_prompt"], seen["penalty_window"]) == (
[1, 2, 3],
"",
[1, 2, 3],
2,
)
+ assert seen["grammar_constraint"].built == ("G", backend._tokenizer)
backend._make_row_cache = lambda: None
assert backend._batch_prompt_row(plan.prompt, None)[1] == [1, 2, 3]
kwargs = []
@@ -3641,7 +3877,6 @@ def test_mlx_batch_rows_plan_a_raw_completion_prompt_untemplated(monkeypatch):
vision = backend._plan_vlm_request({"prompt_ids": [4, 5], "messages": []})
assert (vision.prompt, vision.max_tokens) == ([4, 5], ([4, 5], [4, 5]))
assert backend._vlm_batch_prompt([4, 5]) == {"prompt_token_ids": [4, 5]}
- assert mlx_inference._request_batch_gap({"grammar": "G"})
_COMPLETION_TOK = SimpleNamespace(
@@ -3856,13 +4091,14 @@ def test_mlx_batch_releases_an_over_budget_cache_while_another_row_decodes(monke
uid = handle,
cache_key = handle,
all_tokens = [],
- plan = SimpleNamespace(stream = stream),
+ plan = SimpleNamespace(stream = stream, streamed = None),
ready_at = None,
processing_at = None,
prompt_len = 1,
generated = 1,
cached = 0,
reason = "stop",
+ failure = None,
)
for handle in banked
}
@@ -7010,7 +7246,7 @@ def _drive_vlm_generation(
image = None,
max_new_tokens = 1,
session = None,
- video = None,
+ videos = None,
on_stream = None,
**request,
):
@@ -7048,7 +7284,7 @@ def _drive_vlm_generation(
backend.generate_chat_response(
messages = [{"role": "user", "content": "hi"}],
image = image,
- video = video,
+ videos = videos,
max_new_tokens = max_new_tokens,
**request,
)
@@ -7119,7 +7355,7 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):
assert "draft_n" not in backend.last_generation_stats["timings"]
# Only the drafted reply entered the verify scope: open from preparing the draft through decoding.
assert len(prepared) == 1 and scopes == [["open"]] * 2 and verify_scope == ["open", "closed"]
- opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r: None
+ opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r, **_k: None
backend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)
assert backend.batch_unavailable_reason([{}, {}]) == "no stream"
monkeypatch.setattr(
@@ -7221,7 +7457,7 @@ def test_the_context_budget_counts_the_request_in_tokens_the_processor_would_pro
def test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monkeypatch):
- """A video is counted from a readable file, not the route's base64."""
+ """Every clip is counted from a readable file, not the route's base64."""
import base64
import os
@@ -7235,15 +7471,17 @@ def test_the_vlm_entry_point_hands_the_limit_a_clip_the_processor_can_read(monke
def _limit(*_a, **kwargs):
seen.update(kwargs)
seen["on_disk"] = [os.path.exists(clip) for clip in kwargs.get("videos") or ()]
+ seen["contents"] = [Path(clip).read_bytes() for clip in kwargs.get("videos") or ()]
return 1
monkeypatch.setattr(backend, "_generation_limit", _limit)
- clip = base64.b64encode(b"not a readable clip").decode()
+ clips = [base64.b64encode(b"not a readable clip").decode(), "BBBB"]
with contextlib.suppress(Exception):
- _drive_vlm_generation(backend, monkeypatch, video = clip)
- assert seen["videos"] != [clip]
- assert seen["on_disk"] == [True]
- assert not os.path.exists(seen["videos"][0])
+ _drive_vlm_generation(backend, monkeypatch, videos = clips)
+ assert seen["videos"] != clips
+ assert seen["on_disk"] == [True, True]
+ assert seen["contents"] == [base64.b64decode(clip) for clip in clips]
+ assert not any(os.path.exists(path) for path in seen["videos"])
def test_media_is_prepared_only_under_a_budget_and_only_once(monkeypatch):
@@ -8099,7 +8337,7 @@ def _open_vision_session(rows):
session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)
session._rows = {row.handle: row for row in rows}
session._by_row, session._settled = {row.row: row for row in rows}, {}
- session._speculative = None
+ session._speculative, session._decodes_ids = None, False
return session
@@ -8115,7 +8353,7 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto
[
mlx_inference._VisionBatchRow(
handle = handle,
- plan = SimpleNamespace(stream = _vlm_row(backend, stop = ["HALT"])),
+ plan = SimpleNamespace(stream = _vlm_row(backend, stop = ["HALT"]), streamed = None),
row = number,
)
for number, handle in enumerate(handles)
@@ -8148,6 +8386,32 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto
), "a row the engine has already retired is not handed back again"
+def test_a_resident_vision_tool_row_decodes_its_ids_and_keeps_the_stop_that_closes_its_call(
+ monkeypatch,
+):
+ from core.inference import mlx_inference
+
+ pieces = ("<|tool_call>", "call:terminal{command:id}", "<tool_call|>")
+ turn = _SpmTurn(
+ pieces, specials = pieces[::2], eos = "<tool_call|>", chat_template = "x", ends = "stop"
+ )
+ backend = _vlm_backend(monkeypatch, markers = None)
+ backend._tokenizer = backend._processor.tokenizer = turn
+ plan = backend._plan_vlm_row(
+ [{"role": "user", "content": "hi"}],
+ None,
+ **dict(temperature = 0, top_p = 1, top_k = 0, min_p = 0, max_new_tokens = 9),
+ repetition_penalty = 1.0,
+ tools = _TOOL,
+ decodes_ids = True,
+ )
+ session = _open_vision_session([mlx_inference._VisionBatchRow(handle = "r", plan = plan, row = 0)])
+ ended = SimpleNamespace(finish_reason = "stop", prompt_token_count = 3, token_ids = turn.ids[:2])
+ event = SimpleNamespace(index = 0, delta = "", token_ids = tuple(turn.ids), result = ended)
+ session.stream = SimpleNamespace(step = lambda: [event])
+ assert list(session.step())[-2:] == [("r", "".join(pieces)), ("r", None)]
+
+
def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatch):
engine = _batch_engine(monkeypatch)
monkeypatch.setattr(engine, "GenerationRequest", SimpleNamespace, raising = False)
@@ -8156,14 +8420,16 @@ def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypat
monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: True, raising = False)
backend = _vlm_backend(monkeypatch, markers = None)
backend._tokenizer.bos_token = "<s>"
- backend._plan_vlm_request = lambda request: SimpleNamespace(
+ backend._plan_vlm_request = lambda request, **_k: SimpleNamespace(
prompt = "<s>hi",
images = None,
+ videos = None,
max_tokens = 4,
sampling = {},
processors = None,
think_prefix = "",
stream = None,
+ streamed = None,
)
session = _open_vision_session([])
session.backend, session._adapter_state, session._resumes_rows = backend, None, False
@@ -8226,8 +8492,10 @@ def test_a_vision_row_times_its_prefill_wherever_the_batch_runs_it(monkeypatch,
processors = None,
think_prefix = "",
stream = _vlm_row(backend),
+ streamed = None,
+ speculates = True,
)
- backend._plan_vlm_request = lambda request: plan
+ backend._plan_vlm_request = lambda request, **_k: plan
backend._reads_vision = False
session = _open_vision_session([])
session.backend, session._adapter_state, session._resumes_rows = backend, None, False
@@ -8276,6 +8544,48 @@ def test_a_vision_row_times_its_prefill_wherever_the_batch_runs_it(monkeypatch,
assert timings["predicted_per_second"] == pytest.approx(10.0)
+def test_a_fixed_vision_batch_marks_each_row_s_clips_and_shares_their_decode_budget(monkeypatch):
+ import base64
+
+ from core.inference import mlx_inference
+
+ engine, backend = _batch_engine(monkeypatch), _vlm_backend(monkeypatch, markers = None)
+ for name in ("GenerationRequest", "GenerationDefaults"):
+ monkeypatch.setattr(engine, name, SimpleNamespace, raising = False)
+ monkeypatch.setattr(engine, "SamplingParams", dict, raising = False)
+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: False, raising = False)
+ too_large, refuse = (_ for _ in ()).throw, mlx_inference._ClipsTooLarge
+ fps = lambda p, proc, rows = 1: rows / 4 if rows < 3 else too_large(refuse())
+ monkeypatch.setattr(mlx_inference, "_video_sampling_fps", fps)
+ seen, paths = [], []
+ clips, other = ([base64.b64encode(data).decode()] for data in (b"one", b"two"))
+ backend._render_vlm_prompt = lambda messages, images, videos, **k: (
+ seen.append(([p["type"] for p in messages[-1]["content"]], videos)) or ("p", None, None)
+ )
+ backend._media_generation_limit = lambda p, n, ids, images, videos: seen.append(videos) or n
+ backend._kv_window_generate_kwargs, backend._generation_lock = dict, threading.Lock()
+ backend._release_vlm_snapshots = lambda: seen.append("freed")
+
+ def stream_batch(model, processor, batch, defaults):
+ seen.extend((row.fps, [Path(path).read_bytes() for path in row.videos]) for row in batch)
+ paths.extend(path for row in batch for path in row.videos)
+ return iter(())
+
+ monkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)
+ request = {"messages": [{"role": "user", "content": "hi"}], "videos": clips}
+ with pytest.raises(RuntimeError, match = "unfinished"):
+ list(backend._generate_vlm_batch([request, request, {**request, "videos": other}]))
+ planned = [(["video", "text"], clips), clips] * 2 + [(["video", "text"], other), other]
+ assert seen[:7] == planned + ["freed"]
+ rates = [(0.5, [b"one"])] * 2 + [(0.25, [b"two"])]
+ assert seen[7:] == rates and paths[0] == paths[1] != paths[2], "a group shares files and budget"
+ seen.clear()
+ with pytest.raises(RuntimeError, match = "unfinished"):
+ list(backend._generate_vlm_batch([request] * 3))
+ assert seen[-3:] == [(0.25, [b"one"])] * 3 and len(set(paths[3:])) == 3, "too large to share"
+ assert not any(map(os.path.exists, paths)), "removed when the batch ends"
+
+
def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(monkeypatch):
from core.inference import mlx_inference
@@ -8294,7 +8604,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
backend._vlm_is_diffusion_model = lambda model: False
backend._vlm_snapshot_scope = lambda state, images: (None, "m|img" if images else "m", list)
- backend._plan_vlm_request = lambda request: SimpleNamespace(
+ backend._plan_vlm_request = lambda request, **_k: SimpleNamespace(
prompt = "p",
images = request.get("images"),
max_tokens = 4,
@@ -8302,6 +8612,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
processors = None,
think_prefix = "",
stream = _vlm_row(backend),
+ streamed = None,
)
session = _open_vision_session([])
session.backend, session._adapter_state = backend, None
@@ -8380,15 +8691,12 @@ def test_a_text_load_whose_engine_cannot_batch_says_so_before_it_commits(monkeyp
(
{"images": [object()]},
{"images_base64": ["AAAA"]},
- {"video": "AAAAGGZ0eXBtcDQy"},
- {"video_base64": "AAAAGGZ0eXBtcDQy"},
- {"tools": [{"type": "function", "function": {"name": "lookup"}}]},
- {"tool_protocol_active": True},
- {"response_format": {"type": "json_object"}},
+ {"videos": ["AAAAGGZ0eXBtcDQy"]},
+ {"videos_base64": ["AAAAGGZ0eXBtcDQy"]},
),
)
def test_a_reply_the_batch_cannot_serve_decodes_alone(monkeypatch, extra):
- """Replayed tool images, a clip and the native tool protocol live only on the"""
+ """Replayed tool images and a clip live only on the single path."""
from core.inference.mlx_inference import MLXInferenceBackend
engine = _batch_engine(monkeypatch)
@@ -8404,6 +8712,55 @@ def test_a_reply_the_batch_cannot_serve_decodes_alone(monkeypatch, extra):
assert backend.batch_unavailable_reason([plain, {**plain, **extra}]) is not None
+ "extra, refusal",
+ (
+ ({"tools": [{"type": "function"}]}, "a reply runs the tool protocol"),
+ ({"tool_protocol_active": True}, "a reply runs the tool protocol"),
+ (
+ {"response_format": {"type": "json_object"}},
+ "a reply is constrained to a response format",
+ ),
+ ({"grammar": 'root ::= "a"'}, "a reply is constrained to a response format"),
+ ),
+)
+def test_a_tool_or_constrained_reply_joins_a_batch_that_decodes_it_from_its_ids(
+ monkeypatch, extra, refusal
+):
+ from dataclasses import make_dataclass
+
+ from core.inference import mlx_inference
+
+ engine = _batch_engine(monkeypatch)
+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)
+ monkeypatch.setattr(mlx_inference, "_batched_vision_engine_gap", lambda _engine: None)
+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)
+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False
+ assert backend.batch_unavailable_reason([{}, extra]) is None
+ backend._model, backend._is_vlm = SimpleNamespace(_is_vlm_model = True), True
+ backend._processor, backend._speculative_draft = object(), None
+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": None}, None
+ backend._turboquant = False
+ assert backend.resident_unavailable_reason({}) is None
+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)
+ monkeypatch.setattr(engine, "BatchStream", lambda *_a, **_k: None, raising = False)
+ for fields, draft, joins in (
+ (("token_ids",), None, True),
+ ((), None, False),
+ (("token_ids",), 1, False),
+ (("drafts",), 1, False),
+ (("token_ids", "drafts"), 1, True),
+ ):
+ for name in ("GenerationEvent", "GenerationRequest"):
+ monkeypatch.setattr(engine, name, make_dataclass(name, fields), raising = False)
+ backend._speculative_draft = draft
+ assert backend.resident_unavailable_reason(extra) == (None if joins else refusal)
+ assert mlx_inference._VisionBatchSession(backend, width = 2)._decodes_ids == joins
+ backend._speculative_draft = None # The fixed multi-reply command decodes the batch's text.
+ assert backend.batch_unavailable_reason([{}, extra]) == refusal
+ assert backend.resident_unavailable_reason({"response_format": {"type": "text"}}) is None
+
+
def test_a_quantized_or_budgeted_kv_cache_does_not_batch(monkeypatch):
from core.inference.mlx_inference import MLXInferenceBackend
@@ -8429,7 +8786,7 @@ def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):
backend._model, backend._processor, backend._is_vlm = object(), object(), True
backend._kv_quant, backend._kv_context_budget = {"kv_bits": 4}, 4096
backend._turboquant = False
- backend._vlm_batch_unavailable_reason = lambda requests: None
+ backend._vlm_batch_unavailable_reason = lambda requests, **_k: None
monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)
capable = lambda: None
monkeypatch.setattr(
@@ -8446,6 +8803,24 @@ def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):
assert backend.resident_unavailable_reason({}) == "old mlx-vlm"
[email protected]("gap", (None, "speculative", "old zoo", "no decoder", "no vision"))
+def test_a_video_reply_batches_only_as_a_fixed_command_of_a_load_that_reads_it(monkeypatch, gap):
+ from core.inference import mlx_inference
+
+ engine, backend = _batch_engine(monkeypatch), _vlm_backend(monkeypatch, markers = None)
+ backend._speculative_draft = gap == "speculative" or None
+ backend._reads_vision, available = gap != "no vision", lambda request: None
+ backend._vlm_batch_unavailable_reason = backend._vlm_resident_unavailable_reason = available
+ monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: gap != "no decoder")
+ fields = SimpleNamespace(__dataclass_fields__ = ("image", "videos")[: 1 + (gap != "old zoo")])
+ monkeypatch.setattr(engine, "GenerationRequest", fields, raising = False)
+ clip, sent = {"videos": [_CLIP_B64]}, {"videos_base64": [_CLIP_B64]}
+ for row in (clip, sent):
+ assert (backend.batch_unavailable_reason([{}, row]) is None) == (gap is None)
+ assert backend.resident_unavailable_reason(sent) == "a reply carries a video clip"
+ assert backend.resident_unavailable_reason({}) is None
+
+
def test_a_turboquant_vision_batch_quantizes_rows_as_the_single_path_does(monkeypatch):
from core.inference import mlx_inference
@@ -10147,7 +10522,7 @@ def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch)
{"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}
]
args = (turn, None, 0, 1, 0, 0, 1, 1, None)
- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]
((_, stream_kwargs, _),) = streams
assert "prompt_cache" not in stream_kwargs
assert "prompt_cache_state" not in stream_kwargs
@@ -10176,7 +10551,7 @@ def test_mlx_vlm_a_video_turn_releases_retained_snapshots(monkeypatch):
]
args = (turn, None, 0, 1, 0, 0, 1, 1, None)
- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]
assert released == [True]
@@ -10197,7 +10572,7 @@ def test_mlx_vlm_an_unset_budget_is_capped_for_a_video_turn(monkeypatch):
]
args = (turn, None, 0, 1, 0, 0, None, 1, None)
- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]
assert streams[-1][1]["max_tokens"] == UNSET_GENERATION_BUDGET
@@ -10212,13 +10587,13 @@ def test_mlx_vlm_video_clip_lives_on_disk_only_for_the_stream(monkeypatch):
]
args = (turn, None, 0, 1, 0, 0, 1, 1, None)
- assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]
((stream_args, stream_kwargs, contents),) = streams
assert stream_args[3] is None, "no image was attached"
assert contents == [base64.b64decode(_CLIP_B64)]
assert not any(Path(p).exists() for p in stream_kwargs["video"])
- gen = backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)
+ gen = backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])
assert next(gen) == "ok"
(path,) = streams[-1][1]["video"]
assert Path(path).exists()
@@ -10226,6 +10601,45 @@ def test_mlx_vlm_video_clip_lives_on_disk_only_for_the_stream(monkeypatch):
assert not Path(path).exists()
+def test_mlx_vlm_every_clip_reaches_mlx_vlm_in_order_at_one_rate(monkeypatch):
+ """Releases before 0.7.3 apply one ``fps`` to every clip, so the slowest affordable rate
+ is the one that keeps every clip inside its share of the decode budget."""
+ import base64
+
+ from core.inference import mlx_inference
+
+ streams = []
+ backend = _video_vlm_backend(monkeypatch, streams)
+ # The slowest clip sits between faster ones, so neither the first nor the last rate passes.
+ rates = {b"fast clip": 2.0, b"slow clip": 0.5, b"medium clip": 1.0}
+ shares = []
+
+ def _rate(
+ path,
+ _processor,
+ clips = 1,
+ ):
+ shares.append(clips)
+ return rates[Path(path).read_bytes()]
+
+ monkeypatch.setattr(mlx_inference, "_video_frame_rate", _rate)
+ clips = [base64.b64encode(content).decode() for content in rates]
+ turn = [
+ {
+ "role": "user",
+ "content": [{"type": "video"}] * 3 + [{"type": "text", "text": "which"}],
+ }
+ ]
+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)
+
+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = clips)) == ["ok"]
+ ((_, stream_kwargs, contents),) = streams
+ assert contents == list(rates)
+ assert stream_kwargs["fps"] == 0.5
+ assert shares == [3, 3, 3]
+ assert not any(Path(p).exists() for p in stream_kwargs["video"])
+
+
def _tiny_clip(
path,
width = 32,
@@ -10264,6 +10678,11 @@ def test_mlx_vlm_the_decoded_frame_stack_is_bounded(monkeypatch, tmp_path):
monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 4)
assert mlx_inference._video_frame_rate(clip, plain) == pytest.approx(4 / 3)
+ # mlx-vlm holds every decoded clip at once, so two clips split the budget.
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 8)
+ assert mlx_inference._video_frame_rate(clip, plain) == 2.0
+ assert mlx_inference._video_sampling_fps([clip, clip], plain) == pytest.approx(4 / 3)
+
monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 3)
with pytest.raises(RuntimeError, match = "32x24 frames are too large"):
mlx_inference._video_frame_rate(clip, plain)
@@ -10350,14 +10769,14 @@ def test_mlx_vlm_the_frame_rate_rides_the_stream(monkeypatch, tmp_path):
assert list(
backend._generate_vlm(
- turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64
+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [clip_b64]
)
) == ["ok"]
assert streams[0][1]["fps"] == pytest.approx(4 / 3)
monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 3)
gen = backend._generate_vlm(
- turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64
+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [clip_b64]
)
with pytest.raises(RuntimeError, match = "too large to decode"):
next(gen)
@@ -10376,7 +10795,7 @@ def test_mlx_vlm_a_video_turn_whose_render_is_unusable_is_refused_not_recovered(
with pytest.raises(RuntimeError, match = "for a video turn"):
list(
backend._generate_vlm(
- turn, [object()], 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = _CLIP_B64
+ turn, [object()], 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [_CLIP_B64]
)
)
@@ -10387,11 +10806,11 @@ def test_mlx_vlm_an_undecodable_clip_leaves_no_file_behind(monkeypatch, tmp_path
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
backend = _video_vlm_backend(monkeypatch, [])
- turn = [{"role": "user", "content": [{"type": "video"}]}]
+ turn = [{"role": "user", "content": [{"type": "video"}, {"type": "video"}]}]
with pytest.raises(binascii.Error):
list(
backend._generate_vlm(
- turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = "A"
+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, videos = [_CLIP_B64, "A"]
)
)
assert list(tmp_path.iterdir()) == []
@@ -10402,7 +10821,7 @@ def test_mlx_vlm_structured_video_items_must_match_attached_clips(monkeypatch):
args = (None, 0, 1, 0, 0, 1, 1, None)
without_part = [{"role": "user", "content": [{"type": "text", "text": "what moves"}]}]
with pytest.raises(RuntimeError, match = "0 structured video item.*1 attached video"):
- list(backend._generate_vlm(without_part, *args, _adapter_state = False, video = _CLIP_B64))
+ list(backend._generate_vlm(without_part, *args, _adapter_state = False, videos = [_CLIP_B64]))
with_part = [{"role": "user", "content": [{"type": "video"}]}]
with pytest.raises(RuntimeError, match = "1 structured video item.*0 attached video"):
list(backend._generate_vlm(with_part, *args, _adapter_state = False))
@@ -10422,21 +10841,35 @@ def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_cli
captured.append((messages, kwargs)) or iter(())
)
list(
- backend.generate_chat_response([{"role": "user", "content": "what moves"}], video = _CLIP_B64)
+ backend.generate_chat_response(
+ [{"role": "user", "content": "what moves"}], videos = [_CLIP_B64]
+ )
)
messages, kwargs = captured[0]
assert messages[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]
- assert kwargs["video"] == _CLIP_B64
+ assert kwargs["videos"] == [_CLIP_B64]
+ list(
+ backend.generate_chat_response(
+ [{"role": "user", "content": "what moves"}], videos = [_CLIP_B64, "BBBB"]
+ )
+ )
+ messages, kwargs = captured[1]
+ assert messages[-1]["content"][:2] == [{"type": "video"}, {"type": "video"}]
+ assert kwargs["videos"] == [_CLIP_B64, "BBBB"]
backend._reads_vision = False
with pytest.raises(RuntimeError, match = "loaded model does not read video"):
- list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))
+ list(
+ backend.generate_chat_response([{"role": "user", "content": "hi"}], videos = [_CLIP_B64])
+ )
backend._is_vlm = True
backend._reads_vision = True
monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)
with pytest.raises(RuntimeError, match = "installed mlx-vlm does not read video"):
- list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))
+ list(
+ backend.generate_chat_response([{"role": "user", "content": "hi"}], videos = [_CLIP_B64])
+ )
def test_mlx_vlm_decodes_video_only_with_its_clip_decoder(monkeypatch):
@@ -10517,6 +10950,61 @@ def test_mlx_reads_video_rejects_a_clip_rendered_as_prose(monkeypatch):
assert _mlx_reads_video(unnamed) is True
+def _same_template_processor(template = "T", drops_video = True):
+ """A processor that would render, with its tokenizer holding ``template``. ``drops_video`` is
+ mlx-vlm's Gemma 4 processor, which loses a bare video part its tokenizer renders as written."""
+ return SimpleNamespace(
+ apply_chat_template = lambda *_a, **_k: "unused",
+ chat_template = "T",
+ video_processor = object(),
+ video_token = "<|video|>",
+ drops_video = drops_video,
+ tokenizer = SimpleNamespace(chat_template = template),
+ )
+
+
+def _render_by_target(monkeypatch):
+ def _render(target, messages, **_kwargs):
+ has_video = any(
+ isinstance(part, dict) and part.get("type") == "video"
+ for message in messages
+ for part in (message.get("content") or [])
+ )
+ marked = has_video and not getattr(target, "drops_video", False)
+ return "<|video|> hi" if marked else "hi"
+
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation", _render
+ )
+
+
+def test_a_video_turn_renders_through_the_tokenizer_only_when_the_processor_drops_it(monkeypatch):
+ from core.inference.mlx_inference import MLXInferenceBackend, _mlx_reads_video
+
+ _require_video_stack()
+ _render_by_target(monkeypatch)
+ drops = _same_template_processor()
+ assert _mlx_reads_video(drops) is True
+ # A template of the processor's own is what it must render, as before.
+ assert _mlx_reads_video(_same_template_processor(template = "other")) is False
+
+ backend = MLXInferenceBackend()
+ video_turn = [{"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "hi"}]}]
+ image_turn = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]}]
+ backend._processor = drops
+ prompt, target, _ = backend._render_vlm_prompt(video_turn, None, videos = ["clip.mp4"])
+ assert (prompt, target) == ("<|video|> hi", drops.tokenizer)
+ _, target, _ = backend._render_vlm_prompt(image_turn, [object()])
+ assert target is drops
+
+ # Some processors deliberately differ from their tokenizer, so one that keeps the marker renders.
+ keeps = _same_template_processor(drops_video = False)
+ backend._processor = keeps
+ prompt, target, _ = backend._render_vlm_prompt(video_turn, None, videos = ["clip.mp4"])
+ assert (prompt, target) == ("<|video|> hi", keeps)
+ assert _mlx_reads_video(keeps) is True
+
+
def _sampling_release(monkeypatch, **utils_fields):
"""Stand mlx_vlm.utils up in one release's shape, so one processor can be read on each."""
mlx_vlm = _require_video_stack_module()
@@ -11279,3 +11767,30 @@ def test_mlx_precision_folder_loads_from_its_own_download(monkeypatch, tmp_path)
assert calls[-1][0] == (repo,)
assert backend.models[repo]["gguf_variant"] is None
assert len(fetched) == 1
+
+
+def test_a_batch_row_without_an_independent_detokenizer_is_refused_not_crashed():
+ from core.inference import mlx_inference
+ from core.inference.worker import RowRefused
+
+ decoder = SimpleNamespace(suppresses = lambda _id: False)
+ mlx_inference._require_row_detokenizer(None)
+ mlx_inference._require_row_detokenizer(
+ mlx_inference._MlxStreamedText(SimpleNamespace(), decoder, None, ())
+ )
+ with pytest.raises(RowRefused, match = "detokenizer"):
+ mlx_inference._require_row_detokenizer(
+ mlx_inference._MlxStreamedText(None, decoder, None, ())
+ )
+
+
+def test_a_flushed_tail_releases_its_text_before_a_stop_it_completes():
+ from core.inference.mlx_inference import _RowStream
+
+ stream = _RowStream(sequences = ("HALT",))
+ assert stream.feed(text = "already ") == ["already "]
+ assert stream.flush("remaining HALT") == ["already remaining "]
+ assert stream.stopped and stream.settle() == []
+ plain = _RowStream(sequences = ("HALT",))
+ plain.feed(text = "all ")
+ assert plain.flush("of it") == ["all of it"] and plain.settle() == []
diff --git a/studio/backend/tests/test_mlx_resident_batch.py b/studio/backend/tests/test_mlx_resident_batch.py
index 6a89df1c63b..855dddfe711 100644
--- a/studio/backend/tests/test_mlx_resident_batch.py
+++ b/studio/backend/tests/test_mlx_resident_batch.py
@@ -161,6 +161,54 @@ def test_replies_a_command_asked_for_together_are_addressed_and_counted_by_row()
assert resp.sent[-1]["stats"] is None
+def test_a_reply_whose_own_decode_fails_is_answered_alone_and_the_batch_decodes_on():
+ batch, resp = _batch()
+ batch.admit(_cmd("r1", rows = [{"seed": 1}, {"seed": 2}], parallel_slots = 4), None)
+ batch.admit(_cmd("r2", parallel_slots = 4), None)
+ batch.admit(_cmd("r3", parallel_slots = 4), None)
+ session = batch.session
+
+ def step(_waiting):
+ session.rows.remove(("r1", 0))
+ session.rows.remove(("r3", None))
+ yield ("r1", 0), ValueError("schema")
+ yield ("r3", None), KeyError("bind")
+ yield ("r2", None), "b"
+ # The failed reply's sibling ending in the same step does not answer it as done.
+ yield ("r1", 1), None
+
+ session.step = step
+ batch.step()
+ sent = [(m["type"], m["request_id"], m.get("error")) for m in resp.sent]
+ assert sent == [
+ ("gen_error", "r1", "schema"),
+ ("gen_error", "r3", "'bind'"),
+ ("token", "r2", None),
+ ]
+ assert "ValueError" in resp.sent[0]["stack"] and "KeyError" in resp.sent[1]["stack"]
+ assert session.rows == [("r2", None)] and not session.closed
+
+
+def test_a_failed_reply_whose_siblings_cannot_be_withdrawn_is_still_answered():
+ batch, resp = _batch()
+ batch.admit(_cmd("r1", rows = [{"seed": 1}, {"seed": 2}], parallel_slots = 4), None)
+ batch.admit(_cmd("r2", parallel_slots = 4), None)
+ session = batch.session
+
+ def step(_waiting):
+ session.rows.remove(("r1", 0))
+ yield ("r1", 0), ValueError("schema")
+
+ def withdraw(_handles):
+ raise RuntimeError("metal")
+ yield
+
+ session.step, session.withdraw = step, withdraw
+ batch.step()
+ errors = {m["request_id"]: m["error"] for m in resp.sent if m["type"] == "gen_error"}
+ assert errors == {"r1": "schema", "r2": "schema"} and batch.session is None
+
+
def test_raw_completion_rows_join_the_batch_and_decode_alone_through_the_completion_path():
batch, _ = _batch()
cmd = _cmd(
diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.py
index fa3eeeeb815..0a53e32e312 100644
--- a/studio/backend/tests/test_presence_penalty.py
+++ b/studio/backend/tests/test_presence_penalty.py
@@ -351,7 +351,7 @@ def test_the_mlx_vlm_prefill_predicate_matches_its_decoder_gate():
def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_it():
- """The clip rides the command like an image; a backend without ``video`` fails the request."""
+ """Clips ride the command like images; a backend without ``videos`` fails the request."""
from pathlib import Path
from core.inference.orchestrator import InferenceOrchestrator, _mirrored_model_entry
@@ -360,11 +360,11 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_i
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
o.models, o.active_model_name = {}, None
o._stop_ledger = o._pending_teardowns = None
- assert "video_base64" not in o._build_generate_cmd("r", None, messages = [])
+ assert "videos_base64" not in o._build_generate_cmd("r", None, messages = [])
cmd = o._build_generate_cmd(
- "r", None, messages = [{"role": "user", "content": "hi"}], video_b64 = "AAAA"
+ "r", None, messages = [{"role": "user", "content": "hi"}], videos_b64 = ["AAAA", "BBBB"]
)
- assert cmd["video_base64"] == "AAAA"
+ assert cmd["videos_base64"] == ["AAAA", "BBBB"]
class _FakeQueue:
def __init__(self):
@@ -379,10 +379,10 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_i
def generate_chat_response(
self,
- video = None,
+ videos = None,
**kwargs,
):
- self.received = video
+ self.received = videos
return iter(())
class _TextBackend:
@@ -395,7 +395,7 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_i
takes = _VideoBackend()
_handle_generate(takes, cmd, _FakeQueue(), threading.Event())
- assert takes.received == "AAAA"
+ assert takes.received == ["AAAA", "BBBB"]
refuses, queue = _TextBackend(), _FakeQueue()
_handle_generate(refuses, cmd, queue, threading.Event())
assert refuses.called is False
@@ -440,10 +440,10 @@ def test_both_orchestrator_entry_points_forward_the_clip_into_the_command():
o._build_generate_cmd = _build
turn = [{"role": "user", "content": "hi"}]
with pytest.raises(_Built):
- list(o.generate_chat_response(messages = turn, video = "AAAA"))
+ list(o.generate_chat_response(messages = turn, videos = ["AAAA"]))
with pytest.raises(_Built):
- list(o.generate_with_adapter_control(use_adapter = True, messages = turn, video = "AAAA"))
- assert [kw["video_b64"] for kw in built] == ["AAAA", "AAAA"]
+ list(o.generate_with_adapter_control(use_adapter = True, messages = turn, videos = ["AAAA"]))
+ assert [kw["videos_b64"] for kw in built] == [["AAAA"], ["AAAA"]]
def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_serves_it():
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index a251a7ab8fb..1c7a491c982 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -2268,7 +2268,7 @@ def test_video_turn_with_tools_enabled_keeps_the_client_tool_catalog(monkeypatch
body = _json_body(_call(payload, monkeypatch, backend))
assert backend.calls[0]["tools"] == [LOOKUP_TOOL]
- assert backend.calls[0]["video"] == clip
+ assert backend.calls[0]["videos"] == [clip]
assert body["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "lookup"
@@ -2299,7 +2299,7 @@ def test_a_nudge_retry_keeps_the_video_on_the_question_turn(monkeypatch):
assert len(backend.calls) == 2, "the nudge retry did not run"
retry = backend.calls[1]["messages"]
- assert backend.calls[1]["video"] == clip
+ assert backend.calls[1]["videos"] == [clip]
question = next(m for m in retry if m["role"] == "user")
assert question["content"][0] == {"type": "video"}
assert retry[-1]["role"] == "user" and isinstance(retry[-1]["content"], str)
diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.py
index f7d72db5ea5..ca886ae9cbc 100644
--- a/studio/backend/tests/test_video_attachment_part.py
+++ b/studio/backend/tests/test_video_attachment_part.py
@@ -162,19 +162,19 @@ def _model_info(**fields):
def test_a_video_backend_is_handed_the_bare_clip():
from fastapi import HTTPException
from models.inference import ChatCompletionRequest
- from routes.inference import _VIDEO_INPUT_REFUSAL, _local_video_clip
+ from routes.inference import _VIDEO_INPUT_REFUSAL, _local_video_clips
payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)
- assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64
+ assert _local_video_clips(payload, _model_info(has_video_input = True)) == [_CLIP_B64]
payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _CLIP_B64)
- assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64
+ assert _local_video_clips(payload, _model_info(has_video_input = True)) == [_CLIP_B64]
with pytest.raises(HTTPException) as exc:
- _local_video_clip(payload, _model_info())
+ _local_video_clips(payload, _model_info())
assert exc.value.status_code == 400 and exc.value.detail == _VIDEO_INPUT_REFUSAL
payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = "data:video/mp4;base64,")
with pytest.raises(HTTPException) as exc:
- _local_video_clip(payload, _model_info(has_video_input = True))
+ _local_video_clips(payload, _model_info(has_video_input = True))
assert exc.value.status_code == 400
@@ -182,15 +182,15 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera
"""The gate precedes the early-returning dispatches, and the clip rides the generation kwargs."""
source = _inference_source()
handler = source.index("using_gguf = llama_backend.is_loaded")
- gate = source.index("_video_clip = _local_video_clip(payload, model_info)", handler)
+ gate = source.index("_video_clips = _local_video_clips(payload, model_info)", handler)
speech = source.index("return await _monitored_generate_audio(model_name)", handler)
audio_input = source.index("# ── Audio INPUT path", handler)
assert gate < speech and gate < audio_input
- assert 'gen_kwargs["video"] = _video_clip' in source
+ assert 'gen_kwargs["videos"] = _video_clips' in source
use_tools = source.index("_sf_use_tools = (", handler)
tools_block = source[use_tools : source.index("\n )", use_tools)]
tools_block = "\n".join(line.split("#")[0] for line in tools_block.splitlines())
- assert "and _video_clip is None" in tools_block
+ assert "and not _video_clips" in tools_block
# Structural, not literal: #10970 widened the image half to `_sf_has_image`.
client_tools = source.index("_sf_client_tools = (", handler)
block = source[client_tools : source.index("\n )", client_tools)]
@@ -200,7 +200,7 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera
assert "and not _sf_use_tools" in escape, escape
# Image and clip must be ALTERNATIVES, each read positively, in either order.
image = r"(?:\bimage is not None\b|\b_sf_has_(?:any_)?image\b)"
- clip = r"\b_video_clip is not None\b"
+ clip = r"\bbool\(_video_clips\)"
assert re.search(rf"{image}\s+or\s+{clip}|{clip}\s+or\s+{image}", escape), escape
assert not re.search(r"\bimage is None\b|\bnot\s+_sf_has_(?:any_)?image\b", escape), escape
# Settled at the gate: a model without audio input never enters the audio-input path.
@@ -208,7 +208,9 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera
assert conflict < speech
assert "_AUDIO_VIDEO_INPUT_DETAIL" in source[conflict : conflict + 200]
# The gate reads both spellings, so a video_url part reaches MLX exactly as the field does.
- assert "if _request_has_video(payload):\n _video_clip = _local_video_clip(" in source
+ assert (
+ "if _request_has_video(payload):\n _video_clips = _local_video_clips(" in source
+ )
def test_token_counting_refuses_video_like_image_and_audio():
@@ -222,10 +224,10 @@ def test_token_counting_refuses_video_like_image_and_audio():
def test_both_video_checks_share_one_rule():
- """Two size checks that drift let the pre-switch one pass what the post-load
- one refuses, which is the model load this was meant to avoid."""
+ """Size checks that drift let the pre-switch one pass what a post-load one
+ refuses, which is the model load this was meant to avoid."""
source = _inference_source()
- assert source.count("= _request_video_rejection(payload)") == 2
+ assert source.count("= _request_video_rejection(payload)") == 3
def _video_url_request(
diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.py
index 49d77c6453d..dcd4fe1661f 100644
--- a/studio/backend/tests/test_video_chat_completions_behaviour.py
+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py
@@ -238,7 +238,7 @@ def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(mon
def _gate(payload, model_info):
reached.append(model_info)
- return _CLIP_B64
+ return [_CLIP_B64]
monkeypatch.setattr(
inference_route,
@@ -248,7 +248,7 @@ def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(mon
models = {"mlx-model": {"is_vision": True, "has_video_input": True}},
),
)
- monkeypatch.setattr(inference_route, "_local_video_clip", _gate)
+ monkeypatch.setattr(inference_route, "_local_video_clips", _gate)
with _client(monkeypatch, None) as client:
response = client.post("/v1/chat/completions", json = body)
@@ -259,15 +259,15 @@ def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(mon
def test_the_non_gguf_gate_reads_both_spellings(monkeypatch):
- """_local_video_clip predates the part, so left reading payload.video_base64 alone it would
+ """_local_video_clips predates the part, so left reading payload.video_base64 alone it would
hand an MLX model nothing while the legacy field worked."""
from models.inference import ChatCompletionRequest
info = {"is_vision": True, "has_video_input": True}
field = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)
part = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
- assert inference_route._local_video_clip(field, info) == _CLIP_B64
- assert inference_route._local_video_clip(part, info) == _CLIP_B64
+ assert inference_route._local_video_clips(field, info) == [_CLIP_B64]
+ assert inference_route._local_video_clips(part, info) == [_CLIP_B64]
def test_a_non_gguf_backend_without_video_refuses_by_name():
@@ -276,34 +276,42 @@ def test_a_non_gguf_backend_without_video_refuses_by_name():
payload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
with pytest.raises(HTTPException) as exc:
- inference_route._local_video_clip(payload, {"is_vision": True})
+ inference_route._local_video_clips(payload, {"is_vision": True})
assert exc.value.status_code == 400
assert exc.value.detail == inference_route._VIDEO_INPUT_REFUSAL
assert "MLX" in inference_route._VIDEO_INPUT_REFUSAL
-def test_a_second_clip_is_refused_rather_than_dropped_on_a_non_gguf_backend():
- """Generation takes one video kwarg. Taking clips[0] silently changed the prompt by
- backend: GGUF forwards every clip, so the same request meant two different things."""
+def test_every_clip_is_served_in_request_order_on_a_non_gguf_backend():
+ """GGUF forwards every clip, so the non-GGUF path must too rather than serve one; the
+ request-wide count and size caps still apply, since the pre-switch check may not have run."""
from fastapi import HTTPException
from models.inference import ChatCompletionRequest
info = {"is_vision": True, "has_video_input": True}
- two = ChatCompletionRequest.model_validate(_part_body(_DATA_URI, _DATA_URI))
- with pytest.raises(HTTPException) as exc:
- inference_route._local_video_clip(two, info)
- assert exc.value.status_code == 400
- assert "Only one video" in exc.value.detail
+ second = "BBBBGGZ0eXBtcDQy"
+ two = ChatCompletionRequest.model_validate(
+ _part_body(_DATA_URI, f"data:video/mp4;base64,{second}")
+ )
+ assert inference_route._local_video_clips(two, info) == [_CLIP_B64, second]
both = ChatCompletionRequest.model_validate(
- _part_body(_DATA_URI, **{"video_base64": _DATA_URI})
+ _part_body(f"data:video/mp4;base64,{second}", **{"video_base64": _DATA_URI})
+ )
+ assert inference_route._local_video_clips(both, info) == [_CLIP_B64, second]
+
+ too_many = ChatCompletionRequest.model_validate(
+ _part_body(*[_DATA_URI] * (inference_route._MAX_VIDEO_CLIPS_PER_REQUEST + 1))
)
with pytest.raises(HTTPException) as exc:
- inference_route._local_video_clip(both, info)
- assert exc.value.status_code == 400
+ inference_route._local_video_clips(too_many, info)
+ assert exc.value.status_code == 400 and "Too many videos" in exc.value.detail
- one = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
- assert inference_route._local_video_clip(one, info) == _CLIP_B64
+ half = "data:video/mp4;base64," + "A" * (inference_route._MAX_VIDEO_B64_CHARS // 2 + 10)
+ oversized = ChatCompletionRequest.model_validate(_part_body(half, half))
+ with pytest.raises(HTTPException) as exc:
+ inference_route._local_video_clips(oversized, info)
+ assert exc.value.status_code == 413
def test_an_unsupported_scheme_is_refused_on_the_non_gguf_path_too():
@@ -314,7 +322,7 @@ def test_an_unsupported_scheme_is_refused_on_the_non_gguf_path_too():
payload = ChatCompletionRequest.model_validate(_part_body("file:///etc/passwd"))
with pytest.raises(HTTPException) as exc:
- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ inference_route._local_video_clips(payload, {"is_vision": True, "has_video_input": True})
assert exc.value.status_code == 400
assert "Unsupported video URL scheme" in exc.value.detail
@@ -352,7 +360,7 @@ def test_a_remote_clip_is_refused_on_a_non_gguf_backend():
payload = ChatCompletionRequest.model_validate(_part_body(_REMOTE))
with pytest.raises(HTTPException) as exc:
- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ inference_route._local_video_clips(payload, {"is_vision": True, "has_video_input": True})
assert exc.value.status_code == 400
assert "Remote video URLs are not supported" in exc.value.detail
@@ -665,14 +673,14 @@ def _older_turn_body(clip = _DATA_URI):
def test_a_clip_on_an_older_turn_is_refused_on_a_non_gguf_backend():
"""GGUF keeps the part on the turn that carried it; the non-GGUF path flattens the parts and
- hands generation one video kwarg, which MLX attaches to the newest turn. Refusing keeps the
+ hands generation the clips alone, which MLX attaches to the newest turn. Refusing keeps the
two backends from reading the same request as different conversations."""
from fastapi import HTTPException
from models.inference import ChatCompletionRequest
payload = ChatCompletionRequest.model_validate(_older_turn_body())
with pytest.raises(HTTPException) as exc:
- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ inference_route._local_video_clips(payload, {"is_vision": True, "has_video_input": True})
assert exc.value.status_code == 400
assert "attached to the latest message" in exc.value.detail
@@ -680,10 +688,9 @@ def test_a_clip_on_an_older_turn_is_refused_on_a_non_gguf_backend():
def test_a_clip_on_the_latest_turn_is_still_served_on_a_non_gguf_backend():
from models.inference import ChatCompletionRequest
payload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
- assert (
- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
- == _CLIP_B64
- )
+ assert inference_route._local_video_clips(
+ payload, {"is_vision": True, "has_video_input": True}
+ ) == [_CLIP_B64]
def test_the_legacy_field_is_not_treated_as_an_older_turn():
@@ -692,10 +699,9 @@ def test_the_legacy_field_is_not_treated_as_an_older_turn():
payload = ChatCompletionRequest.model_validate(_field_body())
assert inference_route._video_is_on_an_older_turn(payload) is False
- assert (
- inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
- == _CLIP_B64
- )
+ assert inference_route._local_video_clips(
+ payload, {"is_vision": True, "has_video_input": True}
+ ) == [_CLIP_B64]
def test_gguf_still_keeps_a_clip_on_the_turn_that_carried_it(monkeypatch):
diff --git a/studio/backend/tests/test_video_pr9057_simulation.py b/studio/backend/tests/test_video_pr9057_simulation.py
index 965238eea59..2fc1a16985f 100644
--- a/studio/backend/tests/test_video_pr9057_simulation.py
+++ b/studio/backend/tests/test_video_pr9057_simulation.py
@@ -371,7 +371,7 @@ def _routes_source() -> str:
"needle",
[
"raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)",
- "_video_clip = _local_video_clip(payload, model_info)",
+ "_video_clips = _local_video_clips(payload, model_info)",
'if not getattr(llama_backend, "_has_video_input", False):',
'"Video input is not supported together with guided decoding or client-supplied tools yet."',
'"Cannot count tokens for messages containing video."',
diff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.py
index 850d12c5651..cc35722ccaa 100644
--- a/studio/backend/tests/test_vision_client_tools.py
+++ b/studio/backend/tests/test_vision_client_tools.py
@@ -891,7 +891,7 @@ def test_video_tool_support_is_classified_from_the_processor_template():
assert backend.calls, "generation never ran"
assert backend.calls[0]["tools"] == [passthrough.LOOKUP_TOOL]
- assert backend.calls[0]["video"] == clip
+ assert backend.calls[0]["videos"] == [clip]
def test_mlx_selects_structured_content_for_a_processor_render():