Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read a video clip on MLX chat models (#10480)

Long Yixing82fd97282128
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 828ecad5e91..57a88f352de 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -2826,23 +2826,34 @@ def last_user_text(messages: list) -> str:
return ""
-def count_structured_images(content) -> int:
- """Number of structured image parts in a message *content* (or a bare part)."""
+_STRUCTURED_IMAGE_TYPES = ("image", "image_url", "input_image")
+_STRUCTURED_VIDEO_TYPES = ("video", "video_url", "input_video")
+
+
+def _count_structured_parts(content, types) -> int:
if isinstance(content, list):
- return sum(count_structured_images(item) for item in content)
+ return sum(_count_structured_parts(item, types) for item in content)
if not isinstance(content, dict):
return 0
- if str(content.get("type", "")).lower() in ("image", "image_url", "input_image"):
+ if str(content.get("type", "")).lower() in types:
return 1
- return count_structured_images(content.get("content"))
+ return _count_structured_parts(content.get("content"), types)
+
+
+def count_structured_images(content) -> int:
+ return _count_structured_parts(content, _STRUCTURED_IMAGE_TYPES)
+
+
+def count_structured_videos(content) -> int:
+ return _count_structured_parts(content, _STRUCTURED_VIDEO_TYPES)
def structured_media_reprs(content) -> set:
- """Every spelling a template could print a structured image part as."""
+ media_types = _STRUCTURED_IMAGE_TYPES + _STRUCTURED_VIDEO_TYPES
if isinstance(content, list):
values = (
{str(content), json.dumps(content, ensure_ascii = False)}
- if count_structured_images(content)
+ if _count_structured_parts(content, media_types)
else set()
)
for item in content:
@@ -2850,7 +2861,7 @@ def structured_media_reprs(content) -> set:
return values
if not isinstance(content, dict):
return set()
- if str(content.get("type", "")).lower() in ("image", "image_url", "input_image"):
+ if str(content.get("type", "")).lower() in media_types:
return {str(content), json.dumps(content, ensure_ascii = False)}
return structured_media_reprs(content.get("content"))
@@ -2900,15 +2911,17 @@ def messages_with_attached_image(
system_prompt: str = "",
fallback_user_text: str = "",
structured_content: bool = False,
+ image: bool = True,
+ video: bool = False,
) -> list:
- """The conversation to render for a turn that carries an attached image.
+ """The conversation to render for a turn that carries attached media.
- Prepends *system_prompt* as a leading system turn, then injects an ``{"type": "image"}`` part
- into the LAST user turn and leaves every other turn -- assistant ``tool_calls`` and
- ``role="tool"`` results included -- exactly as the caller sent it. Rebuilding from the newest
- user TEXT instead dropped the folded system instruction and the tool history an OpenAI tool loop
- replays (#10092). Nothing the caller owns is mutated: callers still read those dicts after
- generation, and a retry re-renders the same list.
+ Prepends *system_prompt* as a leading system turn, then injects an ``{"type": "image"}`` or
+ ``{"type": "video"}`` part into the LAST user turn and leaves every other turn -- assistant
+ ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.
+ Rebuilding from the newest user TEXT instead dropped the folded system instruction and the
+ tool history an OpenAI tool loop replays (#10092). Nothing the caller owns is mutated: callers
+ still read those dicts after generation, and a retry re-renders the same list.
*structured_content* wraps content in part lists, as a processor template expects; MLX may
render through the nested text tokenizer, whose template expects a string. *fallback_user_text*
@@ -2943,13 +2956,20 @@ def messages_with_attached_image(
),
},
)
- # Once: a reverse scan would mark a nudge retry's correction, not the question.
- if any(
- isinstance(m, dict)
- and isinstance(m.get("content"), list)
- and count_structured_images(m["content"])
- for m in conversation
- ):
+ # Once per medium: a reverse scan would mark a nudge retry's correction, not the question.
+ parts = [
+ {"type": part_type}
+ for part_type, wanted, counter in (
+ ("image", image, count_structured_images),
+ ("video", video, count_structured_videos),
+ )
+ if wanted
+ and not any(
+ isinstance(m, dict) and isinstance(m.get("content"), list) and counter(m["content"])
+ for m in conversation
+ )
+ ]
+ if not parts:
return conversation
for index in range(len(conversation) - 1, -1, -1):
message = conversation[index]
@@ -2957,21 +2977,14 @@ def messages_with_attached_image(
continue
content = message.get("content", "")
if isinstance(content, str):
- parts = [{"type": "image"}, {"type": "text", "text": content or fallback_user_text}]
- elif isinstance(content, list):
- parts = list(content)
- if not count_structured_images(parts):
- parts.insert(0, {"type": "image"})
- else:
+ content = [{"type": "text", "text": content or fallback_user_text}]
+ elif not isinstance(content, list):
break
- conversation[index] = {**message, "content": parts}
+ conversation[index] = {**message, "content": parts + list(content)}
return conversation
if fallback_user_text:
conversation.append(
- {
- "role": "user",
- "content": [{"type": "image"}, {"type": "text", "text": fallback_user_text}],
- }
+ {"role": "user", "content": parts + [{"type": "text", "text": fallback_user_text}]}
)
return conversation
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 5c807669598..ad58408e5a0 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -29,6 +29,7 @@ from core.inference.runtime_context import (
from core.inference.chat_template_helpers import (
# Aliased to this module's historic names; bodies moved to the shared helper (#10092).
count_structured_images as _count_vlm_images,
+ count_structured_videos as _count_vlm_videos,
detect_reasoning_channel_markers,
make_reasoning_normalizer,
markup_for_tokenizer,
@@ -817,6 +818,166 @@ _IMAGE_PROBE_MESSAGES = [
{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]}
]
_TEXT_PROBE_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
+_VIDEO_PROBE_MESSAGES = [
+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "hi"}]}
+]
+
+
+def _mlx_vlm_decodes_video() -> bool:
+ """Both halves of the decoder, ``load_video`` (0.5.0+) and cv2, so a clip missing either is
+ refused by name rather than by a ModuleNotFoundError mid-stream."""
+ try:
+ from mlx_vlm import utils as vlm_utils
+ except ImportError:
+ return False
+ if not callable(getattr(vlm_utils, "load_video", None)):
+ return False
+ try:
+ import cv2 # noqa: F401
+ except ImportError:
+ return False
+ return True
+
+
+def _video_placeholder(processor):
+ """The model's own video placeholder, when it names one."""
+ for source in (processor, getattr(processor, "tokenizer", None)):
+ token = getattr(source, "video_token", None)
+ if isinstance(token, str) and token:
+ return token
+ return None
+
+
+def _mlx_reads_video(processor) -> bool:
+ """Processor carries a video component AND the template places a video marker: a render that
+ merely differs from the text one can still be prose, as in the image probe."""
+ if processor is None or getattr(processor, "video_processor", None) is None:
+ return False
+ if not _mlx_vlm_decodes_video():
+ return False
+ from core.inference.chat_template_helpers import (
+ apply_chat_template_for_generation,
+ chat_render_target,
+ )
+
+ try:
+ target = chat_render_target(processor)
+ marked = apply_chat_template_for_generation(target, _VIDEO_PROBE_MESSAGES)
+ if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):
+ return False
+ if _vlm_prompt_issue(marked, _VIDEO_PROBE_MESSAGES):
+ return False
+ placeholder = _video_placeholder(processor)
+ return placeholder is None or placeholder in marked
+ except BaseException:
+ # A load must never fail on a probe, matching _image_marker_survives.
+ return False
+
+
+def _write_video_clip(video_b64: str) -> str:
+ import base64
+ import tempfile
+
+ data = base64.b64decode(video_b64)
+ with tempfile.NamedTemporaryFile(prefix = "unsloth-video-", delete = False) as handle:
+ handle.write(data)
+ return handle.name
+
+
+# mlx-vlm holds two native-resolution RGB copies of the sampled frames before the processor resizes.
+_VIDEO_DECODE_BUDGET_BYTES = 2 << 30
+
+
+# load_video's own defaults, unchanged 0.5.0 to 0.7.0, for a release that settles a knob nowhere.
+_VIDEO_SAMPLING_FALLBACK = {"fps": 2.0, "min_frames": 4}
+
+
+def _sampled(name, value):
+ return _VIDEO_SAMPLING_FALLBACK[name] if value is None else value
+
+
+def _declared_video_sampling(processor) -> dict:
+ """Read the way mlx-vlm's ``processor_video_sampling`` reads it, so both branches agree."""
+ component = getattr(processor, "video_processor", None)
+ if component is None:
+ return {}
+ for owner in (component, processor):
+ hook = getattr(owner, "video_sampling_defaults", None)
+ if callable(hook):
+ declared = hook()
+ return dict(declared) if isinstance(declared, dict) else vars(declared)
+ return {name: getattr(component, name, None) for name in ("fps", "min_frames", "max_frames")}
+
+
+def _video_sampling_target(processor) -> tuple[float, int, Optional[int]]:
+ """mlx-vlm's (fps, min_frames, nframes); older releases pass only ``fps`` to ``load_video``."""
+ import inspect
+
+ from mlx_vlm import utils as vlm_utils
+
+ resolve = getattr(vlm_utils, "resolve_video_sampling", None)
+ if resolve is not None:
+ sampling = resolve(processor, {})
+ # Unset is unreachable today; the fallback keeps the rate divisible for a future release.
+ return (
+ _sampled("fps", sampling.fps),
+ _sampled("min_frames", sampling.min_frames),
+ sampling.nframes,
+ )
+ # Below 0.7.0, where install_python_stack.py pins it, prepare_inputs forwards only ``fps``,
+ # so nothing else applies the model's rate. ``min_frames`` stays the count really decoded.
+ defaults = inspect.signature(vlm_utils.load_video).parameters
+
+ def _default(name):
+ parameter = defaults.get(name)
+ if parameter is None or parameter.default is inspect.Parameter.empty:
+ return None
+ return parameter.default
+
+ declared_fps = _declared_video_sampling(processor).get("fps")
+ return (
+ _sampled("fps", _default("fps") if declared_fps is None else declared_fps),
+ _sampled("min_frames", _default("min_frames")),
+ None,
+ )
+
+
+def _video_frame_rate(clip_path: str, processor) -> Optional[float]:
+ """The sampling ``fps`` keeping the decoded stack within the budget; None when the clip has no
+ readable frame geometry or count, which mlx-vlm then refuses itself."""
+ import cv2
+
+ capture = cv2.VideoCapture(clip_path)
+ try:
+ if not capture.isOpened():
+ return None
+ width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+ height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ total_frames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+ native_fps = capture.get(cv2.CAP_PROP_FPS)
+ finally:
+ capture.release()
+ if min(width, height, total_frames) <= 0:
+ return None
+ if not native_fps or native_fps <= 0:
+ # The one measurement mlx-vlm substitutes rather than refusing; match its substitution.
+ native_fps = 1.0
+ wanted_fps, min_frames, nframes = _video_sampling_target(processor)
+ affordable = _VIDEO_DECODE_BUDGET_BYTES // (2 * 3 * width * height)
+ needed = min_frames if nframes is None else nframes
+ if affordable < needed:
+ raise RuntimeError(
+ f"The video's {width}x{height} frames are too large to decode: {needed} of them "
+ f"exceed the {_VIDEO_DECODE_BUDGET_BYTES >> 20} MiB frame budget."
+ )
+ return min(wanted_fps, affordable * native_fps / total_frames)
+
+
+def _discard_video_clip(path: str) -> None:
+ try:
+ os.remove(path)
+ except OSError:
+ pass
def _classify_mlx_audio_type(
@@ -2117,11 +2278,16 @@ class MLXInferenceBackend:
adapter_state,
images = None,
prompt = None,
+ has_video = False,
):
store = self._vlm_prompt_cache_store()
if store is None or self._vlm_is_diffusion_model(self._model):
# Diffusion generation never consults the prefix hook.
return None
+ if has_video:
+ # Keyed by token ids the clip does not vary: a later clip would resume the first
+ # one's mRoPE grid.
+ return None
media_ids = self._vlm_media_token_ids(getattr(self._model, "config", None))
if images and not media_ids:
# Neither side can tell where the image's rows end without them.
@@ -2527,6 +2693,7 @@ class MLXInferenceBackend:
"is_audio": _audio_type is not None and _audio_type != "audio_vlm",
"audio_type": _audio_type,
"has_audio_input": is_audio_input_type(_audio_type),
+ "has_video_input": is_vision and _mlx_reads_video(self._processor),
"context_length": _served_ctx,
# Parity with llama.cpp's requested_n_ctx: the served window cannot say whether anything was asked for,
# and that decides reuse and "pinned".
@@ -2796,15 +2963,21 @@ class MLXInferenceBackend:
# Unrestricted mode runs the tool protocol with an EMPTY tools list, so bool(tools)
# cannot tell that the wrappers below still have to survive decoding.
tool_protocol_active = None,
+ video = None,
) -> Generator[str, None, None]:
if self._model is None:
raise RuntimeError("No model loaded")
+ if video is not None:
+ if not _mlx_vlm_decodes_video():
+ raise RuntimeError("The installed mlx-vlm does not read video.")
+ if not self._is_vlm:
+ raise RuntimeError("The loaded model does not read video.")
# Reset so a failed run cannot surface stale stats.
self.last_generation_stats = None
# Shared with the transformers vision path so both render the same turns (#10092).
- if self._is_vlm and image is not None:
+ if self._is_vlm and (image is not None or video is not None):
# Processor templates want part lists, the tokenizer fallback wants strings.
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
@@ -2817,6 +2990,8 @@ class MLXInferenceBackend:
messages,
system_prompt = system_prompt,
structured_content = _renders_via_processor,
+ image = image is not None,
+ video = video is not None,
)
else:
full_messages = self._with_system_prompt(messages, system_prompt)
@@ -2844,6 +3019,7 @@ class MLXInferenceBackend:
_adapter_state = _adapter_state,
stop = stop,
tool_protocol_active = tool_protocol_active,
+ video = video,
)
else:
stream = self._generate_text(
@@ -3160,6 +3336,7 @@ class MLXInferenceBackend:
messages,
images,
*,
+ videos = None,
tools = None,
enable_thinking = None,
reasoning_effort = None,
@@ -3190,6 +3367,17 @@ class MLXInferenceBackend:
f"VLM conversation contains {structured_images} structured image "
f"item(s) for {attached_images} attached image(s)."
)
+ attached_videos = 0 if videos is None else len(videos)
+ structured_videos = sum(
+ _count_vlm_videos(message.get("content"))
+ for message in messages
+ if isinstance(message, dict)
+ )
+ if structured_videos != attached_videos:
+ raise RuntimeError(
+ f"VLM conversation contains {structured_videos} structured video "
+ f"item(s) for {attached_videos} attached video(s)."
+ )
prompt = None
has_tool_history = _vlm_messages_have_tool_history(messages)
prompt_error = None
@@ -3215,6 +3403,12 @@ class MLXInferenceBackend:
f"VLM chat template returned {prompt_issue} and cannot be recovered "
"without dropping tool-call history."
) from prompt_error
+ if prompt_issue and attached_videos:
+ # mlx-vlm's registered renderers take image and audio counts only.
+ raise RuntimeError(
+ f"VLM chat template returned {prompt_issue} and cannot be recovered "
+ "for a video turn."
+ ) from prompt_error
if images is not None and prompt_issue:
if tools or any(
@@ -3286,6 +3480,7 @@ class MLXInferenceBackend:
_adapter_state = None,
tool_protocol_active = None,
stop = None,
+ video = None,
):
from mlx_vlm import stream_generate as vlm_stream
@@ -3293,6 +3488,7 @@ class MLXInferenceBackend:
prompt, chat_target = self._render_vlm_prompt(
messages,
images,
+ videos = [video] if video is not None else None,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
@@ -3323,14 +3519,15 @@ class MLXInferenceBackend:
stopped = False
if max_new_tokens is None:
max_new_tokens = self._unset_generation_budget(prompt)
- if image is not None:
- # An image expands past its one placeholder token, so the counted prompt is short
- # of the real one: cap at the default, but stay under the rotating cache window.
+ if image is not None or video is not None:
+ # Media expands past its placeholder (a clip by a frame each), so the counted
+ # prompt is short: cap at the default, under the cache window.
max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)
logger.info(
- "VLM generating: prompt_len=%d, has_image=%s",
+ "VLM generating: prompt_len=%d, has_image=%s, has_video=%s",
len(prompt),
image is not None,
+ video is not None,
)
# stream_generate forwards **kwargs into generate_step (which builds the sampler + logits_processors
# internally). GOTCHA: generate_step expects temperature= (long form); temp= is silently ignored, stuck at
@@ -3395,7 +3592,9 @@ class MLXInferenceBackend:
else ()
)
- session = self._vlm_prompt_cache_session(_adapter_state, images, prompt)
+ session = self._vlm_prompt_cache_session(
+ _adapter_state, images, prompt, has_video = video is not None
+ )
if session is not None:
vlm_kwargs["prompt_cache"] = session.cache
vlm_kwargs["prompt_cache_state"] = session
@@ -3417,13 +3616,21 @@ class MLXInferenceBackend:
ExitStack() as generation_scope,
session_scope,
):
- if images and session is None:
+ if session is None and (images or video is not None):
# The vision pass gets the headroom, under the lock so nothing refills it.
+ # A video turn resumes no snapshot, so retained ones are pure occupancy.
self._release_vlm_snapshots()
generation_scope.enter_context(_mlx_fused_moe_gate_up(self._model))
generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))
final_response = None
+ clip_path = None
try:
+ if video is not None:
+ clip_path = _write_video_clip(video)
+ vlm_kwargs["video"] = [clip_path]
+ frame_rate = _video_frame_rate(clip_path, self._processor)
+ if frame_rate is not None:
+ vlm_kwargs["fps"] = frame_rate
# Emit any prefilled <think> block before the first token so the UI renders it during prefill,
# matching _generate_text. Done inside the adapter context so an unsupported request raises before
# any output escapes.
@@ -3466,6 +3673,8 @@ class MLXInferenceBackend:
if sequences and not stopped and released < len(sampled):
yield prefill + sampled
finally:
+ if clip_path is not None:
+ _discard_video_clip(clip_path)
cached_n = produced_n = 0
produced_s = 0.0
if session is not None:
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 17e3ae930bb..eda2749ee59 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -212,6 +212,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"is_audio": model_info.get("is_audio", False),
"audio_type": model_info.get("audio_type"),
"has_audio_input": model_info.get("has_audio_input", False),
+ "has_video_input": model_info.get("has_video_input", False),
"context_length": model_info.get("context_length"),
"native_context_length": model_info.get("native_context_length"),
"max_context_length": model_info.get("max_context_length"),
@@ -985,6 +986,7 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
+ video_b64: Optional[str] = None,
) -> dict:
"""Build the 'generate' command shared by the locked and dispatched paths."""
cmd = {
@@ -1007,6 +1009,8 @@ class InferenceOrchestrator:
cmd["seed"] = seed
if stop:
cmd["stop"] = stop
+ if video_b64:
+ cmd["video_base64"] = video_b64
if use_adapter is not None:
cmd["use_adapter"] = use_adapter
if tools is not None:
@@ -1212,6 +1216,7 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
+ video: Optional[str] = None,
) -> Generator[str, None, None]:
"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-request
mailbox for tokens so two compare-mode requests can be queued at once. The subprocess
@@ -1274,6 +1279,7 @@ class InferenceOrchestrator:
continue_final_message = continue_final_message,
tool_protocol_active = tool_protocol_active,
seed = seed,
+ video_b64 = video,
)
mailbox: queue.Queue = queue.Queue()
@@ -2041,6 +2047,7 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
+ video: Optional[str] = None,
) -> Generator[str, None, None]:
"""Generate response, streaming tokens from subprocess. ``tools`` / ``enable_thinking`` /
``reasoning_effort`` / ``preserve_thinking`` are forwarded so the template can render
@@ -2072,6 +2079,7 @@ class InferenceOrchestrator:
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
stop = stop,
+ video = video,
)
def generate_chat_completion_with_tools(
@@ -2293,6 +2301,7 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
+ video: Optional[str] = None,
) -> Generator[str, None, None]:
"""Inner generation logic: sends the command to the subprocess and yields tokens. Serialized
by _gen_lock (one generation at a time) so concurrent readers don't consume each other's
@@ -2341,6 +2350,7 @@ class InferenceOrchestrator:
continue_final_message = continue_final_message,
tool_protocol_active = tool_protocol_active,
seed = seed,
+ video_b64 = video,
)
# Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the lock above, having
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index acc84d7ca03..da2223be4f8 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -519,9 +519,13 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
# backend does not answer, which is not the same as a confirmed False.
if _entry.get("context_length_enforced") is not None:
model_info["context_length_enforced"] = bool(_entry["context_length_enforced"])
- # Backend post-load audio classification outranks pre-load config.
+ # Backend post-load audio and video classification outranks pre-load config.
model_info.update(
- {k: _entry[k] for k in ("is_audio", "audio_type", "has_audio_input") if k in _entry}
+ {
+ k: _entry[k]
+ for k in ("is_audio", "audio_type", "has_audio_input", "has_video_input")
+ if k in _entry
+ }
)
# Resolved MLX runtime knobs; only the backend knows what it honored.
model_info.update(
@@ -710,6 +714,11 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
for gated in ("seed", "frequency_penalty", "logit_bias", "stop", "tool_protocol_active"):
if gated in cmd and _backend_declares(backend, gated):
gen_kwargs[gated] = cmd[gated]
+ # A clip cannot be dropped like an unknown sampling knob: the answer would ignore it.
+ if cmd.get("video_base64"):
+ if not _backend_declares(backend, "video"):
+ raise RuntimeError("The loaded backend does not read video.")
+ gen_kwargs["video"] = cmd["video_base64"]
use_adapter = cmd.get("use_adapter")
if use_adapter is not None:
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 586968186b1..cf40255eb77 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -2108,7 +2108,7 @@ class ChatCompletionRequest(BaseModel):
None,
description = (
"[x-unsloth] Base64-encoded video (mp4/mov/webm/mkv/avi) for video-input "
- "models. GGUF only: llama-server samples frames with ffmpeg."
+ "models: a GGUF served by llama-server, or an MLX model whose processor reads video."
),
)
use_adapter: Optional[Union[bool, str]] = Field(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 559dd5b3c4e..a4ac1620968 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7520,6 +7520,37 @@ def _target_accepts_audio_input(load_path: str) -> bool:
return True
+def _names_video_token(config) -> bool:
+ if not isinstance(config, dict):
+ return False
+ return any(
+ (key in ("video_token_id", "video_token_index") and value is not None)
+ or _names_video_token(value)
+ for key, value in config.items()
+ )
+
+
+def _target_accepts_video_input(load_path: str) -> bool:
+ """Whether an MLX target's config names a video placeholder token; a config that cannot
+ be read is left to the load."""
+ from utils.hardware import DeviceType, get_device
+
+ if get_device() != DeviceType.MLX:
+ return False
+ from core.inference.mlx_inference import _mlx_vlm_decodes_video
+
+ if not _mlx_vlm_decodes_video():
+ return False
+ try:
+ config = json.loads(
+ (Path(load_path).expanduser() / "config.json").read_text(encoding = "utf-8")
+ )
+ except Exception as exc:
+ logger.debug("auto-switch: video probe failed for %s: %s", load_path, exc)
+ return True
+ return _names_video_token(config)
+
+
def _target_accepts_request_input(
load_path: str,
is_gguf: bool,
@@ -7535,7 +7566,7 @@ def _target_accepts_request_input(
A local GGUF takes both capabilities from its companion mmproj, so one probe
answers for either, modality-aware through ``need_image``. Other checkpoints
declare them apart: vision in the config architecture, audio input in the
- tokenizer's special tokens.
+ tokenizer's special tokens, video input as a placeholder token in the config.
"""
if is_gguf:
if not gguf_companion_roots:
@@ -7546,8 +7577,7 @@ def _target_accepts_request_input(
need_image,
gguf_companion_roots,
)
- # input_video is llama.cpp's own part type, so a clip is refused right after the load.
- if needs_video:
+ if needs_video and not _target_accepts_video_input(load_path):
return False
if needs_audio and not _target_accepts_audio_input(load_path):
return False
@@ -7794,6 +7824,7 @@ def _preflight_speech_codec_for_switch(
_AUDIO_IMAGE_INPUT_DETAIL = (
"This model takes audio or an image in one message, not both. Send the image on its own turn."
)
+_AUDIO_VIDEO_INPUT_DETAIL = "This model takes audio or a video in one message, not both."
async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: bool) -> None:
@@ -7831,6 +7862,8 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo
),
) from None
return
+ if audio_preflight.get("has_video"):
+ raise HTTPException(status_code = 400, detail = _AUDIO_VIDEO_INPUT_DETAIL)
if audio_preflight.get("continue_final"):
raise HTTPException(
status_code = 400,
@@ -8730,8 +8763,8 @@ async def _maybe_auto_switch_model(
can't strand a preview-owned model as Unsloth-owned. ``require_image`` makes that
rejection modality-aware for a GGUF, whose one projector carries both, and
``require_audio_input`` covers a non-GGUF checkpoint, which declares the two
- separately, and ``require_video`` rules a non-GGUF target out entirely, since
- only llama.cpp takes a clip. ``modality_label`` names the inputs attached, so the
+ separately, and ``require_video`` asks a non-GGUF target for a video placeholder
+ token. ``modality_label`` names the inputs attached, so the
rejection does not report a modality the request never carried. ``gguf_only``
marks an endpoint that reads llama.cpp alone, where loading a non-GGUF model
would unload the resident one and leave the handler with nothing to serve.
@@ -14710,6 +14743,7 @@ async def _load_model_impl(
is_audio = _model_info.get("is_audio", False),
audio_type = _model_info.get("audio_type"),
has_audio_input = _model_info.get("has_audio_input", False),
+ has_video_input = _model_info.get("has_video_input", False),
is_mlx = bool(_model_info.get("is_mlx", False)),
mlx_kv_bits = _model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),
@@ -15502,6 +15536,7 @@ async def _load_model_impl(
is_audio = _model_info.get("is_audio", config.is_audio),
audio_type = _model_info.get("audio_type", config.audio_type),
has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),
+ has_video_input = _model_info.get("has_video_input", False),
is_mlx = bool(_model_info.get("is_mlx", False)),
mlx_kv_bits = _model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),
@@ -17658,6 +17693,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)):
is_audio = False
audio_type = None
has_audio_input = False
+ has_video_input = False
model_info = {}
if backend.active_model_name:
model_info = backend.models.get(backend.active_model_name, {})
@@ -17665,6 +17701,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)):
is_audio = model_info.get("is_audio", False)
audio_type = model_info.get("audio_type")
has_audio_input = model_info.get("has_audio_input", False)
+ has_video_input = model_info.get("has_video_input", False)
chat_template_info = model_info.get("chat_template_info", {})
chat_template = (
chat_template_info.get("template") if isinstance(chat_template_info, dict) else None
@@ -17695,6 +17732,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)):
is_audio = is_audio,
audio_type = audio_type,
has_audio_input = has_audio_input,
+ has_video_input = has_video_input,
is_mlx = bool(model_info.get("is_mlx", False)),
mlx_kv_bits = model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = model_info.get("mlx_kv_bits_requested"),
@@ -20080,6 +20118,21 @@ def _prepare_audio_for_llama(b64: str) -> tuple[str, str]:
return base64.b64encode(_mono_f32_to_wav_bytes(arr, sr)).decode("ascii"), "wav"
+_VIDEO_INPUT_REFUSAL = (
+ "Video input is only supported on a local GGUF or MLX model with video support."
+)
+
+
+def _local_video_clip(payload, model_info) -> str:
+ """The clip, as bare base64, that a non-GGUF backend is handed, else a refusal by name."""
+ if not model_info.get("has_video_input"):
+ raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)
+ video_b64, rejection = _video_b64_rejection(payload.video_base64)
+ if rejection is not None:
+ raise HTTPException(status_code = rejection[0], detail = rejection[1])
+ return video_b64
+
+
def _video_b64_rejection(video_b64: str) -> tuple[str, Optional[tuple[int, str]]]:
"""The clip's base64 without its data URI header, plus why it is refused.
@@ -22186,13 +22239,9 @@ async def produce_openai_chat_completions(
untrack_current_request(request.scope)
if _wants_multiple_choices(payload):
_raise_unsupported_n("external provider chat completions")
- # input_video is llama.cpp's own part type, so the proxy has nowhere to
- # put the clip. Say so rather than answering as if there were no video.
+ # The proxy has nowhere to put the clip; say so rather than answering without it.
if payload.video_base64:
- raise HTTPException(
- status_code = 400,
- detail = "Video input is only supported on a local GGUF model with video support.",
- )
+ raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)
# _build_external_messages carries no input_audio case, so the recording would be
# stripped and the provider would answer the text alone -- a plausible reply to a
# question about audio nobody heard. Refuse it the way video is refused.
@@ -22390,6 +22439,7 @@ async def produce_openai_chat_completions(
"continue_final": _continue_final_message(payload),
"has_image": _images_in_last_user_message(payload.messages)
or _legacy_image_is_distinct(payload),
+ "has_video": _needs_video,
}
if _needs_audio_input
else None
@@ -22534,6 +22584,14 @@ async def produce_openai_chat_completions(
# Clean public id so the response never echoes a local path; the audio
# branch below receives this sanitized label too.
model_name = _orchestrator_public_model_id(backend) or payload.model
+ model_info = backend.models.get(backend.active_model_name, {})
+ # Before the speech and audio-input dispatches, which return before the clip is attached.
+ _video_clip = None
+ if payload.video_base64:
+ _video_clip = _local_video_clip(payload, model_info)
+ # Settled here: a model without audio input never enters the audio-input path.
+ if payload.audio_base64:
+ raise HTTPException(status_code = 400, detail = _AUDIO_VIDEO_INPUT_DETAIL)
# Restated here because the pre-switch check runs only when an automatic
# load may: one SSE stream carries a single choice either way.
if payload.stream and _wants_multiple_choices(payload):
@@ -22889,15 +22947,6 @@ async def produce_openai_chat_completions(
),
)
- # Injection lives in the GGUF branch below, since input_video is llama.cpp's
- # own part type. Without this a transformers model answers as if the clip
- # were never attached.
- if payload.video_base64 and not using_gguf:
- raise _reject(
- 400,
- "Video input is only supported on a local GGUF model with video support.",
- )
-
# Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to the
# fields the client omitted, so agents and API clients get the model's tuned defaults
# unless they set the field explicitly. Placed after external-provider routing (which
@@ -25001,6 +25050,7 @@ async def produce_openai_chat_completions(
(_sf_tools_on or _sf_mcp_allowed)
and _sf_features.get("supports_tools", False)
and image is None
+ and _video_clip is None
and not _sf_is_gptoss
and _sf_tool_budget > 0
)
@@ -25513,6 +25563,8 @@ async def produce_openai_chat_completions(
logit_bias = payload.logit_bias,
stop = normalized_stop,
)
+ if _video_clip is not None:
+ gen_kwargs["video"] = _video_clip
# Forward reasoning kwargs; the worker/template wrapper peels off any the
# template doesn't accept.
if payload.enable_thinking is not None:
@@ -25530,9 +25582,10 @@ async def produce_openai_chat_completions(
# supports_tools=False falls through to plain relay (GGUF gate parity).
_sf_has_tool_msgs = any(m.role == "tool" or m.tool_calls for m in payload.messages)
# Resolved BEFORE the capability gate below, which classifies from this body (#10092).
+ # A clip renders through the processor as an image does.
_sf_image_tpl = (
(_sf_model_info.get("chat_template_info") or {}).get("processor_template")
- if image is not None
+ if image is not None or _video_clip is not None
else None
)
# Differs from processor_template: a template-less processor still places the image.
@@ -25555,9 +25608,8 @@ async def produce_openai_chat_completions(
# Read the resolved value, not a fresh _effective_enable_tools: the gate
# above withdraws the launcher default for exactly these requests, and
# recomputing here would hide that and drop the client catalog.
- # Once an image rules out the server loop the passthrough takes the request, or
- # image-plus-tools is answered with prose and no schemas at all (#10092).
- (not _sf_tools_on or (image is not None and not _sf_use_tools))
+ # An image or a clip rules out the server loop, so the passthrough takes the request (#10092).
+ (not _sf_tools_on or ((image is not None or _video_clip is not None) and not _sf_use_tools))
and not _sf_use_tools
and not _sf_is_gptoss
and _sf_supports_tools
@@ -26033,11 +26085,15 @@ async def produce_openai_chat_completions(
# Mark the owning turn before the correction is appended,
# or the reverse scan attaches the picture to it (#10092).
_nudge_base = gen_kwargs["messages"]
- if _sf_renders_image:
+ if _sf_renders_image or _video_clip is not None:
from core.inference.chat_template_helpers import (
messages_with_attached_image as _nudge_attach,
)
- _nudge_base = _nudge_attach(_nudge_base)
+ _nudge_base = _nudge_attach(
+ _nudge_base,
+ image = _sf_renders_image,
+ video = _video_clip is not None,
+ )
retry_messages = [
*_nudge_base,
*nudge_messages(_data, _sf_heal),
diff --git a/studio/backend/tests/test_inference_status_route.py b/studio/backend/tests/test_inference_status_route.py
index 126e6443e8d..f83cfc63992 100644
--- a/studio/backend/tests/test_inference_status_route.py
+++ b/studio/backend/tests/test_inference_status_route.py
@@ -219,6 +219,35 @@ def test_status_keeps_the_resident_model_visible_during_a_load(monkeypatch):
assert response.loading == ["org/incoming-model"]
+def test_status_reports_the_mlx_video_capability(monkeypatch):
+ """The composer refuses a clip unless the status says the model reads one."""
+ backend = _FakeInferenceBackend()
+ backend.active_model_name = "org/mlx-video-model"
+ backend.models = {
+ "org/mlx-video-model": {"is_vision": True, "is_mlx": True, "has_video_input": True}
+ }
+ backend.loading_models = set()
+ _patch_fast_status(monkeypatch, backend)
+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)
+
+ response = asyncio.run(inference_route.get_status(current_subject = "test"))
+
+ assert response.has_video_input is True
+
+
+def test_status_reports_no_video_capability_for_a_model_without_one(monkeypatch):
+ backend = _FakeInferenceBackend()
+ backend.active_model_name = "org/plain-vision-model"
+ backend.models = {"org/plain-vision-model": {"is_vision": True, "is_mlx": True}}
+ backend.loading_models = set()
+ _patch_fast_status(monkeypatch, backend)
+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)
+
+ response = asyncio.run(inference_route.get_status(current_subject = "test"))
+
+ assert response.has_video_input is False
+
+
def test_load_is_registered_before_the_lifecycle_gate_and_always_cleared(monkeypatch):
from core.inference import llama_keepwarm
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index ec1f9ae1234..935c3e1809a 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -5355,3 +5355,561 @@ def test_mlx_unset_budget_falls_back_when_the_prompt_cannot_be_counted(monkeypat
backend._tokenizer = None
assert backend._unset_generation_budget("P") == UNSET_GENERATION_BUDGET
+
+
+_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header, decoded byte-for-byte by the backend
+
+
+def _require_video_stack_module():
+ """`_require_video_stack`, returning the package so a test can restand its ``utils``."""
+ _require_video_stack()
+ import mlx_vlm
+
+ return mlx_vlm
+
+
+def _require_video_stack():
+ """Both halves: guarding on one alone let a test needing both run on a runner with only the
+ other, and the decoder lands in 0.5.0 while the install floor is 0.4.4."""
+ utils = pytest.importorskip("mlx_vlm.utils")
+ if not callable(getattr(utils, "load_video", None)):
+ pytest.skip("the installed mlx-vlm predates the 0.5.0 clip decoder")
+ pytest.importorskip("cv2")
+ return utils
+
+
+def _video_vlm_backend(monkeypatch, streams):
+ from core.inference import mlx_inference
+
+ @contextmanager
+ def _adapter_state(_model, _state):
+ yield
+
+ monkeypatch.setattr(mlx_inference, "_temporary_mlx_adapter_state", _adapter_state)
+ mlx_vlm = types.ModuleType("mlx_vlm")
+ mlx_vlm.prompt_utils = SimpleNamespace(MODEL_CONFIG = {})
+
+ def _vlm_stream(*args, **kwargs):
+ paths = kwargs.get("video")
+ streams.append((args, kwargs, [Path(p).read_bytes() for p in paths] if paths else None))
+ yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1)
+
+ mlx_vlm.stream_generate = _vlm_stream
+ real_vlm_utils = _require_video_stack()
+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)
+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", real_vlm_utils)
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda _t, _m, **_k: "<video> marked",
+ )
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model = SimpleNamespace(config = {"model_type": "qwen3_5"})
+ backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())
+ backend._is_vlm = True
+ return backend
+
+
+def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch):
+ """A snapshot spans the video rows but is keyed by token ids the clip does not vary, so a
+ later clip would resume the earlier one's vision rows and its mRoPE grid."""
+ streams = []
+ backend = _video_vlm_backend(monkeypatch, streams)
+ backend._vlm_is_diffusion_model = lambda _model: False
+ monkeypatch.setattr(type(backend), "_vlm_prompt_cache_store", lambda self: object())
+ monkeypatch.setattr(type(backend), "_vlm_media_token_ids", lambda self, _config: (7,))
+ monkeypatch.setattr(type(backend), "_vlm_media_block", lambda self, *_a: None)
+ cache_module = types.ModuleType("mlx_vlm.models.cache")
+ cache_module.make_prompt_cache = lambda _model, max_kv_size = None: []
+ monkeypatch.setitem(sys.modules, "mlx_vlm.models", types.ModuleType("mlx_vlm.models"))
+ monkeypatch.setitem(sys.modules, "mlx_vlm.models.cache", cache_module)
+
+ # Control: without a clip the same call builds a session, so the None below is the video rule.
+ assert backend._vlm_prompt_cache_session(False, None, "prompt") is not None
+ assert backend._vlm_prompt_cache_session(False, None, "prompt", has_video = True) is None
+
+ # And the stream says so: a clip reaches mlx-vlm with no cache to resume from.
+ turn = [
+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}
+ ]
+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)
+ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ ((_, stream_kwargs, _),) = streams
+ assert "prompt_cache" not in stream_kwargs
+ assert "prompt_cache_state" not in stream_kwargs
+
+
+def test_mlx_vlm_a_video_turn_releases_retained_snapshots(monkeypatch):
+ """A video turn resumes nothing, so retained snapshots are pure occupancy against the
+ frame budget it is about to allocate."""
+ streams = []
+ backend = _video_vlm_backend(monkeypatch, streams)
+ released = []
+ monkeypatch.setattr(type(backend), "_release_vlm_snapshots", lambda self: released.append(True))
+ turn = [
+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}
+ ]
+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)
+
+ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ assert released == [True]
+
+
+def test_mlx_vlm_an_unset_budget_is_capped_for_a_video_turn(monkeypatch):
+ """A clip expands furthest past its placeholder, so the counted prompt understates the real
+ one by more than an image does."""
+ from core.inference.runtime_context import UNSET_GENERATION_BUDGET
+
+ streams = []
+ backend = _video_vlm_backend(monkeypatch, streams)
+ monkeypatch.setattr(
+ type(backend),
+ "_unset_generation_budget",
+ lambda self, *_args, **_kwargs: UNSET_GENERATION_BUDGET * 8,
+ )
+ turn = [
+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}
+ ]
+ args = (turn, None, 0, 1, 0, 0, None, 1, None)
+
+ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ assert streams[-1][1]["max_tokens"] == UNSET_GENERATION_BUDGET
+
+
+def test_mlx_vlm_video_clip_lives_on_disk_only_for_the_stream(monkeypatch):
+ """The clip lives on disk only for the stream, including when the caller closes it early."""
+ import base64
+
+ streams = []
+ backend = _video_vlm_backend(monkeypatch, streams)
+ turn = [
+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}
+ ]
+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)
+
+ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]
+ ((stream_args, stream_kwargs, contents),) = streams
+ assert stream_args[3] is None, "no image was attached"
+ assert contents == [base64.b64decode(_CLIP_B64)]
+ assert not any(Path(p).exists() for p in stream_kwargs["video"])
+
+ gen = backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)
+ assert next(gen) == "ok"
+ (path,) = streams[-1][1]["video"]
+ assert Path(path).exists()
+ gen.close()
+ assert not Path(path).exists()
+
+
+def _tiny_clip(
+ path,
+ width = 32,
+ height = 24,
+ frames = 12,
+):
+ # importorskip: two of the three callers reach here before any other guard.
+ cv2 = pytest.importorskip("cv2")
+ np = pytest.importorskip("numpy")
+
+ writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), 4.0, (width, height))
+ for index in range(frames):
+ frame = np.zeros((height, width, 3), np.uint8)
+ frame[:, index : index + 4] = 255
+ writer.write(frame)
+ writer.release()
+ return str(path)
+
+
+def test_mlx_vlm_the_decoded_frame_stack_is_bounded(monkeypatch, tmp_path):
+ """A small upload can decode to gigabytes; the rate is the one knob every release passes on."""
+ import base64
+
+ from core.inference import mlx_inference
+
+ _require_video_stack()
+ clip = _tiny_clip(tmp_path / "clip.mp4")
+ per_frame = 2 * 3 * 32 * 24
+ plain = SimpleNamespace()
+ slower = SimpleNamespace(video_processor = SimpleNamespace(fps = 1.0))
+
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 6)
+ assert mlx_inference._video_frame_rate(clip, plain) == 2.0
+ assert mlx_inference._video_frame_rate(clip, slower) == 1.0
+
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 4)
+ assert mlx_inference._video_frame_rate(clip, plain) == pytest.approx(4 / 3)
+
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 3)
+ with pytest.raises(RuntimeError, match = "32x24 frames are too large"):
+ mlx_inference._video_frame_rate(clip, plain)
+
+ unopenable = tmp_path / "header"
+ unopenable.write_bytes(base64.b64decode(_CLIP_B64))
+ assert mlx_inference._video_frame_rate(str(unopenable), plain) is None
+
+
+def test_mlx_vlm_a_clip_with_an_unreadable_rate_is_still_bounded(monkeypatch):
+ """mlx-vlm decodes such a clip at an assumed rate instead of refusing it, so skipping the
+ budget here is the one path around it."""
+ from core.inference import mlx_inference
+
+ _require_video_stack()
+ cv2 = pytest.importorskip("cv2")
+ measurements = {
+ cv2.CAP_PROP_FRAME_WIDTH: 32.0,
+ cv2.CAP_PROP_FRAME_HEIGHT: 24.0,
+ cv2.CAP_PROP_FRAME_COUNT: 12.0,
+ cv2.CAP_PROP_FPS: 0.0,
+ }
+
+ class _RatelessCapture:
+ def __init__(self, _path):
+ pass
+
+ def isOpened(self):
+ return True
+
+ def get(self, prop):
+ return measurements[prop]
+
+ def release(self):
+ pass
+
+ monkeypatch.setattr(cv2, "VideoCapture", _RatelessCapture)
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 6)
+ assert mlx_inference._video_frame_rate("clip.mp4", SimpleNamespace()) == 0.5
+
+
+def test_mlx_vlm_the_frame_rate_follows_an_older_mlx_vlm_decoder(monkeypatch, tmp_path):
+ """Older releases hand load_video only ``fps``, so its signature is the sampling source."""
+ from core.inference import mlx_inference
+
+ mlx_vlm = _require_video_stack_module()
+
+ def load_video(
+ video_path,
+ fps = 1.5,
+ nframes = None,
+ min_frames = 8,
+ max_frames = 768,
+ ):
+ raise AssertionError("never decoded here")
+
+ monkeypatch.setattr(mlx_vlm, "utils", SimpleNamespace(load_video = load_video))
+ clip = _tiny_clip(tmp_path / "clip.mp4")
+ per_frame = 2 * 3 * 32 * 24
+
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 12)
+ assert mlx_inference._video_frame_rate(clip, SimpleNamespace()) == 1.5
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 7)
+ with pytest.raises(RuntimeError, match = "8 of them"):
+ mlx_inference._video_frame_rate(clip, SimpleNamespace())
+
+
+def test_mlx_vlm_the_frame_rate_rides_the_stream(monkeypatch, tmp_path):
+ import base64
+ import tempfile
+
+ from core.inference import mlx_inference
+
+ clip_b64 = base64.b64encode(Path(_tiny_clip(tmp_path / "clip.mp4")).read_bytes()).decode()
+ spool = tmp_path / "spool"
+ spool.mkdir()
+ monkeypatch.setattr(tempfile, "tempdir", str(spool))
+ streams = []
+ backend = _video_vlm_backend(monkeypatch, streams)
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 4)
+ turn = [
+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}
+ ]
+
+ assert list(
+ backend._generate_vlm(
+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64
+ )
+ ) == ["ok"]
+ assert streams[0][1]["fps"] == pytest.approx(4 / 3)
+
+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 3)
+ gen = backend._generate_vlm(
+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64
+ )
+ with pytest.raises(RuntimeError, match = "too large to decode"):
+ next(gen)
+ assert len(streams) == 1, "a refused clip never reaches mlx-vlm"
+ assert list(spool.glob("unsloth-video-*")) == []
+
+
+def test_mlx_vlm_a_video_turn_whose_render_is_unusable_is_refused_not_recovered(monkeypatch):
+ """Registered renderers take image and audio counts only, so recovery would drop the clip."""
+ backend = _video_vlm_backend(monkeypatch, [])
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda _t, messages, **_k: f"{messages[0]['content'][0]} serialized",
+ )
+ turn = [{"role": "user", "content": [{"type": "image"}, {"type": "video"}]}]
+ with pytest.raises(RuntimeError, match = "for a video turn"):
+ list(
+ backend._generate_vlm(
+ turn, object(), 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = _CLIP_B64
+ )
+ )
+
+
+def test_mlx_vlm_an_undecodable_clip_leaves_no_file_behind(monkeypatch, tmp_path):
+ import binascii
+ import tempfile
+
+ monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
+ backend = _video_vlm_backend(monkeypatch, [])
+ turn = [{"role": "user", "content": [{"type": "video"}]}]
+ with pytest.raises(binascii.Error):
+ list(
+ backend._generate_vlm(
+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = "A"
+ )
+ )
+ assert list(tmp_path.iterdir()) == []
+
+
+def test_mlx_vlm_structured_video_items_must_match_attached_clips(monkeypatch):
+ backend = _video_vlm_backend(monkeypatch, [])
+ args = (None, 0, 1, 0, 0, 1, 1, None)
+ without_part = [{"role": "user", "content": [{"type": "text", "text": "what moves"}]}]
+ with pytest.raises(RuntimeError, match = "0 structured video item.*1 attached video"):
+ list(backend._generate_vlm(without_part, *args, _adapter_state = False, video = _CLIP_B64))
+ with_part = [{"role": "user", "content": [{"type": "video"}]}]
+ with pytest.raises(RuntimeError, match = "1 structured video item.*0 attached video"):
+ list(backend._generate_vlm(with_part, *args, _adapter_state = False))
+
+
+def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_clip(monkeypatch):
+ from core.inference import mlx_inference
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ _require_video_stack()
+ backend = MLXInferenceBackend()
+ backend._model = object()
+ backend._is_vlm = True
+ captured = []
+ backend._generate_vlm = lambda messages, *_args, **kwargs: (
+ captured.append((messages, kwargs)) or iter(())
+ )
+ list(
+ backend.generate_chat_response([{"role": "user", "content": "what moves"}], video = _CLIP_B64)
+ )
+ messages, kwargs = captured[0]
+ assert messages[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]
+ assert kwargs["video"] == _CLIP_B64
+
+ backend._is_vlm = False
+ with pytest.raises(RuntimeError, match = "loaded model does not read video"):
+ list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))
+
+ backend._is_vlm = True
+ monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)
+ with pytest.raises(RuntimeError, match = "installed mlx-vlm does not read video"):
+ list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))
+
+
+def test_mlx_vlm_decodes_video_only_with_its_clip_decoder(monkeypatch):
+ """The declared mlx-vlm range starts before load_video existed."""
+ from core.inference import mlx_inference
+
+ mlx_vlm = _require_video_stack_module()
+
+ assert mlx_inference._mlx_vlm_decodes_video() is True
+ monkeypatch.setattr(mlx_vlm, "utils", SimpleNamespace(prepare_inputs = object()))
+ assert mlx_inference._mlx_vlm_decodes_video() is False
+
+
+def test_mlx_reads_video_asks_the_processor_and_the_template(monkeypatch):
+ """The processor must carry a video component and the template must place a marker."""
+ from core.inference import mlx_inference
+ from core.inference.mlx_inference import _VIDEO_PROBE_MESSAGES, _mlx_reads_video
+
+ _require_video_stack()
+ renders = {}
+
+ def _render(_target, messages, **_kwargs):
+ if messages is _VIDEO_PROBE_MESSAGES:
+ outcome = renders["video"]
+ if isinstance(outcome, Exception):
+ raise outcome
+ return outcome
+ return "hi"
+
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation", _render
+ )
+ with_video = SimpleNamespace(video_processor = object(), tokenizer = SimpleNamespace())
+ renders["video"] = "<|video_pad|> hi"
+ assert _mlx_reads_video(with_video) is True
+ assert _mlx_reads_video(SimpleNamespace(tokenizer = SimpleNamespace())) is False
+ assert _mlx_reads_video(None) is False
+ renders["video"] = "hi"
+ assert _mlx_reads_video(with_video) is False
+ renders["video"] = f"{_VIDEO_PROBE_MESSAGES[0]['content'][0]} hi"
+ assert _mlx_reads_video(with_video) is False
+ renders["video"] = ValueError("no video in this template")
+ assert _mlx_reads_video(with_video) is False
+ renders["video"] = "<|video_pad|> hi"
+ monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)
+ assert _mlx_reads_video(with_video) is False, "a release without the decoder"
+
+
+def test_mlx_reads_video_rejects_a_clip_rendered_as_prose(monkeypatch):
+ """An installed override can render the part as ordinary text, which differs from the text
+ render while marking nothing -- the model would then answer without the clip."""
+ from core.inference.mlx_inference import _VIDEO_PROBE_MESSAGES, _mlx_reads_video
+
+ _require_video_stack()
+ renders = {}
+
+ def _render(_target, messages, **_kwargs):
+ return renders["video"] if messages is _VIDEO_PROBE_MESSAGES else "hi"
+
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation", _render
+ )
+ named = SimpleNamespace(
+ video_processor = object(),
+ video_token = "<|video_pad|>",
+ tokenizer = SimpleNamespace(),
+ )
+
+ renders["video"] = "video hi"
+ assert _mlx_reads_video(named) is False
+ renders["video"] = "[part] hi"
+ assert _mlx_reads_video(named) is False
+ renders["video"] = "<|video_pad|> hi"
+ assert _mlx_reads_video(named) is True
+
+ # A model naming no placeholder keeps the render difference as the only evidence there is.
+ unnamed = SimpleNamespace(video_processor = object(), tokenizer = SimpleNamespace())
+ renders["video"] = "video hi"
+ assert _mlx_reads_video(unnamed) is True
+
+
+def _sampling_release(monkeypatch, **utils_fields):
+ """Stand mlx_vlm.utils up in one release's shape, so one processor can be read on each."""
+ mlx_vlm = _require_video_stack_module()
+ monkeypatch.setattr(mlx_vlm, "utils", SimpleNamespace(**utils_fields))
+
+
+def _legacy_load_video(
+ video_path,
+ fps = 2.0,
+ nframes = None,
+ min_frames = 4,
+ max_frames = 768,
+):
+ raise AssertionError("never decoded here")
+
+
+def _resolving_utils():
+ """The 0.7.0 shape: a resolver that reads the processor, terminating in the library defaults."""
+
+ def resolve_video_sampling(processor, _overrides):
+ component = getattr(processor, "video_processor", None)
+ declared = {}
+ if component is not None:
+ hook = getattr(component, "video_sampling_defaults", None)
+ declared = (
+ hook()
+ if callable(hook)
+ else {n: getattr(component, n, None) for n in ("fps", "min_frames")}
+ )
+ return SimpleNamespace(
+ fps = declared.get("fps") or 2.0,
+ min_frames = declared.get("min_frames") or 4,
+ nframes = declared.get("nframes"),
+ )
+
+ return {"load_video": _legacy_load_video, "resolve_video_sampling": resolve_video_sampling}
+
+
+ "processor, expected_fps",
+ [
+ (SimpleNamespace(), 2.0),
+ (SimpleNamespace(video_processor = SimpleNamespace(fps = 1.0)), 1.0),
+ (SimpleNamespace(video_processor = SimpleNamespace(fps = 4.0)), 4.0),
+ (SimpleNamespace(video_processor = SimpleNamespace()), 2.0),
+ (
+ SimpleNamespace(
+ video_processor = SimpleNamespace(
+ video_sampling_defaults = lambda: {"fps": 0.5, "min_frames": 4}
+ )
+ ),
+ 0.5,
+ ),
+ ],
+)
+def test_mlx_vlm_every_release_reads_one_models_rate_the_same_way(
+ monkeypatch, processor, expected_fps
+):
+ """A model's declared rate must not depend on which mlx-vlm is installed.
+
+ Studio pins ``mlx-vlm>=0.4.4,<0.7.0``, so the release without a resolver is the one a real
+ install runs; reading the rate only through the resolver left every shipped install sampling
+ at the library default instead of the rate the checkpoint asked for.
+ """
+ from core.inference import mlx_inference
+
+ seen = []
+ for utils_fields in (
+ {"load_video": _legacy_load_video}, # 0.5.0 and the whole 0.6 line
+ _resolving_utils(), # 0.7.0
+ ):
+ with pytest.MonkeyPatch.context() as patch:
+ _sampling_release(patch, **utils_fields)
+ seen.append(mlx_inference._video_sampling_target(processor))
+
+ assert seen[0] == seen[1], "the installed mlx-vlm changed the sampling this model asked for"
+ assert seen[0] == (expected_fps, 4, None)
+
+
+def test_mlx_vlm_a_release_naming_no_fps_knob_still_bounds_a_clip(monkeypatch):
+ """Forward compatibility: an unknown load_video signature must not make a clip a KeyError."""
+ from core.inference import mlx_inference
+
+ def load_video(
+ video_path,
+ sampling = None,
+ **sampling_kwargs,
+ ):
+ raise AssertionError("never decoded here")
+
+ _sampling_release(monkeypatch, load_video = load_video)
+ assert mlx_inference._video_sampling_target(SimpleNamespace()) == (2.0, 4, None)
+ assert mlx_inference._video_sampling_target(
+ SimpleNamespace(video_processor = SimpleNamespace(fps = 1.0))
+ ) == (1.0, 4, None)
+
+
+def test_mlx_vlm_without_opencv_refuses_a_clip_by_name(monkeypatch):
+ """The capability must not promise what the decoder cannot deliver.
+
+ mlx-vlm's ``load_video`` and the frame budget both import cv2. OpenCV ships as an mlx-vlm
+ dependency, but a stack that lost it used to still report ``has_video_input``, so the composer
+ offered video and the request died with ModuleNotFoundError mid-stream instead of being refused.
+ """
+ from core.inference import mlx_inference
+
+ _require_video_stack()
+ assert mlx_inference._mlx_vlm_decodes_video() is True
+
+ monkeypatch.setitem(sys.modules, "cv2", None)
+ assert mlx_inference._mlx_vlm_decodes_video() is False
+ assert (
+ _mlx_reads_video_probe(monkeypatch) is False
+ ), "a model must not advertise video once the decoder is gone"
+
+
+def _mlx_reads_video_probe(monkeypatch):
+ from core.inference.mlx_inference import _mlx_reads_video
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda _t, _m, **_k: "<video> marked",
+ )
+ return _mlx_reads_video(SimpleNamespace(video_processor = object(), tokenizer = SimpleNamespace()))
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 76a33f0ed08..998ac99e3c3 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -4125,6 +4125,9 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):
audio_preflight_has_image = (
audio_preflight.get("has_image") if audio_preflight is not None else None
),
+ audio_preflight_has_video = (
+ audio_preflight.get("has_video") if audio_preflight is not None else None
+ ),
)
raise _Reached()
@@ -4141,6 +4144,7 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):
"claim_resident": False,
"require_audio_input": True,
"audio_preflight_has_image": False,
+ "audio_preflight_has_video": False,
}
# An image in the same request does need the vision tower.
@@ -4158,8 +4162,19 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):
"claim_resident": False,
"require_audio_input": True,
"audio_preflight_has_image": True,
+ "audio_preflight_has_video": False,
}
+ # A clip beside the recording is refused after the load, so the switch must know first.
+ payload = _chat_request(
+ model = "org/B-GGUF", audio_base64 = "AAAA", video_base64 = "AAAAGGZ0eXBtcDQy"
+ )
+ with pytest.raises(_Reached):
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+ assert captured["modality_label"] == "audio or video"
+ assert captured["audio_preflight_has_image"] is False
+ assert captured["audio_preflight_has_video"] is True
+
# an image on an earlier turn stays valid: the non-GGUF audio route listens to
# the current recording and deliberately permits referring back to prior images.
payload = _chat_request(
@@ -9024,34 +9039,41 @@ def test_a_video_request_labels_the_switch_refusal_video(monkeypatch):
}
-def test_a_video_request_never_switches_to_a_non_gguf_target():
- """A clip is served through llama.cpp's input_video part alone, so the chat handler
- rejects one on any other backend. Without this the swap unloads the resident GGUF
- and the request 400s straight after, which is what the guard exists to prevent."""
- # need_image False is the combination that used to fall through to the accepting branch.
- assert (
- inference_route._target_accepts_request_input(
- "/srv/models/VL-MLX",
- False,
- True,
- False,
- None,
- False,
- True,
+def test_a_video_request_switches_to_a_non_gguf_target_only_with_a_video_token(
+ tmp_path, monkeypatch
+):
+ """Loaded for video only if the config names a video token; no config is left to the load."""
+ from utils.hardware import hardware as hw
+
+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX, raising = False)
+ monkeypatch.setattr("core.inference.mlx_inference._mlx_vlm_decodes_video", lambda: True)
+
+ def _target(name, config):
+ target = tmp_path / name
+ target.mkdir()
+ if config is not None:
+ (target / "config.json").write_text(json.dumps(config), encoding = "utf-8")
+ return str(target)
+
+ def _accepts(path):
+ # need_image False is the combination that used to fall through to the accepting branch.
+ return inference_route._target_accepts_request_input(
+ path, False, True, False, None, False, True
)
- is False
- )
- # an image alongside the clip must not talk it back into the swap either.
+
+ assert _accepts(_target("image-only", {"model_type": "gemma3", "vision_config": {}})) is False
+ assert _accepts(_target("flat", {"model_type": "qwen2_vl", "video_token_id": 151656})) is True
+ assert _accepts(_target("nested", {"text_config": {"video_token_index": 7}})) is True
+ assert _accepts(_target("unset", {"video_token_id": None})) is False
+ assert _accepts(_target("no-config", None)) is True
+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.CUDA, raising = False)
+ assert _accepts(_target("cuda", {"model_type": "qwen2_vl", "video_token_id": 151656})) is False
+
+ # Without the clip decoder the load leaves has_video_input unset, so no load either.
+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX, raising = False)
+ monkeypatch.setattr("core.inference.mlx_inference._mlx_vlm_decodes_video", lambda: False)
assert (
- inference_route._target_accepts_request_input(
- "/srv/models/VL-MLX",
- False,
- True,
- False,
- None,
- True,
- True,
- )
+ _accepts(_target("old-mlx-vlm", {"model_type": "qwen2_vl", "video_token_id": 151656}))
is False
)
# the GGUF arm still decides on the companion mmproj, so needs_video does not short it.
@@ -10392,6 +10414,52 @@ def test_mixed_audio_and_image_is_rejected_before_a_non_gguf_switch(monkeypatch)
assert llama.is_loaded is True
+def test_audio_beside_a_clip_is_rejected_before_a_non_gguf_switch(monkeypatch):
+ """The clip conflict is the first audio rule served, so the switch orders it first."""
+ llama = _FakeBackend("org/A-GGUF")
+
+ class _FakeOrchestrator:
+ active_model_name = None
+ models: dict = {}
+
+ recorder = _LoadRecorder(llama)
+ _wire(
+ monkeypatch,
+ enabled = True,
+ resolves_to = ("/srv/models/Audio-VLM", None, "org/Audio-VLM"),
+ backend = llama,
+ recorder = recorder,
+ )
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: _FakeOrchestrator())
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: _FakeOrchestrator())
+ monkeypatch.setattr(resolver, "local_target_is_gguf", lambda *_a, **_kw: False)
+ monkeypatch.setattr(inference_route, "_target_accepts_request_input", lambda *_a: True)
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/Audio-VLM",
+ object(),
+ "tester",
+ require_vision = True,
+ require_image = False,
+ require_audio_input = True,
+ require_video = True,
+ audio_preflight = {
+ "b64": "AAAA",
+ "continue_final": True,
+ "has_image": True,
+ "has_video": True,
+ },
+ )
+ )
+
+ assert exc.value.status_code == 400
+ assert exc.value.detail == inference_route._AUDIO_VIDEO_INPUT_DETAIL
+ assert recorder.calls == []
+ assert llama.is_loaded is True
+
+
def test_the_gguf_audio_preflight_takes_the_base64_llama_cpp_takes():
"""The preflight must refuse exactly what _prepare_audio_for_llama refuses.
diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.py
index 12f1992f50b..5c7b433f2eb 100644
--- a/studio/backend/tests/test_presence_penalty.py
+++ b/studio/backend/tests/test_presence_penalty.py
@@ -354,3 +354,91 @@ def test_the_mlx_vlm_prefill_predicate_matches_its_decoder_gate():
predicate = after[: after.index("decoder_preserves_token")]
for condition in ("tools", "tool_protocol_active", "vlm_reasoning_markers is not None"):
assert condition in predicate, f"the VLM prefill predicate omits {condition}"
+
+
+def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_it():
+ """The clip rides the command like an image; a backend without ``video`` fails the request."""
+ from pathlib import Path
+
+ from core.inference.orchestrator import InferenceOrchestrator, _mirrored_model_entry
+ from core.inference.worker import _handle_generate
+
+ o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ assert "video_base64" not in o._build_generate_cmd("r", None, messages = [])
+ cmd = o._build_generate_cmd(
+ "r", None, messages = [{"role": "user", "content": "hi"}], video_b64 = "AAAA"
+ )
+ assert cmd["video_base64"] == "AAAA"
+
+ class _FakeQueue:
+ def __init__(self):
+ self.items = []
+
+ def put(self, item):
+ self.items.append(item)
+
+ class _VideoBackend:
+ last_generation_stats = None
+ received = None
+
+ def generate_chat_response(
+ self,
+ video = None,
+ **kwargs,
+ ):
+ self.received = video
+ return iter(())
+
+ class _TextBackend:
+ last_generation_stats = None
+ called = False
+
+ def generate_chat_response(self, **kwargs):
+ self.called = True
+ return iter(())
+
+ takes = _VideoBackend()
+ _handle_generate(takes, cmd, _FakeQueue(), threading.Event())
+ assert takes.received == "AAAA"
+ refuses, queue = _TextBackend(), _FakeQueue()
+ _handle_generate(refuses, cmd, queue, threading.Event())
+ assert refuses.called is False
+ assert [item["type"] for item in queue.items] == ["gen_error"]
+
+ # The classified capability has to reach the parent, or the route refuses every clip.
+ assert _mirrored_model_entry({"has_video_input": True}, "m")["has_video_input"] is True
+ assert _mirrored_model_entry({}, "m")["has_video_input"] is False
+ worker_source = (
+ Path(__file__).resolve().parents[1] / "core" / "inference" / "worker.py"
+ ).read_text(encoding = "utf-8")
+ assert '("is_audio", "audio_type", "has_audio_input", "has_video_input")' in worker_source
+
+
+def test_both_orchestrator_entry_points_forward_the_clip_into_the_command():
+ """The locked and the dispatched paths build the command separately."""
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ class _Built(Exception):
+ pass
+
+ built = []
+ o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o._gen_lock = threading.Lock()
+ o._unload_pending = False
+ o._exclusive_tts_pending = False
+ o.active_model_name = "m"
+ o._ensure_subprocess_alive = lambda: True
+ o._wait_dispatcher_idle = lambda: None
+ o._start_dispatcher = lambda: False
+
+ def _build(*_args, **kwargs):
+ built.append(kwargs)
+ raise _Built()
+
+ o._build_generate_cmd = _build
+ turn = [{"role": "user", "content": "hi"}]
+ with pytest.raises(_Built):
+ list(o.generate_chat_response(messages = turn, video = "AAAA"))
+ with pytest.raises(_Built):
+ list(o.generate_with_adapter_control(use_adapter = True, messages = turn, video = "AAAA"))
+ assert [kw["video_b64"] for kw in built] == ["AAAA", "AAAA"]
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index a9d69fa094e..4a7c20825a3 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -1225,3 +1225,106 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):
assert backend.calls[0]["tools"] == [LOOKUP_TOOL]
assert backend.calls[0]["image"] is not None
+
+
+def test_video_turn_with_tools_enabled_keeps_the_client_tool_catalog(monkeypatch):
+ """A clip rules out the server loop like an image; the passthrough keeps catalog and clip."""
+ backend = _vision_backend(_CALL_XML)
+ backend.models["sf-model"]["has_video_input"] = True
+ clip = "AAAAGGZ0eXBtcDQy"
+ payload = _request(
+ messages = [ChatMessage(role = "user", content = "run the tests")],
+ video_base64 = clip,
+ tools = [LOOKUP_TOOL],
+ enable_tools = True,
+ stream = False,
+ )
+ body = _json_body(_call(payload, monkeypatch, backend))
+
+ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]
+ assert backend.calls[0]["video"] == clip
+ assert body["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "lookup"
+
+
+def test_a_nudge_retry_keeps_the_video_on_the_question_turn(monkeypatch):
+ """Without the clip's turn marked first, the retry's correction turn would take the clip."""
+ truncated = '<tool_call>{"name": "lookup"'
+
+ def responder(messages, tools):
+ nudged = any(
+ "native tool-call format" in (m.get("content") or "")
+ for m in messages
+ if m.get("role") == "user" and isinstance(m.get("content"), str)
+ )
+ return [_CALL_XML] if nudged else [truncated]
+
+ backend = _vision_backend(_CALL_XML)
+ backend._responder = responder
+ backend.models["sf-model"]["has_video_input"] = True
+ clip = "AAAAGGZ0eXBtcDQy"
+ payload = _request(
+ messages = [ChatMessage(role = "user", content = "run the tests")],
+ video_base64 = clip,
+ tools = [LOOKUP_TOOL],
+ stream = False,
+ nudge_tool_calls = True,
+ )
+ _call(payload, monkeypatch, backend)
+
+ assert len(backend.calls) == 2, "the nudge retry did not run"
+ retry = backend.calls[1]["messages"]
+ assert backend.calls[1]["video"] == clip
+ question = next(m for m in retry if m["role"] == "user")
+ assert question["content"][0] == {"type": "video"}
+ assert retry[-1]["role"] == "user" and isinstance(retry[-1]["content"], str)
+
+
+def test_an_input_audio_part_beside_a_clip_is_refused_too(monkeypatch):
+ """The part is lifted onto audio_base64 before the clip gate, so one rule covers both spellings."""
+ from fastapi import HTTPException
+
+ import routes.inference as inf
+
+ backend = _vision_backend("a plain answer")
+ backend.models["sf-model"]["has_video_input"] = True
+ payload = _request(
+ video_base64 = "AAAAGGZ0eXBtcDQy",
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_audio",
+ "input_audio": {"data": "AAAA", "format": "wav"},
+ },
+ {"type": "text", "text": "what do you hear and see?"},
+ ],
+ }
+ ],
+ stream = False,
+ )
+
+ with pytest.raises(HTTPException) as exc:
+ _call(payload, monkeypatch, backend)
+
+ assert exc.value.status_code == 400
+ assert exc.value.detail == inf._AUDIO_VIDEO_INPUT_DETAIL
+ assert backend.calls == []
+
+
+def test_audio_beside_a_clip_is_refused_before_any_dispatch(monkeypatch):
+ """A model without audio input never enters the audio path, so the conflict is settled first."""
+ from fastapi import HTTPException
+
+ import routes.inference as inf
+
+ backend = _vision_backend("a plain answer")
+ backend.models["sf-model"]["has_video_input"] = True
+ payload = _request(video_base64 = "AAAAGGZ0eXBtcDQy", audio_base64 = "AAAA", stream = False)
+
+ with pytest.raises(HTTPException) as exc:
+ _call(payload, monkeypatch, backend)
+
+ assert exc.value.status_code == 400
+ assert exc.value.detail == inf._AUDIO_VIDEO_INPUT_DETAIL
+ assert backend.calls == []
diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.py
index 571b98b8623..bcf722dd6b4 100644
--- a/studio/backend/tests/test_video_attachment_part.py
+++ b/studio/backend/tests/test_video_attachment_part.py
@@ -18,6 +18,9 @@ from pathlib import Path
import pytest
+_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header, decoded byte-for-byte by the backend
+_DATA_URI = f"data:video/mp4;base64,{_CLIP_B64}"
+
pytest.importorskip("torch")
from routes.inference import _inject_video_part # noqa: E402
@@ -156,14 +159,48 @@ def test_an_external_provider_refuses_video_rather_than_ignoring_it():
start = source.index("if payload.provider_id or payload.provider_type:")
branch = source[start : source.index("_proxy_to_external_provider(payload", start)]
assert "payload.video_base64" in branch
- assert "Video input is only supported on a local GGUF model" in branch
+ assert "_VIDEO_INPUT_REFUSAL" in branch
+
+
+def _model_info(**fields):
+ return {"is_vision": True, **fields}
+
+
+def test_a_video_backend_is_handed_the_bare_clip():
+ from fastapi import HTTPException
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _VIDEO_INPUT_REFUSAL, _local_video_clip
+
+ payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)
+ assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64
+ payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _CLIP_B64)
+ assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64
+
+ with pytest.raises(HTTPException) as exc:
+ _local_video_clip(payload, _model_info())
+ assert exc.value.status_code == 400 and exc.value.detail == _VIDEO_INPUT_REFUSAL
+ payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = "data:video/mp4;base64,")
+ with pytest.raises(HTTPException) as exc:
+ _local_video_clip(payload, _model_info(has_video_input = True))
+ assert exc.value.status_code == 400
-def test_a_non_gguf_model_refuses_video_rather_than_ignoring_it():
- """Injection lives in the GGUF branch, so a transformers model would answer
- as if nothing were attached."""
+def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_generation():
+ """The gate precedes the early-returning dispatches, and the clip rides the generation kwargs."""
source = _inference_source()
- assert "if payload.video_base64 and not using_gguf:" in source
+ handler = source.index("using_gguf = llama_backend.is_loaded")
+ gate = source.index("_video_clip = _local_video_clip(payload, model_info)", handler)
+ speech = source.index("return await _monitored_generate_audio(model_name)", handler)
+ audio_input = source.index("# ── Audio INPUT path", handler)
+ assert gate < speech and gate < audio_input
+ assert 'gen_kwargs["video"] = _video_clip' in source
+ use_tools = source.index("_sf_use_tools = (", handler)
+ assert "and _video_clip is None" in source[use_tools : use_tools + 400]
+ assert "(image is not None or _video_clip is not None) and not _sf_use_tools" in source
+ # Settled at the gate: a model without audio input never enters the audio-input path.
+ conflict = source.index("if payload.audio_base64:", gate)
+ assert conflict < speech
+ assert "_AUDIO_VIDEO_INPUT_DETAIL" in source[conflict : conflict + 200]
def test_token_counting_refuses_video_like_image_and_audio():
@@ -180,4 +217,4 @@ def test_both_video_checks_share_one_rule():
"""Two size checks that drift let the pre-switch one pass what the post-load
one refuses, which is the model load this was meant to avoid."""
source = _inference_source()
- assert source.count("_video_b64_rejection(payload.video_base64)") == 2
+ assert source.count("_video_b64_rejection(payload.video_base64)") == 3
diff --git a/studio/backend/tests/test_video_pr9057_simulation.py b/studio/backend/tests/test_video_pr9057_simulation.py
index 74f58c0dab0..8cb88eebe07 100644
--- a/studio/backend/tests/test_video_pr9057_simulation.py
+++ b/studio/backend/tests/test_video_pr9057_simulation.py
@@ -413,9 +413,9 @@ def _routes_source() -> str:
"needle",
[
# external provider (OpenAI / Anthropic / any proxied backend)
- 'raise HTTPException(\n status_code = 400,\n detail = "Video input is only supported on a local GGUF model with video support.",',
- # local non-GGUF (transformers, MLX)
- "if payload.video_base64 and not using_gguf:",
+ "raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)",
+ # local non-GGUF: served on a backend reporting video input, refused elsewhere
+ "_video_clip = _local_video_clip(payload, model_info)",
# GGUF that cannot take video
'if not getattr(llama_backend, "_has_video_input", False):',
# tool / guided-decoding passthrough
diff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.py
index 16840d73545..976dfb2f4cf 100644
--- a/studio/backend/tests/test_vision_client_tools.py
+++ b/studio/backend/tests/test_vision_client_tools.py
@@ -599,6 +599,36 @@ def test_a_nudge_retry_keeps_the_image_on_the_question_turn():
)
+def test_a_video_is_attached_the_way_an_image_is():
+ """Every attached medium lands on the newest user turn, media before text, never doubled."""
+ from core.inference.chat_template_helpers import messages_with_attached_image
+
+ alone = messages_with_attached_image(
+ [{"role": "user", "content": "what moves"}], image = False, video = True
+ )
+ assert alone[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]
+
+ both = messages_with_attached_image(
+ [{"role": "user", "content": [{"type": "text", "text": "compare"}]}], video = True
+ )
+ assert both[-1]["content"] == [
+ {"type": "image"},
+ {"type": "video"},
+ {"type": "text", "text": "compare"},
+ ]
+
+ placed = messages_with_attached_image(
+ [
+ {"role": "user", "content": [{"type": "video_url", "video_url": {"url": "x"}}]},
+ {"role": "assistant", "content": "ok"},
+ {"role": "user", "content": "and now?"},
+ ],
+ video = True,
+ )
+ assert [p["type"] for p in placed[-1]["content"]] == ["image", "text"]
+ assert [p["type"] for p in placed[0]["content"]] == ["video_url"]
+
+
def test_an_mlx_processor_without_apply_chat_template_is_not_mirrored():
"""A processor template alone does not mean the render selects it; mirroring it anyway
profiles an unused body with processor semantics (#10092)."""
@@ -700,6 +730,58 @@ def test_image_tool_support_is_classified_from_the_processor_template():
assert backend.calls[0]["tools"] == [passthrough.LOOKUP_TOOL]
+def test_video_tool_support_is_classified_from_the_processor_template():
+ """A clip renders through the processor, so tool support is read off the processor template."""
+ import asyncio
+ import os
+ import sys
+
+ import pytest as _pytest
+
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+ import routes.inference as inf
+ import test_sf_client_tools_passthrough as passthrough
+ from models.inference import ChatCompletionRequest, ChatMessage
+
+ backend = passthrough._ScriptedBackend(passthrough._fixed("a plain answer"))
+ backend.models["sf-model"]["is_vision"] = True
+ backend.models["sf-model"]["has_video_input"] = True
+ backend.models["sf-model"]["chat_template_info"] = {
+ "template": _PROCESSOR_TEMPLATE_NO_TOOLS,
+ "processor_template": _CHATML_WITH_TOOLS,
+ }
+ clip = "AAAAGGZ0eXBtcDQy"
+ payload = ChatCompletionRequest(
+ model = "default",
+ messages = [ChatMessage(role = "user", content = "what moves")],
+ video_base64 = clip,
+ tools = [passthrough.LOOKUP_TOOL],
+ stream = False,
+ )
+
+ monkeypatch = _pytest.MonkeyPatch()
+ try:
+ passthrough._install(monkeypatch, backend)
+ monkeypatch.setattr(
+ inf,
+ "_detect_safetensors_features",
+ lambda _backend, template, **k: {"supports_tools": template == _CHATML_WITH_TOOLS},
+ )
+
+ async def _run():
+ return await inf.openai_chat_completions(
+ payload, request = passthrough._Request(), current_subject = "u"
+ )
+
+ asyncio.run(_run())
+ finally:
+ monkeypatch.undo()
+
+ assert backend.calls, "generation never ran"
+ assert backend.calls[0]["tools"] == [passthrough.LOOKUP_TOOL]
+ assert backend.calls[0]["video"] == clip
+
+
def test_mlx_selects_structured_content_for_a_processor_render():
"""A processor template wants part lists and the nested-tokenizer fallback wants plain
strings, so the choice follows whichever body chat_render_target selects (#10092)."""
diff --git a/studio/frontend/src/features/chat/video-attachment-adapter.ts b/studio/frontend/src/features/chat/video-attachment-adapter.ts
index 52af713b21b..a706380ca09 100644
--- a/studio/frontend/src/features/chat/video-attachment-adapter.ts
+++ b/studio/frontend/src/features/chat/video-attachment-adapter.ts
@@ -48,9 +48,8 @@ export class VideoAttachmentAdapter implements AttachmentAdapter {
externalModelLabel(checkpoint) ||
checkpoint ||
"Current model";
- // Three causes land here and the server does not say which, so name all three: /props reports
- // video only when all of them line up.
- unavailableReason = `${label} cannot accept video. Video needs a GGUF model whose mmproj supports video, a llama.cpp build with video enabled, and ffmpeg installed on this machine.`;
+ // The server does not say which cause applies, so name what each backend needs.
+ unavailableReason = `${label} cannot accept video. Video needs a GGUF model whose mmproj supports video, with a llama.cpp build with video enabled and ffmpeg installed on this machine, or an MLX vision model that reads video.`;
}
if (unavailableReason) {
toast.error(unavailableReason);