Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read a video clip on MLX chat models (#10480)
Long Yixing82fd97282128
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex 828ecad5e91..57a88f352de 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -2826,23 +2826,34 @@ def last_user_text(messages: list) -> str:return ""-def count_structured_images(content) -> int:- """Number of structured image parts in a message *content* (or a bare part)."""+_STRUCTURED_IMAGE_TYPES = ("image", "image_url", "input_image")+_STRUCTURED_VIDEO_TYPES = ("video", "video_url", "input_video")+++def _count_structured_parts(content, types) -> int:if isinstance(content, list):- return sum(count_structured_images(item) for item in content)+ return sum(_count_structured_parts(item, types) for item in content)if not isinstance(content, dict):return 0- if str(content.get("type", "")).lower() in ("image", "image_url", "input_image"):+ if str(content.get("type", "")).lower() in types:return 1- return count_structured_images(content.get("content"))+ return _count_structured_parts(content.get("content"), types)+++def count_structured_images(content) -> int:+ return _count_structured_parts(content, _STRUCTURED_IMAGE_TYPES)+++def count_structured_videos(content) -> int:+ return _count_structured_parts(content, _STRUCTURED_VIDEO_TYPES)def structured_media_reprs(content) -> set:- """Every spelling a template could print a structured image part as."""+ media_types = _STRUCTURED_IMAGE_TYPES + _STRUCTURED_VIDEO_TYPESif isinstance(content, list):values = ({str(content), json.dumps(content, ensure_ascii = False)}- if count_structured_images(content)+ if _count_structured_parts(content, media_types)else set())for item in content:@@ -2850,7 +2861,7 @@ def structured_media_reprs(content) -> set:return valuesif not isinstance(content, dict):return set()- if str(content.get("type", "")).lower() in ("image", "image_url", "input_image"):+ if str(content.get("type", "")).lower() in media_types:return {str(content), json.dumps(content, ensure_ascii = False)}return structured_media_reprs(content.get("content"))@@ -2900,15 +2911,17 @@ def messages_with_attached_image(system_prompt: str = "",fallback_user_text: str = "",structured_content: bool = False,+ image: bool = True,+ video: bool = False,) -> list:- """The conversation to render for a turn that carries an attached image.+ """The conversation to render for a turn that carries attached media.- Prepends *system_prompt* as a leading system turn, then injects an ``{"type": "image"}`` part- into the LAST user turn and leaves every other turn -- assistant ``tool_calls`` and- ``role="tool"`` results included -- exactly as the caller sent it. Rebuilding from the newest- user TEXT instead dropped the folded system instruction and the tool history an OpenAI tool loop- replays (#10092). Nothing the caller owns is mutated: callers still read those dicts after- generation, and a retry re-renders the same list.+ Prepends *system_prompt* as a leading system turn, then injects an ``{"type": "image"}`` or+ ``{"type": "video"}`` part into the LAST user turn and leaves every other turn -- assistant+ ``tool_calls`` and ``role="tool"`` results included -- exactly as the caller sent it.+ Rebuilding from the newest user TEXT instead dropped the folded system instruction and the+ tool history an OpenAI tool loop replays (#10092). Nothing the caller owns is mutated: callers+ still read those dicts after generation, and a retry re-renders the same list.*structured_content* wraps content in part lists, as a processor template expects; MLX mayrender through the nested text tokenizer, whose template expects a string. *fallback_user_text*@@ -2943,13 +2956,20 @@ def messages_with_attached_image(),},)- # Once: a reverse scan would mark a nudge retry's correction, not the question.- if any(- isinstance(m, dict)- and isinstance(m.get("content"), list)- and count_structured_images(m["content"])- for m in conversation- ):+ # Once per medium: a reverse scan would mark a nudge retry's correction, not the question.+ parts = [+ {"type": part_type}+ for part_type, wanted, counter in (+ ("image", image, count_structured_images),+ ("video", video, count_structured_videos),+ )+ if wanted+ and not any(+ isinstance(m, dict) and isinstance(m.get("content"), list) and counter(m["content"])+ for m in conversation+ )+ ]+ if not parts:return conversationfor index in range(len(conversation) - 1, -1, -1):message = conversation[index]@@ -2957,21 +2977,14 @@ def messages_with_attached_image(continuecontent = message.get("content", "")if isinstance(content, str):- parts = [{"type": "image"}, {"type": "text", "text": content or fallback_user_text}]- elif isinstance(content, list):- parts = list(content)- if not count_structured_images(parts):- parts.insert(0, {"type": "image"})- else:+ content = [{"type": "text", "text": content or fallback_user_text}]+ elif not isinstance(content, list):break- conversation[index] = {**message, "content": parts}+ conversation[index] = {**message, "content": parts + list(content)}return conversationif fallback_user_text:conversation.append(- {- "role": "user",- "content": [{"type": "image"}, {"type": "text", "text": fallback_user_text}],- }+ {"role": "user", "content": parts + [{"type": "text", "text": fallback_user_text}]})return conversationdiff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 5c807669598..ad58408e5a0 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -29,6 +29,7 @@ from core.inference.runtime_context import (from core.inference.chat_template_helpers import (# Aliased to this module's historic names; bodies moved to the shared helper (#10092).count_structured_images as _count_vlm_images,+ count_structured_videos as _count_vlm_videos,detect_reasoning_channel_markers,make_reasoning_normalizer,markup_for_tokenizer,@@ -817,6 +818,166 @@ _IMAGE_PROBE_MESSAGES = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "hi"}]}]_TEXT_PROBE_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]+_VIDEO_PROBE_MESSAGES = [+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "hi"}]}+]+++def _mlx_vlm_decodes_video() -> bool:+ """Both halves of the decoder, ``load_video`` (0.5.0+) and cv2, so a clip missing either is+ refused by name rather than by a ModuleNotFoundError mid-stream."""+ try:+ from mlx_vlm import utils as vlm_utils+ except ImportError:+ return False+ if not callable(getattr(vlm_utils, "load_video", None)):+ return False+ try:+ import cv2 # noqa: F401+ except ImportError:+ return False+ return True+++def _video_placeholder(processor):+ """The model's own video placeholder, when it names one."""+ for source in (processor, getattr(processor, "tokenizer", None)):+ token = getattr(source, "video_token", None)+ if isinstance(token, str) and token:+ return token+ return None+++def _mlx_reads_video(processor) -> bool:+ """Processor carries a video component AND the template places a video marker: a render that+ merely differs from the text one can still be prose, as in the image probe."""+ if processor is None or getattr(processor, "video_processor", None) is None:+ return False+ if not _mlx_vlm_decodes_video():+ return False+ from core.inference.chat_template_helpers import (+ apply_chat_template_for_generation,+ chat_render_target,+ )++ try:+ target = chat_render_target(processor)+ marked = apply_chat_template_for_generation(target, _VIDEO_PROBE_MESSAGES)+ if not marked or marked == apply_chat_template_for_generation(target, _TEXT_PROBE_MESSAGES):+ return False+ if _vlm_prompt_issue(marked, _VIDEO_PROBE_MESSAGES):+ return False+ placeholder = _video_placeholder(processor)+ return placeholder is None or placeholder in marked+ except BaseException:+ # A load must never fail on a probe, matching _image_marker_survives.+ return False+++def _write_video_clip(video_b64: str) -> str:+ import base64+ import tempfile++ data = base64.b64decode(video_b64)+ with tempfile.NamedTemporaryFile(prefix = "unsloth-video-", delete = False) as handle:+ handle.write(data)+ return handle.name+++# mlx-vlm holds two native-resolution RGB copies of the sampled frames before the processor resizes.+_VIDEO_DECODE_BUDGET_BYTES = 2 << 30+++# load_video's own defaults, unchanged 0.5.0 to 0.7.0, for a release that settles a knob nowhere.+_VIDEO_SAMPLING_FALLBACK = {"fps": 2.0, "min_frames": 4}+++def _sampled(name, value):+ return _VIDEO_SAMPLING_FALLBACK[name] if value is None else value+++def _declared_video_sampling(processor) -> dict:+ """Read the way mlx-vlm's ``processor_video_sampling`` reads it, so both branches agree."""+ component = getattr(processor, "video_processor", None)+ if component is None:+ return {}+ for owner in (component, processor):+ hook = getattr(owner, "video_sampling_defaults", None)+ if callable(hook):+ declared = hook()+ return dict(declared) if isinstance(declared, dict) else vars(declared)+ return {name: getattr(component, name, None) for name in ("fps", "min_frames", "max_frames")}+++def _video_sampling_target(processor) -> tuple[float, int, Optional[int]]:+ """mlx-vlm's (fps, min_frames, nframes); older releases pass only ``fps`` to ``load_video``."""+ import inspect++ from mlx_vlm import utils as vlm_utils++ resolve = getattr(vlm_utils, "resolve_video_sampling", None)+ if resolve is not None:+ sampling = resolve(processor, {})+ # Unset is unreachable today; the fallback keeps the rate divisible for a future release.+ return (+ _sampled("fps", sampling.fps),+ _sampled("min_frames", sampling.min_frames),+ sampling.nframes,+ )+ # Below 0.7.0, where install_python_stack.py pins it, prepare_inputs forwards only ``fps``,+ # so nothing else applies the model's rate. ``min_frames`` stays the count really decoded.+ defaults = inspect.signature(vlm_utils.load_video).parameters++ def _default(name):+ parameter = defaults.get(name)+ if parameter is None or parameter.default is inspect.Parameter.empty:+ return None+ return parameter.default++ declared_fps = _declared_video_sampling(processor).get("fps")+ return (+ _sampled("fps", _default("fps") if declared_fps is None else declared_fps),+ _sampled("min_frames", _default("min_frames")),+ None,+ )+++def _video_frame_rate(clip_path: str, processor) -> Optional[float]:+ """The sampling ``fps`` keeping the decoded stack within the budget; None when the clip has no+ readable frame geometry or count, which mlx-vlm then refuses itself."""+ import cv2++ capture = cv2.VideoCapture(clip_path)+ try:+ if not capture.isOpened():+ return None+ width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))+ height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))+ total_frames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))+ native_fps = capture.get(cv2.CAP_PROP_FPS)+ finally:+ capture.release()+ if min(width, height, total_frames) <= 0:+ return None+ if not native_fps or native_fps <= 0:+ # The one measurement mlx-vlm substitutes rather than refusing; match its substitution.+ native_fps = 1.0+ wanted_fps, min_frames, nframes = _video_sampling_target(processor)+ affordable = _VIDEO_DECODE_BUDGET_BYTES // (2 * 3 * width * height)+ needed = min_frames if nframes is None else nframes+ if affordable < needed:+ raise RuntimeError(+ f"The video's {width}x{height} frames are too large to decode: {needed} of them "+ f"exceed the {_VIDEO_DECODE_BUDGET_BYTES >> 20} MiB frame budget."+ )+ return min(wanted_fps, affordable * native_fps / total_frames)+++def _discard_video_clip(path: str) -> None:+ try:+ os.remove(path)+ except OSError:+ passdef _classify_mlx_audio_type(@@ -2117,11 +2278,16 @@ class MLXInferenceBackend:adapter_state,images = None,prompt = None,+ has_video = False,):store = self._vlm_prompt_cache_store()if store is None or self._vlm_is_diffusion_model(self._model):# Diffusion generation never consults the prefix hook.return None+ if has_video:+ # Keyed by token ids the clip does not vary: a later clip would resume the first+ # one's mRoPE grid.+ return Nonemedia_ids = self._vlm_media_token_ids(getattr(self._model, "config", None))if images and not media_ids:# Neither side can tell where the image's rows end without them.@@ -2527,6 +2693,7 @@ class MLXInferenceBackend:"is_audio": _audio_type is not None and _audio_type != "audio_vlm","audio_type": _audio_type,"has_audio_input": is_audio_input_type(_audio_type),+ "has_video_input": is_vision and _mlx_reads_video(self._processor),"context_length": _served_ctx,# Parity with llama.cpp's requested_n_ctx: the served window cannot say whether anything was asked for,# and that decides reuse and "pinned".@@ -2796,15 +2963,21 @@ class MLXInferenceBackend:# Unrestricted mode runs the tool protocol with an EMPTY tools list, so bool(tools)# cannot tell that the wrappers below still have to survive decoding.tool_protocol_active = None,+ video = None,) -> Generator[str, None, None]:if self._model is None:raise RuntimeError("No model loaded")+ if video is not None:+ if not _mlx_vlm_decodes_video():+ raise RuntimeError("The installed mlx-vlm does not read video.")+ if not self._is_vlm:+ raise RuntimeError("The loaded model does not read video.")# Reset so a failed run cannot surface stale stats.self.last_generation_stats = None# Shared with the transformers vision path so both render the same turns (#10092).- if self._is_vlm and image is not None:+ if self._is_vlm and (image is not None or video is not None):# Processor templates want part lists, the tokenizer fallback wants strings.from core.inference.chat_template_helpers import (chat_render_target as _chat_render_target,@@ -2817,6 +2990,8 @@ class MLXInferenceBackend:messages,system_prompt = system_prompt,structured_content = _renders_via_processor,+ image = image is not None,+ video = video is not None,)else:full_messages = self._with_system_prompt(messages, system_prompt)@@ -2844,6 +3019,7 @@ class MLXInferenceBackend:_adapter_state = _adapter_state,stop = stop,tool_protocol_active = tool_protocol_active,+ video = video,)else:stream = self._generate_text(@@ -3160,6 +3336,7 @@ class MLXInferenceBackend:messages,images,*,+ videos = None,tools = None,enable_thinking = None,reasoning_effort = None,@@ -3190,6 +3367,17 @@ class MLXInferenceBackend:f"VLM conversation contains {structured_images} structured image "f"item(s) for {attached_images} attached image(s).")+ attached_videos = 0 if videos is None else len(videos)+ structured_videos = sum(+ _count_vlm_videos(message.get("content"))+ for message in messages+ if isinstance(message, dict)+ )+ if structured_videos != attached_videos:+ raise RuntimeError(+ f"VLM conversation contains {structured_videos} structured video "+ f"item(s) for {attached_videos} attached video(s)."+ )prompt = Nonehas_tool_history = _vlm_messages_have_tool_history(messages)prompt_error = None@@ -3215,6 +3403,12 @@ class MLXInferenceBackend:f"VLM chat template returned {prompt_issue} and cannot be recovered ""without dropping tool-call history.") from prompt_error+ if prompt_issue and attached_videos:+ # mlx-vlm's registered renderers take image and audio counts only.+ raise RuntimeError(+ f"VLM chat template returned {prompt_issue} and cannot be recovered "+ "for a video turn."+ ) from prompt_errorif images is not None and prompt_issue:if tools or any(@@ -3286,6 +3480,7 @@ class MLXInferenceBackend:_adapter_state = None,tool_protocol_active = None,stop = None,+ video = None,):from mlx_vlm import stream_generate as vlm_stream@@ -3293,6 +3488,7 @@ class MLXInferenceBackend:prompt, chat_target = self._render_vlm_prompt(messages,images,+ videos = [video] if video is not None else None,tools = tools,enable_thinking = enable_thinking,reasoning_effort = reasoning_effort,@@ -3323,14 +3519,15 @@ class MLXInferenceBackend:stopped = Falseif max_new_tokens is None:max_new_tokens = self._unset_generation_budget(prompt)- if image is not None:- # An image expands past its one placeholder token, so the counted prompt is short- # of the real one: cap at the default, but stay under the rotating cache window.+ if image is not None or video is not None:+ # Media expands past its placeholder (a clip by a frame each), so the counted+ # prompt is short: cap at the default, under the cache window.max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)logger.info(- "VLM generating: prompt_len=%d, has_image=%s",+ "VLM generating: prompt_len=%d, has_image=%s, has_video=%s",len(prompt),image is not None,+ video is not None,)# stream_generate forwards **kwargs into generate_step (which builds the sampler + logits_processors# internally). GOTCHA: generate_step expects temperature= (long form); temp= is silently ignored, stuck at@@ -3395,7 +3592,9 @@ class MLXInferenceBackend:else ())- session = self._vlm_prompt_cache_session(_adapter_state, images, prompt)+ session = self._vlm_prompt_cache_session(+ _adapter_state, images, prompt, has_video = video is not None+ )if session is not None:vlm_kwargs["prompt_cache"] = session.cachevlm_kwargs["prompt_cache_state"] = session@@ -3417,13 +3616,21 @@ class MLXInferenceBackend:ExitStack() as generation_scope,session_scope,):- if images and session is None:+ if session is None and (images or video is not None):# The vision pass gets the headroom, under the lock so nothing refills it.+ # A video turn resumes no snapshot, so retained ones are pure occupancy.self._release_vlm_snapshots()generation_scope.enter_context(_mlx_fused_moe_gate_up(self._model))generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))final_response = None+ clip_path = Nonetry:+ if video is not None:+ clip_path = _write_video_clip(video)+ vlm_kwargs["video"] = [clip_path]+ frame_rate = _video_frame_rate(clip_path, self._processor)+ if frame_rate is not None:+ vlm_kwargs["fps"] = frame_rate# Emit any prefilled <think> block before the first token so the UI renders it during prefill,# matching _generate_text. Done inside the adapter context so an unsupported request raises before# any output escapes.@@ -3466,6 +3673,8 @@ class MLXInferenceBackend:if sequences and not stopped and released < len(sampled):yield prefill + sampledfinally:+ if clip_path is not None:+ _discard_video_clip(clip_path)cached_n = produced_n = 0produced_s = 0.0if session is not None:diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 17e3ae930bb..eda2749ee59 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -212,6 +212,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:"is_audio": model_info.get("is_audio", False),"audio_type": model_info.get("audio_type"),"has_audio_input": model_info.get("has_audio_input", False),+ "has_video_input": model_info.get("has_video_input", False),"context_length": model_info.get("context_length"),"native_context_length": model_info.get("native_context_length"),"max_context_length": model_info.get("max_context_length"),@@ -985,6 +986,7 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,+ video_b64: Optional[str] = None,) -> dict:"""Build the 'generate' command shared by the locked and dispatched paths."""cmd = {@@ -1007,6 +1009,8 @@ class InferenceOrchestrator:cmd["seed"] = seedif stop:cmd["stop"] = stop+ if video_b64:+ cmd["video_base64"] = video_b64if use_adapter is not None:cmd["use_adapter"] = use_adapterif tools is not None:@@ -1212,6 +1216,7 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,+ video: Optional[str] = None,) -> Generator[str, None, None]:"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-requestmailbox for tokens so two compare-mode requests can be queued at once. The subprocess@@ -1274,6 +1279,7 @@ class InferenceOrchestrator:continue_final_message = continue_final_message,tool_protocol_active = tool_protocol_active,seed = seed,+ video_b64 = video,)mailbox: queue.Queue = queue.Queue()@@ -2041,6 +2047,7 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,+ video: Optional[str] = None,) -> Generator[str, None, None]:"""Generate response, streaming tokens from subprocess. ``tools`` / ``enable_thinking`` /``reasoning_effort`` / ``preserve_thinking`` are forwarded so the template can render@@ -2072,6 +2079,7 @@ class InferenceOrchestrator:frequency_penalty = frequency_penalty,logit_bias = logit_bias,stop = stop,+ video = video,)def generate_chat_completion_with_tools(@@ -2293,6 +2301,7 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,+ video: Optional[str] = None,) -> Generator[str, None, None]:"""Inner generation logic: sends the command to the subprocess and yields tokens. Serializedby _gen_lock (one generation at a time) so concurrent readers don't consume each other's@@ -2341,6 +2350,7 @@ class InferenceOrchestrator:continue_final_message = continue_final_message,tool_protocol_active = tool_protocol_active,seed = seed,+ video_b64 = video,)# Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the lock above, havingdiff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex acc84d7ca03..da2223be4f8 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -519,9 +519,13 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:# backend does not answer, which is not the same as a confirmed False.if _entry.get("context_length_enforced") is not None:model_info["context_length_enforced"] = bool(_entry["context_length_enforced"])- # Backend post-load audio classification outranks pre-load config.+ # Backend post-load audio and video classification outranks pre-load config.model_info.update(- {k: _entry[k] for k in ("is_audio", "audio_type", "has_audio_input") if k in _entry}+ {+ k: _entry[k]+ for k in ("is_audio", "audio_type", "has_audio_input", "has_video_input")+ if k in _entry+ })# Resolved MLX runtime knobs; only the backend knows what it honored.model_info.update(@@ -710,6 +714,11 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:for gated in ("seed", "frequency_penalty", "logit_bias", "stop", "tool_protocol_active"):if gated in cmd and _backend_declares(backend, gated):gen_kwargs[gated] = cmd[gated]+ # A clip cannot be dropped like an unknown sampling knob: the answer would ignore it.+ if cmd.get("video_base64"):+ if not _backend_declares(backend, "video"):+ raise RuntimeError("The loaded backend does not read video.")+ gen_kwargs["video"] = cmd["video_base64"]use_adapter = cmd.get("use_adapter")if use_adapter is not None:diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 586968186b1..cf40255eb77 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -2108,7 +2108,7 @@ class ChatCompletionRequest(BaseModel):None,description = ("[x-unsloth] Base64-encoded video (mp4/mov/webm/mkv/avi) for video-input "- "models. GGUF only: llama-server samples frames with ffmpeg."+ "models: a GGUF served by llama-server, or an MLX model whose processor reads video."),)use_adapter: Optional[Union[bool, str]] = Field(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 559dd5b3c4e..a4ac1620968 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -7520,6 +7520,37 @@ def _target_accepts_audio_input(load_path: str) -> bool:return True+def _names_video_token(config) -> bool:+ if not isinstance(config, dict):+ return False+ return any(+ (key in ("video_token_id", "video_token_index") and value is not None)+ or _names_video_token(value)+ for key, value in config.items()+ )+++def _target_accepts_video_input(load_path: str) -> bool:+ """Whether an MLX target's config names a video placeholder token; a config that cannot+ be read is left to the load."""+ from utils.hardware import DeviceType, get_device++ if get_device() != DeviceType.MLX:+ return False+ from core.inference.mlx_inference import _mlx_vlm_decodes_video++ if not _mlx_vlm_decodes_video():+ return False+ try:+ config = json.loads(+ (Path(load_path).expanduser() / "config.json").read_text(encoding = "utf-8")+ )+ except Exception as exc:+ logger.debug("auto-switch: video probe failed for %s: %s", load_path, exc)+ return True+ return _names_video_token(config)++def _target_accepts_request_input(load_path: str,is_gguf: bool,@@ -7535,7 +7566,7 @@ def _target_accepts_request_input(A local GGUF takes both capabilities from its companion mmproj, so one probeanswers for either, modality-aware through ``need_image``. Other checkpointsdeclare them apart: vision in the config architecture, audio input in the- tokenizer's special tokens.+ tokenizer's special tokens, video input as a placeholder token in the config."""if is_gguf:if not gguf_companion_roots:@@ -7546,8 +7577,7 @@ def _target_accepts_request_input(need_image,gguf_companion_roots,)- # input_video is llama.cpp's own part type, so a clip is refused right after the load.- if needs_video:+ if needs_video and not _target_accepts_video_input(load_path):return Falseif needs_audio and not _target_accepts_audio_input(load_path):return False@@ -7794,6 +7824,7 @@ def _preflight_speech_codec_for_switch(_AUDIO_IMAGE_INPUT_DETAIL = ("This model takes audio or an image in one message, not both. Send the image on its own turn.")+_AUDIO_VIDEO_INPUT_DETAIL = "This model takes audio or a video in one message, not both."async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: bool) -> None:@@ -7831,6 +7862,8 @@ async def _preflight_audio_for_switch(audio_preflight: dict, target_is_gguf: boo),) from Nonereturn+ if audio_preflight.get("has_video"):+ raise HTTPException(status_code = 400, detail = _AUDIO_VIDEO_INPUT_DETAIL)if audio_preflight.get("continue_final"):raise HTTPException(status_code = 400,@@ -8730,8 +8763,8 @@ async def _maybe_auto_switch_model(can't strand a preview-owned model as Unsloth-owned. ``require_image`` makes thatrejection modality-aware for a GGUF, whose one projector carries both, and``require_audio_input`` covers a non-GGUF checkpoint, which declares the two- separately, and ``require_video`` rules a non-GGUF target out entirely, since- only llama.cpp takes a clip. ``modality_label`` names the inputs attached, so the+ separately, and ``require_video`` asks a non-GGUF target for a video placeholder+ token. ``modality_label`` names the inputs attached, so therejection does not report a modality the request never carried. ``gguf_only``marks an endpoint that reads llama.cpp alone, where loading a non-GGUF modelwould unload the resident one and leave the handler with nothing to serve.@@ -14710,6 +14743,7 @@ async def _load_model_impl(is_audio = _model_info.get("is_audio", False),audio_type = _model_info.get("audio_type"),has_audio_input = _model_info.get("has_audio_input", False),+ has_video_input = _model_info.get("has_video_input", False),is_mlx = bool(_model_info.get("is_mlx", False)),mlx_kv_bits = _model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),@@ -15502,6 +15536,7 @@ async def _load_model_impl(is_audio = _model_info.get("is_audio", config.is_audio),audio_type = _model_info.get("audio_type", config.audio_type),has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),+ has_video_input = _model_info.get("has_video_input", False),is_mlx = bool(_model_info.get("is_mlx", False)),mlx_kv_bits = _model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),@@ -17658,6 +17693,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)):is_audio = Falseaudio_type = Nonehas_audio_input = False+ has_video_input = Falsemodel_info = {}if backend.active_model_name:model_info = backend.models.get(backend.active_model_name, {})@@ -17665,6 +17701,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)):is_audio = model_info.get("is_audio", False)audio_type = model_info.get("audio_type")has_audio_input = model_info.get("has_audio_input", False)+ has_video_input = model_info.get("has_video_input", False)chat_template_info = model_info.get("chat_template_info", {})chat_template = (chat_template_info.get("template") if isinstance(chat_template_info, dict) else None@@ -17695,6 +17732,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)):is_audio = is_audio,audio_type = audio_type,has_audio_input = has_audio_input,+ has_video_input = has_video_input,is_mlx = bool(model_info.get("is_mlx", False)),mlx_kv_bits = model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = model_info.get("mlx_kv_bits_requested"),@@ -20080,6 +20118,21 @@ def _prepare_audio_for_llama(b64: str) -> tuple[str, str]:return base64.b64encode(_mono_f32_to_wav_bytes(arr, sr)).decode("ascii"), "wav"+_VIDEO_INPUT_REFUSAL = (+ "Video input is only supported on a local GGUF or MLX model with video support."+)+++def _local_video_clip(payload, model_info) -> str:+ """The clip, as bare base64, that a non-GGUF backend is handed, else a refusal by name."""+ if not model_info.get("has_video_input"):+ raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)+ video_b64, rejection = _video_b64_rejection(payload.video_base64)+ if rejection is not None:+ raise HTTPException(status_code = rejection[0], detail = rejection[1])+ return video_b64++def _video_b64_rejection(video_b64: str) -> tuple[str, Optional[tuple[int, str]]]:"""The clip's base64 without its data URI header, plus why it is refused.@@ -22186,13 +22239,9 @@ async def produce_openai_chat_completions(untrack_current_request(request.scope)if _wants_multiple_choices(payload):_raise_unsupported_n("external provider chat completions")- # input_video is llama.cpp's own part type, so the proxy has nowhere to- # put the clip. Say so rather than answering as if there were no video.+ # The proxy has nowhere to put the clip; say so rather than answering without it.if payload.video_base64:- raise HTTPException(- status_code = 400,- detail = "Video input is only supported on a local GGUF model with video support.",- )+ raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)# _build_external_messages carries no input_audio case, so the recording would be# stripped and the provider would answer the text alone -- a plausible reply to a# question about audio nobody heard. Refuse it the way video is refused.@@ -22390,6 +22439,7 @@ async def produce_openai_chat_completions("continue_final": _continue_final_message(payload),"has_image": _images_in_last_user_message(payload.messages)or _legacy_image_is_distinct(payload),+ "has_video": _needs_video,}if _needs_audio_inputelse None@@ -22534,6 +22584,14 @@ async def produce_openai_chat_completions(# Clean public id so the response never echoes a local path; the audio# branch below receives this sanitized label too.model_name = _orchestrator_public_model_id(backend) or payload.model+ model_info = backend.models.get(backend.active_model_name, {})+ # Before the speech and audio-input dispatches, which return before the clip is attached.+ _video_clip = None+ if payload.video_base64:+ _video_clip = _local_video_clip(payload, model_info)+ # Settled here: a model without audio input never enters the audio-input path.+ if payload.audio_base64:+ raise HTTPException(status_code = 400, detail = _AUDIO_VIDEO_INPUT_DETAIL)# Restated here because the pre-switch check runs only when an automatic# load may: one SSE stream carries a single choice either way.if payload.stream and _wants_multiple_choices(payload):@@ -22889,15 +22947,6 @@ async def produce_openai_chat_completions(),)- # Injection lives in the GGUF branch below, since input_video is llama.cpp's- # own part type. Without this a transformers model answers as if the clip- # were never attached.- if payload.video_base64 and not using_gguf:- raise _reject(- 400,- "Video input is only supported on a local GGUF model with video support.",- )-# Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to the# fields the client omitted, so agents and API clients get the model's tuned defaults# unless they set the field explicitly. Placed after external-provider routing (which@@ -25001,6 +25050,7 @@ async def produce_openai_chat_completions((_sf_tools_on or _sf_mcp_allowed)and _sf_features.get("supports_tools", False)and image is None+ and _video_clip is Noneand not _sf_is_gptossand _sf_tool_budget > 0)@@ -25513,6 +25563,8 @@ async def produce_openai_chat_completions(logit_bias = payload.logit_bias,stop = normalized_stop,)+ if _video_clip is not None:+ gen_kwargs["video"] = _video_clip# Forward reasoning kwargs; the worker/template wrapper peels off any the# template doesn't accept.if payload.enable_thinking is not None:@@ -25530,9 +25582,10 @@ async def produce_openai_chat_completions(# supports_tools=False falls through to plain relay (GGUF gate parity)._sf_has_tool_msgs = any(m.role == "tool" or m.tool_calls for m in payload.messages)# Resolved BEFORE the capability gate below, which classifies from this body (#10092).+ # A clip renders through the processor as an image does._sf_image_tpl = ((_sf_model_info.get("chat_template_info") or {}).get("processor_template")- if image is not None+ if image is not None or _video_clip is not Noneelse None)# Differs from processor_template: a template-less processor still places the image.@@ -25555,9 +25608,8 @@ async def produce_openai_chat_completions(# Read the resolved value, not a fresh _effective_enable_tools: the gate# above withdraws the launcher default for exactly these requests, and# recomputing here would hide that and drop the client catalog.- # Once an image rules out the server loop the passthrough takes the request, or- # image-plus-tools is answered with prose and no schemas at all (#10092).- (not _sf_tools_on or (image is not None and not _sf_use_tools))+ # An image or a clip rules out the server loop, so the passthrough takes the request (#10092).+ (not _sf_tools_on or ((image is not None or _video_clip is not None) and not _sf_use_tools))and not _sf_use_toolsand not _sf_is_gptossand _sf_supports_tools@@ -26033,11 +26085,15 @@ async def produce_openai_chat_completions(# Mark the owning turn before the correction is appended,# or the reverse scan attaches the picture to it (#10092)._nudge_base = gen_kwargs["messages"]- if _sf_renders_image:+ if _sf_renders_image or _video_clip is not None:from core.inference.chat_template_helpers import (messages_with_attached_image as _nudge_attach,)- _nudge_base = _nudge_attach(_nudge_base)+ _nudge_base = _nudge_attach(+ _nudge_base,+ image = _sf_renders_image,+ video = _video_clip is not None,+ )retry_messages = [*_nudge_base,*nudge_messages(_data, _sf_heal),diff --git a/studio/backend/tests/test_inference_status_route.py b/studio/backend/tests/test_inference_status_route.pyindex 126e6443e8d..f83cfc63992 100644--- a/studio/backend/tests/test_inference_status_route.py+++ b/studio/backend/tests/test_inference_status_route.py@@ -219,6 +219,35 @@ def test_status_keeps_the_resident_model_visible_during_a_load(monkeypatch):assert response.loading == ["org/incoming-model"]+def test_status_reports_the_mlx_video_capability(monkeypatch):+ """The composer refuses a clip unless the status says the model reads one."""+ backend = _FakeInferenceBackend()+ backend.active_model_name = "org/mlx-video-model"+ backend.models = {+ "org/mlx-video-model": {"is_vision": True, "is_mlx": True, "has_video_input": True}+ }+ backend.loading_models = set()+ _patch_fast_status(monkeypatch, backend)+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)++ response = asyncio.run(inference_route.get_status(current_subject = "test"))++ assert response.has_video_input is True+++def test_status_reports_no_video_capability_for_a_model_without_one(monkeypatch):+ backend = _FakeInferenceBackend()+ backend.active_model_name = "org/plain-vision-model"+ backend.models = {"org/plain-vision-model": {"is_vision": True, "is_mlx": True}}+ backend.loading_models = set()+ _patch_fast_status(monkeypatch, backend)+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)++ response = asyncio.run(inference_route.get_status(current_subject = "test"))++ assert response.has_video_input is False++def test_load_is_registered_before_the_lifecycle_gate_and_always_cleared(monkeypatch):from core.inference import llama_keepwarmdiff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex ec1f9ae1234..935c3e1809a 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -5355,3 +5355,561 @@ def test_mlx_unset_budget_falls_back_when_the_prompt_cannot_be_counted(monkeypatbackend._tokenizer = Noneassert backend._unset_generation_budget("P") == UNSET_GENERATION_BUDGET+++_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header, decoded byte-for-byte by the backend+++def _require_video_stack_module():+ """`_require_video_stack`, returning the package so a test can restand its ``utils``."""+ _require_video_stack()+ import mlx_vlm++ return mlx_vlm+++def _require_video_stack():+ """Both halves: guarding on one alone let a test needing both run on a runner with only the+ other, and the decoder lands in 0.5.0 while the install floor is 0.4.4."""+ utils = pytest.importorskip("mlx_vlm.utils")+ if not callable(getattr(utils, "load_video", None)):+ pytest.skip("the installed mlx-vlm predates the 0.5.0 clip decoder")+ pytest.importorskip("cv2")+ return utils+++def _video_vlm_backend(monkeypatch, streams):+ from core.inference import mlx_inference++ @contextmanager+ def _adapter_state(_model, _state):+ yield++ monkeypatch.setattr(mlx_inference, "_temporary_mlx_adapter_state", _adapter_state)+ mlx_vlm = types.ModuleType("mlx_vlm")+ mlx_vlm.prompt_utils = SimpleNamespace(MODEL_CONFIG = {})++ def _vlm_stream(*args, **kwargs):+ paths = kwargs.get("video")+ streams.append((args, kwargs, [Path(p).read_bytes() for p in paths] if paths else None))+ yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1)++ mlx_vlm.stream_generate = _vlm_stream+ real_vlm_utils = _require_video_stack()+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", real_vlm_utils)+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda _t, _m, **_k: "<video> marked",+ )+ backend = mlx_inference.MLXInferenceBackend()+ backend._model = SimpleNamespace(config = {"model_type": "qwen3_5"})+ backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())+ backend._is_vlm = True+ return backend+++def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch):+ """A snapshot spans the video rows but is keyed by token ids the clip does not vary, so a+ later clip would resume the earlier one's vision rows and its mRoPE grid."""+ streams = []+ backend = _video_vlm_backend(monkeypatch, streams)+ backend._vlm_is_diffusion_model = lambda _model: False+ monkeypatch.setattr(type(backend), "_vlm_prompt_cache_store", lambda self: object())+ monkeypatch.setattr(type(backend), "_vlm_media_token_ids", lambda self, _config: (7,))+ monkeypatch.setattr(type(backend), "_vlm_media_block", lambda self, *_a: None)+ cache_module = types.ModuleType("mlx_vlm.models.cache")+ cache_module.make_prompt_cache = lambda _model, max_kv_size = None: []+ monkeypatch.setitem(sys.modules, "mlx_vlm.models", types.ModuleType("mlx_vlm.models"))+ monkeypatch.setitem(sys.modules, "mlx_vlm.models.cache", cache_module)++ # Control: without a clip the same call builds a session, so the None below is the video rule.+ assert backend._vlm_prompt_cache_session(False, None, "prompt") is not None+ assert backend._vlm_prompt_cache_session(False, None, "prompt", has_video = True) is None++ # And the stream says so: a clip reaches mlx-vlm with no cache to resume from.+ turn = [+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}+ ]+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)+ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ ((_, stream_kwargs, _),) = streams+ assert "prompt_cache" not in stream_kwargs+ assert "prompt_cache_state" not in stream_kwargs+++def test_mlx_vlm_a_video_turn_releases_retained_snapshots(monkeypatch):+ """A video turn resumes nothing, so retained snapshots are pure occupancy against the+ frame budget it is about to allocate."""+ streams = []+ backend = _video_vlm_backend(monkeypatch, streams)+ released = []+ monkeypatch.setattr(type(backend), "_release_vlm_snapshots", lambda self: released.append(True))+ turn = [+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}+ ]+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)++ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ assert released == [True]+++def test_mlx_vlm_an_unset_budget_is_capped_for_a_video_turn(monkeypatch):+ """A clip expands furthest past its placeholder, so the counted prompt understates the real+ one by more than an image does."""+ from core.inference.runtime_context import UNSET_GENERATION_BUDGET++ streams = []+ backend = _video_vlm_backend(monkeypatch, streams)+ monkeypatch.setattr(+ type(backend),+ "_unset_generation_budget",+ lambda self, *_args, **_kwargs: UNSET_GENERATION_BUDGET * 8,+ )+ turn = [+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}+ ]+ args = (turn, None, 0, 1, 0, 0, None, 1, None)++ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ assert streams[-1][1]["max_tokens"] == UNSET_GENERATION_BUDGET+++def test_mlx_vlm_video_clip_lives_on_disk_only_for_the_stream(monkeypatch):+ """The clip lives on disk only for the stream, including when the caller closes it early."""+ import base64++ streams = []+ backend = _video_vlm_backend(monkeypatch, streams)+ turn = [+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}+ ]+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)++ assert list(backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)) == ["ok"]+ ((stream_args, stream_kwargs, contents),) = streams+ assert stream_args[3] is None, "no image was attached"+ assert contents == [base64.b64decode(_CLIP_B64)]+ assert not any(Path(p).exists() for p in stream_kwargs["video"])++ gen = backend._generate_vlm(*args, _adapter_state = False, video = _CLIP_B64)+ assert next(gen) == "ok"+ (path,) = streams[-1][1]["video"]+ assert Path(path).exists()+ gen.close()+ assert not Path(path).exists()+++def _tiny_clip(+ path,+ width = 32,+ height = 24,+ frames = 12,+):+ # importorskip: two of the three callers reach here before any other guard.+ cv2 = pytest.importorskip("cv2")+ np = pytest.importorskip("numpy")++ writer = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), 4.0, (width, height))+ for index in range(frames):+ frame = np.zeros((height, width, 3), np.uint8)+ frame[:, index : index + 4] = 255+ writer.write(frame)+ writer.release()+ return str(path)+++def test_mlx_vlm_the_decoded_frame_stack_is_bounded(monkeypatch, tmp_path):+ """A small upload can decode to gigabytes; the rate is the one knob every release passes on."""+ import base64++ from core.inference import mlx_inference++ _require_video_stack()+ clip = _tiny_clip(tmp_path / "clip.mp4")+ per_frame = 2 * 3 * 32 * 24+ plain = SimpleNamespace()+ slower = SimpleNamespace(video_processor = SimpleNamespace(fps = 1.0))++ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 6)+ assert mlx_inference._video_frame_rate(clip, plain) == 2.0+ assert mlx_inference._video_frame_rate(clip, slower) == 1.0++ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 4)+ assert mlx_inference._video_frame_rate(clip, plain) == pytest.approx(4 / 3)++ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 3)+ with pytest.raises(RuntimeError, match = "32x24 frames are too large"):+ mlx_inference._video_frame_rate(clip, plain)++ unopenable = tmp_path / "header"+ unopenable.write_bytes(base64.b64decode(_CLIP_B64))+ assert mlx_inference._video_frame_rate(str(unopenable), plain) is None+++def test_mlx_vlm_a_clip_with_an_unreadable_rate_is_still_bounded(monkeypatch):+ """mlx-vlm decodes such a clip at an assumed rate instead of refusing it, so skipping the+ budget here is the one path around it."""+ from core.inference import mlx_inference++ _require_video_stack()+ cv2 = pytest.importorskip("cv2")+ measurements = {+ cv2.CAP_PROP_FRAME_WIDTH: 32.0,+ cv2.CAP_PROP_FRAME_HEIGHT: 24.0,+ cv2.CAP_PROP_FRAME_COUNT: 12.0,+ cv2.CAP_PROP_FPS: 0.0,+ }++ class _RatelessCapture:+ def __init__(self, _path):+ pass++ def isOpened(self):+ return True++ def get(self, prop):+ return measurements[prop]++ def release(self):+ pass++ monkeypatch.setattr(cv2, "VideoCapture", _RatelessCapture)+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 6)+ assert mlx_inference._video_frame_rate("clip.mp4", SimpleNamespace()) == 0.5+++def test_mlx_vlm_the_frame_rate_follows_an_older_mlx_vlm_decoder(monkeypatch, tmp_path):+ """Older releases hand load_video only ``fps``, so its signature is the sampling source."""+ from core.inference import mlx_inference++ mlx_vlm = _require_video_stack_module()++ def load_video(+ video_path,+ fps = 1.5,+ nframes = None,+ min_frames = 8,+ max_frames = 768,+ ):+ raise AssertionError("never decoded here")++ monkeypatch.setattr(mlx_vlm, "utils", SimpleNamespace(load_video = load_video))+ clip = _tiny_clip(tmp_path / "clip.mp4")+ per_frame = 2 * 3 * 32 * 24++ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 12)+ assert mlx_inference._video_frame_rate(clip, SimpleNamespace()) == 1.5+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", per_frame * 7)+ with pytest.raises(RuntimeError, match = "8 of them"):+ mlx_inference._video_frame_rate(clip, SimpleNamespace())+++def test_mlx_vlm_the_frame_rate_rides_the_stream(monkeypatch, tmp_path):+ import base64+ import tempfile++ from core.inference import mlx_inference++ clip_b64 = base64.b64encode(Path(_tiny_clip(tmp_path / "clip.mp4")).read_bytes()).decode()+ spool = tmp_path / "spool"+ spool.mkdir()+ monkeypatch.setattr(tempfile, "tempdir", str(spool))+ streams = []+ backend = _video_vlm_backend(monkeypatch, streams)+ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 4)+ turn = [+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}+ ]++ assert list(+ backend._generate_vlm(+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64+ )+ ) == ["ok"]+ assert streams[0][1]["fps"] == pytest.approx(4 / 3)++ monkeypatch.setattr(mlx_inference, "_VIDEO_DECODE_BUDGET_BYTES", 2 * 3 * 32 * 24 * 3)+ gen = backend._generate_vlm(+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = clip_b64+ )+ with pytest.raises(RuntimeError, match = "too large to decode"):+ next(gen)+ assert len(streams) == 1, "a refused clip never reaches mlx-vlm"+ assert list(spool.glob("unsloth-video-*")) == []+++def test_mlx_vlm_a_video_turn_whose_render_is_unusable_is_refused_not_recovered(monkeypatch):+ """Registered renderers take image and audio counts only, so recovery would drop the clip."""+ backend = _video_vlm_backend(monkeypatch, [])+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda _t, messages, **_k: f"{messages[0]['content'][0]} serialized",+ )+ turn = [{"role": "user", "content": [{"type": "image"}, {"type": "video"}]}]+ with pytest.raises(RuntimeError, match = "for a video turn"):+ list(+ backend._generate_vlm(+ turn, object(), 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = _CLIP_B64+ )+ )+++def test_mlx_vlm_an_undecodable_clip_leaves_no_file_behind(monkeypatch, tmp_path):+ import binascii+ import tempfile++ monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))+ backend = _video_vlm_backend(monkeypatch, [])+ turn = [{"role": "user", "content": [{"type": "video"}]}]+ with pytest.raises(binascii.Error):+ list(+ backend._generate_vlm(+ turn, None, 0, 1, 0, 0, 1, 1, None, _adapter_state = False, video = "A"+ )+ )+ assert list(tmp_path.iterdir()) == []+++def test_mlx_vlm_structured_video_items_must_match_attached_clips(monkeypatch):+ backend = _video_vlm_backend(monkeypatch, [])+ args = (None, 0, 1, 0, 0, 1, 1, None)+ without_part = [{"role": "user", "content": [{"type": "text", "text": "what moves"}]}]+ with pytest.raises(RuntimeError, match = "0 structured video item.*1 attached video"):+ list(backend._generate_vlm(without_part, *args, _adapter_state = False, video = _CLIP_B64))+ with_part = [{"role": "user", "content": [{"type": "video"}]}]+ with pytest.raises(RuntimeError, match = "1 structured video item.*0 attached video"):+ list(backend._generate_vlm(with_part, *args, _adapter_state = False))+++def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_clip(monkeypatch):+ from core.inference import mlx_inference+ from core.inference.mlx_inference import MLXInferenceBackend++ _require_video_stack()+ backend = MLXInferenceBackend()+ backend._model = object()+ backend._is_vlm = True+ captured = []+ backend._generate_vlm = lambda messages, *_args, **kwargs: (+ captured.append((messages, kwargs)) or iter(())+ )+ list(+ backend.generate_chat_response([{"role": "user", "content": "what moves"}], video = _CLIP_B64)+ )+ messages, kwargs = captured[0]+ assert messages[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]+ assert kwargs["video"] == _CLIP_B64++ backend._is_vlm = False+ with pytest.raises(RuntimeError, match = "loaded model does not read video"):+ list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))++ backend._is_vlm = True+ monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)+ with pytest.raises(RuntimeError, match = "installed mlx-vlm does not read video"):+ list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))+++def test_mlx_vlm_decodes_video_only_with_its_clip_decoder(monkeypatch):+ """The declared mlx-vlm range starts before load_video existed."""+ from core.inference import mlx_inference++ mlx_vlm = _require_video_stack_module()++ assert mlx_inference._mlx_vlm_decodes_video() is True+ monkeypatch.setattr(mlx_vlm, "utils", SimpleNamespace(prepare_inputs = object()))+ assert mlx_inference._mlx_vlm_decodes_video() is False+++def test_mlx_reads_video_asks_the_processor_and_the_template(monkeypatch):+ """The processor must carry a video component and the template must place a marker."""+ from core.inference import mlx_inference+ from core.inference.mlx_inference import _VIDEO_PROBE_MESSAGES, _mlx_reads_video++ _require_video_stack()+ renders = {}++ def _render(_target, messages, **_kwargs):+ if messages is _VIDEO_PROBE_MESSAGES:+ outcome = renders["video"]+ if isinstance(outcome, Exception):+ raise outcome+ return outcome+ return "hi"++ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation", _render+ )+ with_video = SimpleNamespace(video_processor = object(), tokenizer = SimpleNamespace())+ renders["video"] = "<|video_pad|> hi"+ assert _mlx_reads_video(with_video) is True+ assert _mlx_reads_video(SimpleNamespace(tokenizer = SimpleNamespace())) is False+ assert _mlx_reads_video(None) is False+ renders["video"] = "hi"+ assert _mlx_reads_video(with_video) is False+ renders["video"] = f"{_VIDEO_PROBE_MESSAGES[0]['content'][0]} hi"+ assert _mlx_reads_video(with_video) is False+ renders["video"] = ValueError("no video in this template")+ assert _mlx_reads_video(with_video) is False+ renders["video"] = "<|video_pad|> hi"+ monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)+ assert _mlx_reads_video(with_video) is False, "a release without the decoder"+++def test_mlx_reads_video_rejects_a_clip_rendered_as_prose(monkeypatch):+ """An installed override can render the part as ordinary text, which differs from the text+ render while marking nothing -- the model would then answer without the clip."""+ from core.inference.mlx_inference import _VIDEO_PROBE_MESSAGES, _mlx_reads_video++ _require_video_stack()+ renders = {}++ def _render(_target, messages, **_kwargs):+ return renders["video"] if messages is _VIDEO_PROBE_MESSAGES else "hi"++ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation", _render+ )+ named = SimpleNamespace(+ video_processor = object(),+ video_token = "<|video_pad|>",+ tokenizer = SimpleNamespace(),+ )++ renders["video"] = "video hi"+ assert _mlx_reads_video(named) is False+ renders["video"] = "[part] hi"+ assert _mlx_reads_video(named) is False+ renders["video"] = "<|video_pad|> hi"+ assert _mlx_reads_video(named) is True++ # A model naming no placeholder keeps the render difference as the only evidence there is.+ unnamed = SimpleNamespace(video_processor = object(), tokenizer = SimpleNamespace())+ renders["video"] = "video hi"+ assert _mlx_reads_video(unnamed) is True+++def _sampling_release(monkeypatch, **utils_fields):+ """Stand mlx_vlm.utils up in one release's shape, so one processor can be read on each."""+ mlx_vlm = _require_video_stack_module()+ monkeypatch.setattr(mlx_vlm, "utils", SimpleNamespace(**utils_fields))+++def _legacy_load_video(+ video_path,+ fps = 2.0,+ nframes = None,+ min_frames = 4,+ max_frames = 768,+):+ raise AssertionError("never decoded here")+++def _resolving_utils():+ """The 0.7.0 shape: a resolver that reads the processor, terminating in the library defaults."""++ def resolve_video_sampling(processor, _overrides):+ component = getattr(processor, "video_processor", None)+ declared = {}+ if component is not None:+ hook = getattr(component, "video_sampling_defaults", None)+ declared = (+ hook()+ if callable(hook)+ else {n: getattr(component, n, None) for n in ("fps", "min_frames")}+ )+ return SimpleNamespace(+ fps = declared.get("fps") or 2.0,+ min_frames = declared.get("min_frames") or 4,+ nframes = declared.get("nframes"),+ )++ return {"load_video": _legacy_load_video, "resolve_video_sampling": resolve_video_sampling}+++ "processor, expected_fps",+ [+ (SimpleNamespace(), 2.0),+ (SimpleNamespace(video_processor = SimpleNamespace(fps = 1.0)), 1.0),+ (SimpleNamespace(video_processor = SimpleNamespace(fps = 4.0)), 4.0),+ (SimpleNamespace(video_processor = SimpleNamespace()), 2.0),+ (+ SimpleNamespace(+ video_processor = SimpleNamespace(+ video_sampling_defaults = lambda: {"fps": 0.5, "min_frames": 4}+ )+ ),+ 0.5,+ ),+ ],+)+def test_mlx_vlm_every_release_reads_one_models_rate_the_same_way(+ monkeypatch, processor, expected_fps+):+ """A model's declared rate must not depend on which mlx-vlm is installed.++ Studio pins ``mlx-vlm>=0.4.4,<0.7.0``, so the release without a resolver is the one a real+ install runs; reading the rate only through the resolver left every shipped install sampling+ at the library default instead of the rate the checkpoint asked for.+ """+ from core.inference import mlx_inference++ seen = []+ for utils_fields in (+ {"load_video": _legacy_load_video}, # 0.5.0 and the whole 0.6 line+ _resolving_utils(), # 0.7.0+ ):+ with pytest.MonkeyPatch.context() as patch:+ _sampling_release(patch, **utils_fields)+ seen.append(mlx_inference._video_sampling_target(processor))++ assert seen[0] == seen[1], "the installed mlx-vlm changed the sampling this model asked for"+ assert seen[0] == (expected_fps, 4, None)+++def test_mlx_vlm_a_release_naming_no_fps_knob_still_bounds_a_clip(monkeypatch):+ """Forward compatibility: an unknown load_video signature must not make a clip a KeyError."""+ from core.inference import mlx_inference++ def load_video(+ video_path,+ sampling = None,+ **sampling_kwargs,+ ):+ raise AssertionError("never decoded here")++ _sampling_release(monkeypatch, load_video = load_video)+ assert mlx_inference._video_sampling_target(SimpleNamespace()) == (2.0, 4, None)+ assert mlx_inference._video_sampling_target(+ SimpleNamespace(video_processor = SimpleNamespace(fps = 1.0))+ ) == (1.0, 4, None)+++def test_mlx_vlm_without_opencv_refuses_a_clip_by_name(monkeypatch):+ """The capability must not promise what the decoder cannot deliver.++ mlx-vlm's ``load_video`` and the frame budget both import cv2. OpenCV ships as an mlx-vlm+ dependency, but a stack that lost it used to still report ``has_video_input``, so the composer+ offered video and the request died with ModuleNotFoundError mid-stream instead of being refused.+ """+ from core.inference import mlx_inference++ _require_video_stack()+ assert mlx_inference._mlx_vlm_decodes_video() is True++ monkeypatch.setitem(sys.modules, "cv2", None)+ assert mlx_inference._mlx_vlm_decodes_video() is False+ assert (+ _mlx_reads_video_probe(monkeypatch) is False+ ), "a model must not advertise video once the decoder is gone"+++def _mlx_reads_video_probe(monkeypatch):+ from core.inference.mlx_inference import _mlx_reads_video+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda _t, _m, **_k: "<video> marked",+ )+ return _mlx_reads_video(SimpleNamespace(video_processor = object(), tokenizer = SimpleNamespace()))diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 76a33f0ed08..998ac99e3c3 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -4125,6 +4125,9 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):audio_preflight_has_image = (audio_preflight.get("has_image") if audio_preflight is not None else None),+ audio_preflight_has_video = (+ audio_preflight.get("has_video") if audio_preflight is not None else None+ ),)raise _Reached()@@ -4141,6 +4144,7 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):"claim_resident": False,"require_audio_input": True,"audio_preflight_has_image": False,+ "audio_preflight_has_video": False,}# An image in the same request does need the vision tower.@@ -4158,8 +4162,19 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch):"claim_resident": False,"require_audio_input": True,"audio_preflight_has_image": True,+ "audio_preflight_has_video": False,}+ # A clip beside the recording is refused after the load, so the switch must know first.+ payload = _chat_request(+ model = "org/B-GGUF", audio_base64 = "AAAA", video_base64 = "AAAAGGZ0eXBtcDQy"+ )+ with pytest.raises(_Reached):+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))+ assert captured["modality_label"] == "audio or video"+ assert captured["audio_preflight_has_image"] is False+ assert captured["audio_preflight_has_video"] is True+# an image on an earlier turn stays valid: the non-GGUF audio route listens to# the current recording and deliberately permits referring back to prior images.payload = _chat_request(@@ -9024,34 +9039,41 @@ def test_a_video_request_labels_the_switch_refusal_video(monkeypatch):}-def test_a_video_request_never_switches_to_a_non_gguf_target():- """A clip is served through llama.cpp's input_video part alone, so the chat handler- rejects one on any other backend. Without this the swap unloads the resident GGUF- and the request 400s straight after, which is what the guard exists to prevent."""- # need_image False is the combination that used to fall through to the accepting branch.- assert (- inference_route._target_accepts_request_input(- "/srv/models/VL-MLX",- False,- True,- False,- None,- False,- True,+def test_a_video_request_switches_to_a_non_gguf_target_only_with_a_video_token(+ tmp_path, monkeypatch+):+ """Loaded for video only if the config names a video token; no config is left to the load."""+ from utils.hardware import hardware as hw++ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX, raising = False)+ monkeypatch.setattr("core.inference.mlx_inference._mlx_vlm_decodes_video", lambda: True)++ def _target(name, config):+ target = tmp_path / name+ target.mkdir()+ if config is not None:+ (target / "config.json").write_text(json.dumps(config), encoding = "utf-8")+ return str(target)++ def _accepts(path):+ # need_image False is the combination that used to fall through to the accepting branch.+ return inference_route._target_accepts_request_input(+ path, False, True, False, None, False, True)- is False- )- # an image alongside the clip must not talk it back into the swap either.++ assert _accepts(_target("image-only", {"model_type": "gemma3", "vision_config": {}})) is False+ assert _accepts(_target("flat", {"model_type": "qwen2_vl", "video_token_id": 151656})) is True+ assert _accepts(_target("nested", {"text_config": {"video_token_index": 7}})) is True+ assert _accepts(_target("unset", {"video_token_id": None})) is False+ assert _accepts(_target("no-config", None)) is True+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.CUDA, raising = False)+ assert _accepts(_target("cuda", {"model_type": "qwen2_vl", "video_token_id": 151656})) is False++ # Without the clip decoder the load leaves has_video_input unset, so no load either.+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX, raising = False)+ monkeypatch.setattr("core.inference.mlx_inference._mlx_vlm_decodes_video", lambda: False)assert (- inference_route._target_accepts_request_input(- "/srv/models/VL-MLX",- False,- True,- False,- None,- True,- True,- )+ _accepts(_target("old-mlx-vlm", {"model_type": "qwen2_vl", "video_token_id": 151656}))is False)# the GGUF arm still decides on the companion mmproj, so needs_video does not short it.@@ -10392,6 +10414,52 @@ def test_mixed_audio_and_image_is_rejected_before_a_non_gguf_switch(monkeypatch)assert llama.is_loaded is True+def test_audio_beside_a_clip_is_rejected_before_a_non_gguf_switch(monkeypatch):+ """The clip conflict is the first audio rule served, so the switch orders it first."""+ llama = _FakeBackend("org/A-GGUF")++ class _FakeOrchestrator:+ active_model_name = None+ models: dict = {}++ recorder = _LoadRecorder(llama)+ _wire(+ monkeypatch,+ enabled = True,+ resolves_to = ("/srv/models/Audio-VLM", None, "org/Audio-VLM"),+ backend = llama,+ recorder = recorder,+ )+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: _FakeOrchestrator())+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: _FakeOrchestrator())+ monkeypatch.setattr(resolver, "local_target_is_gguf", lambda *_a, **_kw: False)+ monkeypatch.setattr(inference_route, "_target_accepts_request_input", lambda *_a: True)++ with pytest.raises(HTTPException) as exc:+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ "org/Audio-VLM",+ object(),+ "tester",+ require_vision = True,+ require_image = False,+ require_audio_input = True,+ require_video = True,+ audio_preflight = {+ "b64": "AAAA",+ "continue_final": True,+ "has_image": True,+ "has_video": True,+ },+ )+ )++ assert exc.value.status_code == 400+ assert exc.value.detail == inference_route._AUDIO_VIDEO_INPUT_DETAIL+ assert recorder.calls == []+ assert llama.is_loaded is True++def test_the_gguf_audio_preflight_takes_the_base64_llama_cpp_takes():"""The preflight must refuse exactly what _prepare_audio_for_llama refuses.diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.pyindex 12f1992f50b..5c7b433f2eb 100644--- a/studio/backend/tests/test_presence_penalty.py+++ b/studio/backend/tests/test_presence_penalty.py@@ -354,3 +354,91 @@ def test_the_mlx_vlm_prefill_predicate_matches_its_decoder_gate():predicate = after[: after.index("decoder_preserves_token")]for condition in ("tools", "tool_protocol_active", "vlm_reasoning_markers is not None"):assert condition in predicate, f"the VLM prefill predicate omits {condition}"+++def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_it():+ """The clip rides the command like an image; a backend without ``video`` fails the request."""+ from pathlib import Path++ from core.inference.orchestrator import InferenceOrchestrator, _mirrored_model_entry+ from core.inference.worker import _handle_generate++ o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ assert "video_base64" not in o._build_generate_cmd("r", None, messages = [])+ cmd = o._build_generate_cmd(+ "r", None, messages = [{"role": "user", "content": "hi"}], video_b64 = "AAAA"+ )+ assert cmd["video_base64"] == "AAAA"++ class _FakeQueue:+ def __init__(self):+ self.items = []++ def put(self, item):+ self.items.append(item)++ class _VideoBackend:+ last_generation_stats = None+ received = None++ def generate_chat_response(+ self,+ video = None,+ **kwargs,+ ):+ self.received = video+ return iter(())++ class _TextBackend:+ last_generation_stats = None+ called = False++ def generate_chat_response(self, **kwargs):+ self.called = True+ return iter(())++ takes = _VideoBackend()+ _handle_generate(takes, cmd, _FakeQueue(), threading.Event())+ assert takes.received == "AAAA"+ refuses, queue = _TextBackend(), _FakeQueue()+ _handle_generate(refuses, cmd, queue, threading.Event())+ assert refuses.called is False+ assert [item["type"] for item in queue.items] == ["gen_error"]++ # The classified capability has to reach the parent, or the route refuses every clip.+ assert _mirrored_model_entry({"has_video_input": True}, "m")["has_video_input"] is True+ assert _mirrored_model_entry({}, "m")["has_video_input"] is False+ worker_source = (+ Path(__file__).resolve().parents[1] / "core" / "inference" / "worker.py"+ ).read_text(encoding = "utf-8")+ assert '("is_audio", "audio_type", "has_audio_input", "has_video_input")' in worker_source+++def test_both_orchestrator_entry_points_forward_the_clip_into_the_command():+ """The locked and the dispatched paths build the command separately."""+ from core.inference.orchestrator import InferenceOrchestrator++ class _Built(Exception):+ pass++ built = []+ o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o._gen_lock = threading.Lock()+ o._unload_pending = False+ o._exclusive_tts_pending = False+ o.active_model_name = "m"+ o._ensure_subprocess_alive = lambda: True+ o._wait_dispatcher_idle = lambda: None+ o._start_dispatcher = lambda: False++ def _build(*_args, **kwargs):+ built.append(kwargs)+ raise _Built()++ o._build_generate_cmd = _build+ turn = [{"role": "user", "content": "hi"}]+ with pytest.raises(_Built):+ list(o.generate_chat_response(messages = turn, video = "AAAA"))+ with pytest.raises(_Built):+ list(o.generate_with_adapter_control(use_adapter = True, messages = turn, video = "AAAA"))+ assert [kw["video_b64"] for kw in built] == ["AAAA", "AAAA"]diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex a9d69fa094e..4a7c20825a3 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -1225,3 +1225,106 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):assert backend.calls[0]["tools"] == [LOOKUP_TOOL]assert backend.calls[0]["image"] is not None+++def test_video_turn_with_tools_enabled_keeps_the_client_tool_catalog(monkeypatch):+ """A clip rules out the server loop like an image; the passthrough keeps catalog and clip."""+ backend = _vision_backend(_CALL_XML)+ backend.models["sf-model"]["has_video_input"] = True+ clip = "AAAAGGZ0eXBtcDQy"+ payload = _request(+ messages = [ChatMessage(role = "user", content = "run the tests")],+ video_base64 = clip,+ tools = [LOOKUP_TOOL],+ enable_tools = True,+ stream = False,+ )+ body = _json_body(_call(payload, monkeypatch, backend))++ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]+ assert backend.calls[0]["video"] == clip+ assert body["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "lookup"+++def test_a_nudge_retry_keeps_the_video_on_the_question_turn(monkeypatch):+ """Without the clip's turn marked first, the retry's correction turn would take the clip."""+ truncated = '<tool_call>{"name": "lookup"'++ def responder(messages, tools):+ nudged = any(+ "native tool-call format" in (m.get("content") or "")+ for m in messages+ if m.get("role") == "user" and isinstance(m.get("content"), str)+ )+ return [_CALL_XML] if nudged else [truncated]++ backend = _vision_backend(_CALL_XML)+ backend._responder = responder+ backend.models["sf-model"]["has_video_input"] = True+ clip = "AAAAGGZ0eXBtcDQy"+ payload = _request(+ messages = [ChatMessage(role = "user", content = "run the tests")],+ video_base64 = clip,+ tools = [LOOKUP_TOOL],+ stream = False,+ nudge_tool_calls = True,+ )+ _call(payload, monkeypatch, backend)++ assert len(backend.calls) == 2, "the nudge retry did not run"+ retry = backend.calls[1]["messages"]+ assert backend.calls[1]["video"] == clip+ question = next(m for m in retry if m["role"] == "user")+ assert question["content"][0] == {"type": "video"}+ assert retry[-1]["role"] == "user" and isinstance(retry[-1]["content"], str)+++def test_an_input_audio_part_beside_a_clip_is_refused_too(monkeypatch):+ """The part is lifted onto audio_base64 before the clip gate, so one rule covers both spellings."""+ from fastapi import HTTPException++ import routes.inference as inf++ backend = _vision_backend("a plain answer")+ backend.models["sf-model"]["has_video_input"] = True+ payload = _request(+ video_base64 = "AAAAGGZ0eXBtcDQy",+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "input_audio",+ "input_audio": {"data": "AAAA", "format": "wav"},+ },+ {"type": "text", "text": "what do you hear and see?"},+ ],+ }+ ],+ stream = False,+ )++ with pytest.raises(HTTPException) as exc:+ _call(payload, monkeypatch, backend)++ assert exc.value.status_code == 400+ assert exc.value.detail == inf._AUDIO_VIDEO_INPUT_DETAIL+ assert backend.calls == []+++def test_audio_beside_a_clip_is_refused_before_any_dispatch(monkeypatch):+ """A model without audio input never enters the audio path, so the conflict is settled first."""+ from fastapi import HTTPException++ import routes.inference as inf++ backend = _vision_backend("a plain answer")+ backend.models["sf-model"]["has_video_input"] = True+ payload = _request(video_base64 = "AAAAGGZ0eXBtcDQy", audio_base64 = "AAAA", stream = False)++ with pytest.raises(HTTPException) as exc:+ _call(payload, monkeypatch, backend)++ assert exc.value.status_code == 400+ assert exc.value.detail == inf._AUDIO_VIDEO_INPUT_DETAIL+ assert backend.calls == []diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.pyindex 571b98b8623..bcf722dd6b4 100644--- a/studio/backend/tests/test_video_attachment_part.py+++ b/studio/backend/tests/test_video_attachment_part.py@@ -18,6 +18,9 @@ from pathlib import Pathimport pytest+_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header, decoded byte-for-byte by the backend+_DATA_URI = f"data:video/mp4;base64,{_CLIP_B64}"+pytest.importorskip("torch")from routes.inference import _inject_video_part # noqa: E402@@ -156,14 +159,48 @@ def test_an_external_provider_refuses_video_rather_than_ignoring_it():start = source.index("if payload.provider_id or payload.provider_type:")branch = source[start : source.index("_proxy_to_external_provider(payload", start)]assert "payload.video_base64" in branch- assert "Video input is only supported on a local GGUF model" in branch+ assert "_VIDEO_INPUT_REFUSAL" in branch+++def _model_info(**fields):+ return {"is_vision": True, **fields}+++def test_a_video_backend_is_handed_the_bare_clip():+ from fastapi import HTTPException+ from models.inference import ChatCompletionRequest+ from routes.inference import _VIDEO_INPUT_REFUSAL, _local_video_clip++ payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)+ assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64+ payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = _CLIP_B64)+ assert _local_video_clip(payload, _model_info(has_video_input = True)) == _CLIP_B64++ with pytest.raises(HTTPException) as exc:+ _local_video_clip(payload, _model_info())+ assert exc.value.status_code == 400 and exc.value.detail == _VIDEO_INPUT_REFUSAL+ payload = ChatCompletionRequest(model = "m", messages = [], video_base64 = "data:video/mp4;base64,")+ with pytest.raises(HTTPException) as exc:+ _local_video_clip(payload, _model_info(has_video_input = True))+ assert exc.value.status_code == 400-def test_a_non_gguf_model_refuses_video_rather_than_ignoring_it():- """Injection lives in the GGUF branch, so a transformers model would answer- as if nothing were attached."""+def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_generation():+ """The gate precedes the early-returning dispatches, and the clip rides the generation kwargs."""source = _inference_source()- assert "if payload.video_base64 and not using_gguf:" in source+ handler = source.index("using_gguf = llama_backend.is_loaded")+ gate = source.index("_video_clip = _local_video_clip(payload, model_info)", handler)+ speech = source.index("return await _monitored_generate_audio(model_name)", handler)+ audio_input = source.index("# ── Audio INPUT path", handler)+ assert gate < speech and gate < audio_input+ assert 'gen_kwargs["video"] = _video_clip' in source+ use_tools = source.index("_sf_use_tools = (", handler)+ assert "and _video_clip is None" in source[use_tools : use_tools + 400]+ assert "(image is not None or _video_clip is not None) and not _sf_use_tools" in source+ # Settled at the gate: a model without audio input never enters the audio-input path.+ conflict = source.index("if payload.audio_base64:", gate)+ assert conflict < speech+ assert "_AUDIO_VIDEO_INPUT_DETAIL" in source[conflict : conflict + 200]def test_token_counting_refuses_video_like_image_and_audio():@@ -180,4 +217,4 @@ def test_both_video_checks_share_one_rule():"""Two size checks that drift let the pre-switch one pass what the post-loadone refuses, which is the model load this was meant to avoid."""source = _inference_source()- assert source.count("_video_b64_rejection(payload.video_base64)") == 2+ assert source.count("_video_b64_rejection(payload.video_base64)") == 3diff --git a/studio/backend/tests/test_video_pr9057_simulation.py b/studio/backend/tests/test_video_pr9057_simulation.pyindex 74f58c0dab0..8cb88eebe07 100644--- a/studio/backend/tests/test_video_pr9057_simulation.py+++ b/studio/backend/tests/test_video_pr9057_simulation.py@@ -413,9 +413,9 @@ def _routes_source() -> str:"needle",[# external provider (OpenAI / Anthropic / any proxied backend)- 'raise HTTPException(\n status_code = 400,\n detail = "Video input is only supported on a local GGUF model with video support.",',- # local non-GGUF (transformers, MLX)- "if payload.video_base64 and not using_gguf:",+ "raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)",+ # local non-GGUF: served on a backend reporting video input, refused elsewhere+ "_video_clip = _local_video_clip(payload, model_info)",# GGUF that cannot take video'if not getattr(llama_backend, "_has_video_input", False):',# tool / guided-decoding passthroughdiff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.pyindex 16840d73545..976dfb2f4cf 100644--- a/studio/backend/tests/test_vision_client_tools.py+++ b/studio/backend/tests/test_vision_client_tools.py@@ -599,6 +599,36 @@ def test_a_nudge_retry_keeps_the_image_on_the_question_turn():)+def test_a_video_is_attached_the_way_an_image_is():+ """Every attached medium lands on the newest user turn, media before text, never doubled."""+ from core.inference.chat_template_helpers import messages_with_attached_image++ alone = messages_with_attached_image(+ [{"role": "user", "content": "what moves"}], image = False, video = True+ )+ assert alone[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]++ both = messages_with_attached_image(+ [{"role": "user", "content": [{"type": "text", "text": "compare"}]}], video = True+ )+ assert both[-1]["content"] == [+ {"type": "image"},+ {"type": "video"},+ {"type": "text", "text": "compare"},+ ]++ placed = messages_with_attached_image(+ [+ {"role": "user", "content": [{"type": "video_url", "video_url": {"url": "x"}}]},+ {"role": "assistant", "content": "ok"},+ {"role": "user", "content": "and now?"},+ ],+ video = True,+ )+ assert [p["type"] for p in placed[-1]["content"]] == ["image", "text"]+ assert [p["type"] for p in placed[0]["content"]] == ["video_url"]++def test_an_mlx_processor_without_apply_chat_template_is_not_mirrored():"""A processor template alone does not mean the render selects it; mirroring it anywayprofiles an unused body with processor semantics (#10092)."""@@ -700,6 +730,58 @@ def test_image_tool_support_is_classified_from_the_processor_template():assert backend.calls[0]["tools"] == [passthrough.LOOKUP_TOOL]+def test_video_tool_support_is_classified_from_the_processor_template():+ """A clip renders through the processor, so tool support is read off the processor template."""+ import asyncio+ import os+ import sys++ import pytest as _pytest++ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+ import routes.inference as inf+ import test_sf_client_tools_passthrough as passthrough+ from models.inference import ChatCompletionRequest, ChatMessage++ backend = passthrough._ScriptedBackend(passthrough._fixed("a plain answer"))+ backend.models["sf-model"]["is_vision"] = True+ backend.models["sf-model"]["has_video_input"] = True+ backend.models["sf-model"]["chat_template_info"] = {+ "template": _PROCESSOR_TEMPLATE_NO_TOOLS,+ "processor_template": _CHATML_WITH_TOOLS,+ }+ clip = "AAAAGGZ0eXBtcDQy"+ payload = ChatCompletionRequest(+ model = "default",+ messages = [ChatMessage(role = "user", content = "what moves")],+ video_base64 = clip,+ tools = [passthrough.LOOKUP_TOOL],+ stream = False,+ )++ monkeypatch = _pytest.MonkeyPatch()+ try:+ passthrough._install(monkeypatch, backend)+ monkeypatch.setattr(+ inf,+ "_detect_safetensors_features",+ lambda _backend, template, **k: {"supports_tools": template == _CHATML_WITH_TOOLS},+ )++ async def _run():+ return await inf.openai_chat_completions(+ payload, request = passthrough._Request(), current_subject = "u"+ )++ asyncio.run(_run())+ finally:+ monkeypatch.undo()++ assert backend.calls, "generation never ran"+ assert backend.calls[0]["tools"] == [passthrough.LOOKUP_TOOL]+ assert backend.calls[0]["video"] == clip++def test_mlx_selects_structured_content_for_a_processor_render():"""A processor template wants part lists and the nested-tokenizer fallback wants plainstrings, so the choice follows whichever body chat_render_target selects (#10092)."""diff --git a/studio/frontend/src/features/chat/video-attachment-adapter.ts b/studio/frontend/src/features/chat/video-attachment-adapter.tsindex 52af713b21b..a706380ca09 100644--- a/studio/frontend/src/features/chat/video-attachment-adapter.ts+++ b/studio/frontend/src/features/chat/video-attachment-adapter.ts@@ -48,9 +48,8 @@ export class VideoAttachmentAdapter implements AttachmentAdapter {externalModelLabel(checkpoint) ||checkpoint ||"Current model";- // Three causes land here and the server does not say which, so name all three: /props reports- // video only when all of them line up.- unavailableReason = `${label} cannot accept video. Video needs a GGUF model whose mmproj supports video, a llama.cpp build with video enabled, and ffmpeg installed on this machine.`;+ // The server does not say which cause applies, so name what each backend needs.+ unavailableReason = `${label} cannot accept video. Video needs a GGUF model whose mmproj supports video, with a llama.cpp build with video enabled and ffmpeg installed on this machine, or an MLX vision model that reads video.`;}if (unavailableReason) {toast.error(unavailableReason);