Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: accept OpenAI video_url content parts on /v1/chat/completions (#10439)
Long Yixingf7ab2098fa27
diff --git a/studio/backend/core/inference/context_window.py b/studio/backend/core/inference/context_window.pyindex 840a02a6dc8..de15047f953 100644--- a/studio/backend/core/inference/context_window.py+++ b/studio/backend/core/inference/context_window.py@@ -13,7 +13,7 @@ from typing import Any, Optional_OMITTED_TOOL_EXCHANGE = "[Earlier tool exchange omitted from the rolling context window.]"_UNPRICED_MEDIA_TYPES = frozenset(- ("image_url", "input_audio", "audio", "input_image", "input_video")+ ("image_url", "input_audio", "audio", "input_image", "input_video", "video_url"))# Trim BELOW the budget: trimming to exactly it puts the next turn over again, so the boundary creeps every turn anddiff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 92fa69e607a..c610a07f1ef 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1708,6 +1708,19 @@ class ImageContentPart(BaseModel):image_url: ImageUrl+class VideoUrl(BaseModel):+ """Video URL object: an inline data URI. Remote URLs are not fetched."""++ url: str = Field(..., description = "data:video/mp4;base64,... (inline only)")+++class VideoContentPart(BaseModel):+ """Video content part; served only through llama-server's ``input_video``."""++ type: Literal["video_url"]+ video_url: VideoUrl++class InputDocumentContentPart(BaseModel):"""Document (PDF / file) content part in a multimodal message.@@ -1804,6 +1817,7 @@ _KNOWN_CONTENT_PART_TAGS = frozenset({"text","image_url",+ "video_url","input_audio","input_document","reasoning",@@ -1825,6 +1839,7 @@ ContentPart = Annotated[Union[Annotated[TextContentPart, Tag("text")],Annotated[ImageContentPart, Tag("image_url")],+ Annotated[VideoContentPart, Tag("video_url")],Annotated[InputAudioContentPart, Tag("input_audio")],Annotated[InputDocumentContentPart, Tag("input_document")],Annotated[OpenAIReasoningContentPart, Tag("reasoning")],@@ -1892,6 +1907,14 @@ class ChatMessage(BaseModel):raise ValueError('"tool_calls" is only valid on role="assistant" messages.')if self.tool_call_id is not None and self.role != "tool":raise ValueError('"tool_call_id" is only valid on role="tool" messages.')+ # llama-server renders the marker into whatever turn carried it, so off a user turn the+ # result is template-dependent.+ if (+ self.role != "user"+ and isinstance(self.content, list)+ and any(isinstance(part, VideoContentPart) for part in self.content)+ ):+ raise ValueError(f'"video_url" parts are not valid on role="{self.role}" messages.')if self.role == "tool":# tool_call_id resolution happens at ChatCompletionRequest scope. OpenAI accepts empty tooldiff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.pyindex 9b80743b0dc..8a2bfc87ba0 100644--- a/studio/backend/routes/chat_generation_runs.py+++ b/studio/backend/routes/chat_generation_runs.py@@ -153,7 +153,11 @@ def _sanitize_request(payload: CreateChatGenerationRun) -> dict[str, Any]:detail = safe_validation_errors(exc.errors()),) from exc# Without this an unservable part is queued at 202 and fails where the caller cannot see it.- from routes.inference import _messages_have_input_audio, _reject_unsupported_content_parts+ from routes.inference import (+ _messages_have_input_audio,+ _reject_unsupported_content_parts,+ _request_has_video,+ )_reject_unsupported_content_parts(request)@@ -181,9 +185,13 @@ def _sanitize_request(payload: CreateChatGenerationRun) -> dict[str, Any]:detail = "Durable chat runs are available only for local inference",)# A media turn has no replayable transcript and its payload persists verbatim, so a base64 blob would live in- if any(- raw.get(field) not in (None, "") for field in _MEDIA_FIELDS- ) or _messages_have_input_audio(request.messages):+ # request_json for the life of the thread. _MEDIA_FIELDS is field-shaped, so a video_url part+ # needs _request_has_video.+ if (+ any(raw.get(field) not in (None, "") for field in _MEDIA_FIELDS)+ or _messages_have_input_audio(request.messages)+ or _request_has_video(request)+ ):raise HTTPException(status_code = 400,detail = "Media chat runs use the legacy streaming path",diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex b5858f01c29..d9fac49a316 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1979,6 +1979,10 @@ def _openai_llama_admission_image_tokens(llama_backend) -> int:_ADMISSION_IMAGE_PART_TYPES = ("image_url", "image")+# Both spellings: the server-side tool loop recosts the conversation AFTER+# _translate_video_parts has renamed the part, so matching video_url alone priced the+# whole base64 payload as prompt text and inflated the lease toward the KV budget.+_ADMISSION_VIDEO_PART_TYPES = ("video_url", "input_video")def _openai_llama_admission_compact_image_part(part: dict) -> dict:@@ -2044,6 +2048,11 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],estimate_content.append(part)continue+ # Priced by its own byte allowance, not as an image and not as prompt text.+ if part_type in _ADMISSION_VIDEO_PART_TYPES:+ estimate_content.append({"type": part_type, part_type: {"url": "[video]"}})+ continue+if part_type not in _ADMISSION_IMAGE_PART_TYPES:estimate_content.append(part)continue@@ -2055,11 +2064,35 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],return estimate_messages, image_parts+def _conversation_video_clips(messages) -> list[str]:+ """Clips still present in a tool-loop conversation, in either spelling.++ The loop recosts AFTER ``_translate_video_parts`` has renamed the part, so reading+ ``video_url`` alone would find nothing and charge no video at all.+ """+ clips: list[str] = []+ for msg in messages or []:+ content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)+ if not isinstance(content, list):+ continue+ for part in content:+ if not isinstance(part, dict):+ continue+ media = part.get(part.get("type") or "")+ if part.get("type") not in _ADMISSION_VIDEO_PART_TYPES or not isinstance(media, dict):+ continue+ value = media.get("data") or media.get("url") or ""+ if isinstance(value, str) and value:+ clips.append(value)+ return clips++def _openai_llama_admission_media_tokens(payload,*,message_image_parts: int = 0,image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,+ message_video_clips: Optional[list[str]] = None,) -> int:"""Estimate media KV usage without charging transport bytes as prompt tokens.@@ -2076,10 +2109,26 @@ def _openai_llama_admission_media_tokens(extra += max(0, message_image_parts) * image_tokensif _legacy_image_is_distinct(payload):extra += image_tokens- for attribute in ("audio_base64", "video_base64"):+ # _inject_video_part splices the legacy clip into the conversation as input_video before the+ # loop starts, so during a recost the clips below already include it and charging the field+ # too priced it exactly twice.+ fields = (+ ("audio_base64",) if message_video_clips is not None else ("audio_base64", "video_base64")+ )+ for attribute in fields:value = getattr(payload, attribute, None)if isinstance(value, str) and value:extra += max(1, len(value) // 4)+ # Recosting passes the CURRENT conversation: a clip on a turn that truncate_oldest has since+ # evicted is no longer sent, and charging it from the opening payload kept every later round+ # reserved at the full budget. Images already come from the conversation via message_image_parts.+ clips = (+ message_video_clips+ if message_video_clips is not None+ else _message_video_urls(getattr(payload, "messages", None))+ )+ for clip in clips:+ extra += max(1, len(clip) // 4)return extra@@ -2294,6 +2343,7 @@ def _openai_llama_admission_recost(payload,message_image_parts = message_image_parts,image_tokens = _openai_llama_admission_image_tokens(llama_backend),+ message_video_clips = _conversation_video_clips(conversation),)# Reading "Max" literally here would put the run back on the whole cache at its# first round boundary.@@ -3206,6 +3256,7 @@ from models.inference import (InputAudioContentPart,UnknownContentPart,ImageUrl,+ VideoContentPart,ResponsesRequest,ResponsesInputTextPart,ResponsesInputImagePart,@@ -18571,6 +18622,13 @@ async def generate_audio("messages","Audio input is not supported here; this route speaks the message text.",)+ # A known tag now, so the guard above no longer covers it: the clip would be dropped and+ # the text spoken alone.+ if _request_has_video(payload):+ _raise_unsupported_openai_parameter(+ "messages",+ "Video input is not supported here; this route speaks the message text.",+ )# Extract text from the last user message_, chat_messages, _ = _extract_content_parts(payload.messages)@@ -19923,6 +19981,20 @@ _MAX_AUDIO_B64_CHARS = STT_AUDIO_B64_MAX_CHARS# Flooring instead refused a file of exactly the size the composer allows._MAX_VIDEO_BYTES = 64 * 1024 * 1024_MAX_VIDEO_B64_CHARS = 4 * math.ceil(_MAX_VIDEO_BYTES / 3)+# Longest scheme named back to the caller; bounds the scan over a megabytes-long clip.+_MAX_VIDEO_SCHEME_CHARS = 16+# llama-server, not Studio, would fetch a remote clip, and its downloader sets+# set_follow_location(true) (common/http.h), so a public URL redirecting to a private address+# defeats any host check made here. Refuse the whole shape; #11010 tracks a pinned fetch that+# would let images and video both accept one.+# Each clip is a separate ffprobe + ffmpeg pair, so this bounds process launches per request.+# Four rather than one: GGUF genuinely serves several clips, and the non-GGUF path already+# refuses more than one on its own.+_MAX_VIDEO_CLIPS_PER_REQUEST = 4+_REMOTE_VIDEO_REFUSAL = (+ 400,+ "Remote video URLs are not supported. Send the clip as a data URI instead.",+)_MAX_AUDIO_SECONDS = 30 * 60# The duration cap alone is rate-relative, so a high-rate container retains far# more memory for the same 30 minutes: at 48 kHz that is 86M float32 samples,@@ -20584,15 +20656,66 @@ _VIDEO_INPUT_REFUSAL = (def _local_video_clip(payload, model_info) -> str:- """The clip, as bare base64, that a non-GGUF backend is handed, else a refusal by name."""+ """The clip, as bare base64, that a non-GGUF backend is handed, else a refusal by name.++ Reads both spellings, so MLX serves a ``video_url`` part exactly as it serves the field.+ """if not model_info.get("has_video_input"):raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)- video_b64, rejection = _video_b64_rejection(payload.video_base64)+ clips = _request_video_clips(payload)+ if not clips:+ raise HTTPException(status_code = 400, detail = "Could not read the provided video file.")+ # Generation takes one video kwarg, so a second clip would be dropped in silence and the+ # same request would mean different things on this backend and on GGUF, which forwards all.+ if len(clips) > 1:+ raise HTTPException(+ status_code = 400,+ detail = "Only one video is supported per request on this model.",+ )+ # Generation is handed one clip and no turn, so MLX would move it onto the newest turn and+ # answer as though it had just been attached.+ if _video_is_on_an_older_turn(payload):+ raise HTTPException(+ status_code = 400,+ detail = "A video must be attached to the latest message on this model.",+ )+ # Refused everywhere, so the message must not send the caller to GGUF for it.+ if _is_remote_video(clips[0]):+ raise HTTPException(status_code = _REMOTE_VIDEO_REFUSAL[0], detail = _REMOTE_VIDEO_REFUSAL[1])+ # _request_video_rejection runs only when a pre-switch validation happens, so an unsupported+ # scheme reached here and was decoded as base64 instead of earning the 400 GGUF returns.+ scheme_rejection = _video_scheme_rejection(clips[0])+ if scheme_rejection is not None:+ raise HTTPException(status_code = scheme_rejection[0], detail = scheme_rejection[1])+ video_b64, rejection = _video_b64_rejection(clips[0])if rejection is not None:raise HTTPException(status_code = rejection[0], detail = rejection[1])return video_b64+def _video_payload_chars(clip: str) -> int:+ """Base64 length without the data URI header, measured rather than sliced."""+ if clip[:5].lower() != "data:":+ return len(clip)+ comma = clip.find(",")+ return len(clip) - comma - 1 if comma != -1 else 0+++def _video_size_rejection(clip: str) -> Optional[tuple[int, str]]:+ """``_video_b64_rejection``'s verdict, measured rather than sliced.++ Validation runs twice per request, before the switch and again after the load, and only+ wants the verdict. Slicing the header off to get it copied the whole payload each time:+ 89 MB and 47 ms for a clip at the 64 MB limit, against 0 MB here.+ """+ payload_len = _video_payload_chars(clip)+ if not payload_len:+ return (400, "Could not read the provided video file.")+ if payload_len > _MAX_VIDEO_B64_CHARS:+ return (413, "Video file is too large (max 64 MB).")+ return None++def _video_b64_rejection(video_b64: str) -> tuple[str, Optional[tuple[int, str]]]:"""The clip's base64 without its data URI header, plus why it is refused.@@ -20600,7 +20723,7 @@ def _video_b64_rejection(video_b64: str) -> tuple[str, Optional[tuple[int, str]]size the composer allows. Returned rather than raised so the pre-switch andpost-load checks share one rule while raising their own way."""- if video_b64.startswith("data:"):+ if video_b64[:5].lower() == "data:":video_b64 = video_b64.split(",", 1)[1] if "," in video_b64 else ""if not video_b64:return "", (400, "Could not read the provided video file.")@@ -20725,6 +20848,172 @@ def _normalise_chat_content_parts(payload) -> None:payload.audio_base64 = lifted+def _message_video_urls(messages) -> list[str]:+ urls: list[str] = []+ for msg in messages or []:+ content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)+ if not isinstance(content, list):+ continue+ for part in content:+ if isinstance(part, VideoContentPart):+ urls.append(part.video_url.url)+ elif isinstance(part, dict) and part.get("type") == "video_url":+ video_url = part.get("video_url")+ url = video_url.get("url") if isinstance(video_url, dict) else video_url+ urls.append(url if isinstance(url, str) else "")+ return urls+++def _video_is_on_an_older_turn(payload) -> bool:+ """True when a ``video_url`` part sits anywhere before the newest user turn.++ GGUF keeps a part on the turn that carried it. The non-GGUF path flattens the parts away and+ hands generation one video kwarg, which MLX attaches to the newest user turn, so the same+ request would put the clip on a different turn per backend.+ """+ messages = getattr(payload, "messages", None) or []+ roles = [m.get("role") if isinstance(m, dict) else getattr(m, "role", None) for m in messages]+ if "user" not in roles:+ return False+ return bool(_message_video_urls(messages[: len(roles) - 1 - roles[::-1].index("user")]))+++def _request_video_clips(payload) -> list[str]:+ legacy = getattr(payload, "video_base64", None)+ return ([legacy] if legacy else []) + _message_video_urls(getattr(payload, "messages", None))+++def _request_has_video(payload) -> bool:+ return bool(_request_video_clips(payload))+++def _is_remote_video(url: str) -> bool:+ # Schemes are case-insensitive and a clip is megabytes: read the prefix, not a copy.+ return url[:8].lower().startswith(("http://", "https://"))+++def _video_scheme_rejection(clip: str) -> Optional[tuple[int, str]]:+ """Refuse a clip whose URL names a scheme neither we nor llama-server should honour.++ ``handle_media`` reads ``input_video`` as ``data`` else ``url`` and treats the one string it+ finds the same way, honouring ``file://`` under ``--media-path``. Forwarding such a clip as+ opaque payload is therefore still a local file read, so refuse it by name instead.+ """+ if not clip or clip[:5].lower() == "data:" or _is_remote_video(clip):+ return None+ # ':' is not in the base64 alphabet, so an early colon marks a URL, not payload.+ head = clip[: _MAX_VIDEO_SCHEME_CHARS + 1]+ if ":" not in head:+ return None+ return (+ 400,+ f"Unsupported video URL scheme ('{head.split(':', 1)[0]}:'). "+ "Send the clip as a data URI.",+ )+++def _request_video_rejection(payload) -> Optional[tuple[int, str]]:+ clips = _request_video_clips(payload)+ # Bytes alone do not bound the work: each clip costs its own ffprobe and ffmpeg (30s and+ # 300s ceilings), so thousands of tiny clips stay under the byte cap while holding the+ # request, and its admission lease, for hours. Count is the bound on process launches.+ if len(clips) > _MAX_VIDEO_CLIPS_PER_REQUEST:+ return (+ 400,+ f"Too many videos in one request (max {_MAX_VIDEO_CLIPS_PER_REQUEST}).",+ )+ total = 0+ for clip in clips:+ if _is_remote_video(clip):+ return _REMOTE_VIDEO_REFUSAL+ rejection = _video_scheme_rejection(clip)+ if rejection is not None:+ return rejection+ rejection = _video_size_rejection(clip)+ if rejection is not None:+ return rejection+ total += _video_payload_chars(clip)+ # And bytes bound the per-clip transcode cost the count cannot see.+ if total > _MAX_VIDEO_B64_CHARS:+ return (413, "Videos are too large (max 64 MB per request).")+ return None+++async def _shrink_clip_for_llama(video_b64: str, llama_backend) -> str:+ """Transcode one clip down to the cap, as the legacy field's path does.++ The transcode runs BEFORE llama-server samples and the server can only duplicate what was+ dropped, so an explicit rate must reach it.+ """+ try:+ return await asyncio.to_thread(+ shrink_video_for_llama,+ video_b64,+ _MAX_VIDEO_BYTES,+ sampled_fps = requested_video_fps(getattr(llama_backend, "extra_args", None)),+ )+ except Exception as e:+ # The helper absorbs its own ffmpeg failures, so reaching here means something it does+ # not model went wrong. Reject like the audio path, rather than letting a raw 500 leak+ # the monitor entry with it.+ logger.warning("Video preprocessing failed: %s", e, exc_info = True)+ raise HTTPException(+ status_code = 400, detail = "Could not process the provided video file."+ ) from None+++async def _shrink_video_parts(messages: list[dict], llama_backend) -> None:+ """Transcode every ``video_url`` part in place, before translation renames them.++ The legacy field is shrunk on its own path. Doing it here as well is what keeps the two+ spellings identical: a part-carried clip would otherwise reach llama-server untranscoded and+ at a different resolution than the same clip sent as the field.+ """+ for msg in messages:+ content = msg.get("content")+ if not isinstance(content, list):+ continue+ for part in content:+ if not isinstance(part, dict) or part.get("type") != "video_url":+ continue+ video_url = part.get("video_url")+ url = video_url.get("url") if isinstance(video_url, dict) else video_url+ url = url if isinstance(url, str) else ""+ if _is_remote_video(url) or _video_scheme_rejection(url) is not None:+ continue # _translate_video_parts refuses these by name a moment later.+ clip, rejection = _video_b64_rejection(url)+ if rejection is not None:+ continue+ part["video_url"] = {"url": await _shrink_clip_for_llama(clip, llama_backend)}+++def _translate_video_parts(messages: list[dict]) -> None:+ """Rewrite ``video_url`` parts in place as llama-server's ``input_video``, always as bytes."""+ for msg in messages:+ content = msg.get("content")+ if not isinstance(content, list):+ continue+ for part in content:+ if not isinstance(part, dict) or part.get("type") != "video_url":+ continue+ video_url = part.get("video_url")+ url = video_url.get("url") if isinstance(video_url, dict) else video_url+ url = url if isinstance(url, str) else ""+ # Restated: _request_video_rejection only runs when a pre-switch validation happens,+ # so no caller reaching here by another route may hand llama-server a URL to dial.+ if _is_remote_video(url):+ raise HTTPException(+ status_code = _REMOTE_VIDEO_REFUSAL[0], detail = _REMOTE_VIDEO_REFUSAL[1]+ )+ scheme_rejection = _video_scheme_rejection(url)+ if scheme_rejection is not None:+ raise HTTPException(status_code = scheme_rejection[0], detail = scheme_rejection[1])+ part.clear()+ part.update(+ {"type": "input_video", "input_video": {"data": _video_b64_rejection(url)[0]}}+ )++def _inject_audio_part(messages: list[dict], audio_b64: str, audio_format: str) -> None:"""Append an input_audio part to the last user message, in place.@@ -22698,8 +22987,9 @@ async def produce_openai_chat_completions(untrack_current_request(request.scope)if _wants_multiple_choices(payload):_raise_unsupported_n("external provider chat completions")- # The proxy has nowhere to put the clip; say so rather than answering without it.- if payload.video_base64:+ # The proxy has nowhere to put the clip; say so rather than answering without it. Both+ # spellings, so a video_url part is refused here the way the legacy field is.+ if _request_has_video(payload):raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)# _build_external_messages carries no input_audio case, so the recording would be# stripped and the provider would answer the text alone -- a plausible reply to a@@ -22867,7 +23157,7 @@ async def produce_openai_chat_completions(# Video rides that projector too. Its own /props gate can only run after# the load, so this at least keeps a text-only target from evicting a# working model to serve a clip it could never take.- _needs_video = bool(payload.video_base64)+ _needs_video = _request_has_video(payload)_needs_vision = _needs_image or bool(payload.audio_base64) or _needs_video# Name what is actually attached, so the refusal does not report a# modality the request never carried.@@ -22877,7 +23167,7 @@ async def produce_openai_chat_completions(for name, present in (("image", _needs_image),("audio", bool(payload.audio_base64)),- ("video", bool(payload.video_base64)),+ ("video", _needs_video),)if present)@@ -22885,10 +23175,9 @@ async def produce_openai_chat_completions()# Size is knowable now and the switch is not cheap: refuse an oversized# clip before it costs a model load.- if payload.video_base64:- _, _video_rejection = _video_b64_rejection(payload.video_base64)- if _video_rejection is not None:- raise HTTPException(status_code = _video_rejection[0], detail = _video_rejection[1])+ _video_rejection = _request_video_rejection(payload)+ if _video_rejection is not None:+ raise HTTPException(status_code = _video_rejection[0], detail = _video_rejection[1])_needs_audio_input = bool(payload.audio_base64)# the rest of the audio checks depend on the target's format, so the switch runs them.@@ -23046,7 +23335,7 @@ async def produce_openai_chat_completions(model_info = backend.models.get(backend.active_model_name, {})# Before the speech and audio-input dispatches, which return before the clip is attached._video_clip = None- if payload.video_base64:+ if _request_has_video(payload):_video_clip = _local_video_clip(payload, model_info)# Settled here: a model without audio input never enters the audio-input path.if payload.audio_base64:@@ -23507,7 +23796,7 @@ async def produce_openai_chat_completions(400,"Audio input is not supported together with guided decoding or client-supplied tools yet.",)- if payload.video_base64:+ if _request_has_video(payload):# Same shape: _build_openai_passthrough_body forwards an explicit# field list, so the clip would be dropped and the model would# answer without it.@@ -23609,31 +23898,19 @@ async def produce_openai_chat_completions(# llama-server samples frames but encodes each at the clip's resolution.video_b64 = None- if payload.video_base64:+ if _request_has_video(payload):if not getattr(llama_backend, "_has_video_input", False):raise _reject(400,"Video provided but the current GGUF model cannot take video input. ""It needs an mmproj with video support, and ffmpeg/ffprobe installed.",)- video_b64, video_rejection = _video_b64_rejection(payload.video_base64)+ video_rejection = _request_video_rejection(payload)if video_rejection is not None:raise _reject(*video_rejection)- try:- # The transcode runs BEFORE llama-server samples and the server can- # only duplicate what was dropped, so an explicit rate must reach it.- video_b64 = await asyncio.to_thread(- shrink_video_for_llama,- video_b64,- _MAX_VIDEO_BYTES,- sampled_fps = requested_video_fps(getattr(llama_backend, "extra_args", None)),- )- except Exception as e:- # The helper absorbs its own ffmpeg failures, so reaching here means- # something it does not model went wrong. Reject like the audio path- # above, rather than letting a raw 500 leak the monitor entry with it.- logger.warning("Video preprocessing failed: %s", e, exc_info = True)- raise _reject(400, "Could not process the provided video file.")+ if payload.video_base64:+ video_b64, _ = _video_b64_rejection(payload.video_base64)+ video_b64 = await _shrink_clip_for_llama(video_b64, llama_backend)gguf_messages, _ = await _openai_messages_for_gguf_chat_async(payload,@@ -23645,6 +23922,8 @@ async def produce_openai_chat_completions(_inject_audio_part(gguf_messages, audio_b64, audio_format)if video_b64:_inject_video_part(gguf_messages, video_b64)+ await _shrink_video_parts(gguf_messages, llama_backend)+ _translate_video_parts(gguf_messages)cancel_event = _chat_cancel_event(request)@@ -31754,7 +32033,7 @@ async def chat_count_tokens(detail = "Cannot count tokens for messages containing audio.",)# And video, whose frames llama-server samples at completion time.- if getattr(payload, "video_base64", None):+ if _request_has_video(payload):raise HTTPException(status_code = 503,detail = "Cannot count tokens for messages containing video.",diff --git a/studio/backend/routes/preview.py b/studio/backend/routes/preview.pyindex 018fadfd895..719756e696e 100644--- a/studio/backend/routes/preview.py+++ b/studio/backend/routes/preview.py@@ -19,6 +19,7 @@ from auth.storage import DEFAULT_ADMIN_USERNAMEfrom models.inference import ChatCompletionRequest, LoadRequestfrom routes.inference import (_reject_unsupported_content_parts,+ _request_has_video,disable_openai_auto_switch_for_request,load_model_for_preview,openai_chat_completions,@@ -159,6 +160,12 @@ async def _serve_chat():path = _resolve_or_4xx(run, checkpoint)_reject_unsupported_content_parts(payload)+ # A checkpoint cannot read a clip: refuse before the load the refusal would otherwise follow.+ if _request_has_video(payload):+ raise HTTPException(+ status_code = 400,+ detail = "Video input is only supported on a local GGUF model with video support.",+ )is_lora = (path / "adapter_config.json").exists()payload = _sanitize_preview_payload(payload, is_lora)scope = getattr(request, "scope", None)diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.pyindex bf4537cc0fe..d409e4000a1 100644--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py@@ -901,3 +901,100 @@ class TestARoundIsCostedTheSameWayTheReservationWas:f"round zero shrank the lease from {opened} to {committed}, dropping the "f"non-message prompt the reservation charged")+++class TestARoundStopsPayingForAnEvictedClip:+ """truncate_oldest can drop the turn that carried a clip. The re-cost reads the CURRENT+ conversation for text and images, so reading the opening payload for video kept every later+ round reserved at the full budget for media llama-server is no longer sent.+ """++ class _Backend:+ base_url = "http://llama"+ effective_parallel_slots = 4+ _kv_cache_context_total = 4096+ context_length = 4096+ _mmproj_projector_type = None+ _extra_args = None++ class _Reservation:+ def __init__(self, lease):+ self._lease = lease++ def lease_nowait(self):+ return self._lease++ def _recost(self, payload, conversation):+ from routes.inference import (+ _openai_llama_admission_recost,+ _openai_llama_admission_tokens,+ )+ async def _run():+ queue = LlamaAdmissionQueue("test")+ opened = _openai_llama_admission_tokens(+ payload, budget = 4096, capacity = 4, tool_loop = True+ )+ reservation = queue.reserve(+ capacity = 4,+ config = LlamaAdmissionConfig(),+ tokens = opened,+ budget = 4096,+ )+ lease = reservation.lease_nowait()+ assert lease is not None+ _openai_llama_admission_recost(+ self._Reservation(lease),+ conversation,+ request = None,+ llama_backend = self._Backend(),+ payload = payload,+ output_tokens = 64,+ )+ return queue.snapshot().committed++ return asyncio.run(_run())++ def _payload(self, clip_b64):+ return _Payload(+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "video_url",+ "video_url": {"url": "data:video/mp4;base64," + clip_b64},+ }+ ],+ }+ ],+ video_base64 = None,+ audio_base64 = None,+ image_base64 = None,+ )++ def test_the_clip_is_charged_while_the_conversation_still_carries_it(self):+ clip = "A" * 40_000+ kept = [+ {+ "role": "user",+ "content": [{"type": "input_video", "input_video": {"data": clip}}],+ }+ ]+ assert self._recost(self._payload(clip), kept) > self._recost(+ self._payload(clip), [{"role": "user", "content": "text only"}]+ )++ def test_an_evicted_clip_stops_being_charged(self):+ """The whole point: once the turn is gone the round must not still reserve for it."""+ clip = "A" * 40_000+ evicted = self._recost(self._payload(clip), [{"role": "user", "content": "text only"}])+ no_video_at_all = self._recost(+ _Payload(+ messages = [{"role": "user", "content": "text only"}],+ video_base64 = None,+ audio_base64 = None,+ image_base64 = None,+ ),+ [{"role": "user", "content": "text only"}],+ )+ assert evicted == no_video_at_alldiff --git a/studio/backend/tests/test_llama_video_frame_budget.py b/studio/backend/tests/test_llama_video_frame_budget.pyindex f8929a767ae..89ef7750b44 100644--- a/studio/backend/tests/test_llama_video_frame_budget.py+++ b/studio/backend/tests/test_llama_video_frame_budget.py@@ -412,9 +412,10 @@ def test_the_gguf_route_shrinks_the_clip_after_the_size_check_and_before_injecti)start = source.index('"Video provided but the current GGUF model cannot take video input. "')check = source.index("_video_b64_rejection(payload.video_base64)", start)- # Bare name: the call has been reflowed twice, and each time an anchor on the- # full call raised ValueError instead of reporting an ordering problem.- shrink = source.index("shrink_video_for_llama,", start)+ # The transcode moved into _shrink_clip_for_llama so the video_url parts can reuse it;+ # anchor on the route's call to that helper, not on shrink_video_for_llama, which now+ # only appears inside it (and earlier in the file).+ shrink = source.index("_shrink_clip_for_llama(video_b64, llama_backend)", start)inject = source.index("_inject_video_part(gguf_messages, video_b64)", start)assert check < shrink < inject@@ -439,6 +440,9 @@ def test_the_conversions_result_is_what_gets_injected():call = node.value if isinstance(node, ast.Await) else nodeif not isinstance(call, ast.Call):return False+ # Either spelling: the route calls the helper, and the helper calls the real thing.+ if isinstance(call.func, ast.Name) and call.func.id == "_shrink_clip_for_llama":+ return Truereturn any(isinstance(a, ast.Name) and a.id == "shrink_video_for_llama" for a in call.args)bound_to = {diff --git a/studio/backend/tests/test_preview_routes.py b/studio/backend/tests/test_preview_routes.pyindex ad0643ac531..77774ab91a0 100644--- a/studio/backend/tests/test_preview_routes.py+++ b/studio/backend/tests/test_preview_routes.py@@ -37,9 +37,13 @@ if _BACKEND_DIR not in sys.path:# Mirror test_preview.py: the real `loggers` package pulls in heavy handlers._loggers_stub = _types.ModuleType("loggers")_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)+# __path__ so `loggers.media_progress` still resolves to the real submodule. Without it the stub+# shadows the package rather than its __init__, and any module importing a submodule dies with+# "'loggers' is not a package" as soon as this file shares a pytest process with it (#10995).+_loggers_stub.__path__ = [str(Path(_BACKEND_DIR) / "loggers")]sys.modules.setdefault("loggers", _loggers_stub)-from fastapi import FastAPI+from fastapi import FastAPI, HTTPExceptionfrom fastapi.responses import StreamingResponsefrom fastapi.testclient import TestClient@@ -159,6 +163,11 @@ def test_page_recovers_from_empty_reply(client):assert "Reply cut off before the model returned an answer." in text+ sys.platform == "win32",+ reason = "'<' is not a legal Windows filename character, so a run dir named a<b cannot "+ "exist there and the escaping this covers is unreachable",+)def test_page_escapes_title(tmp_path, monkeypatch, captured):outputs = tmp_path / "outputs"# Run dir name carries an HTML-special char; the page must escape it.@@ -392,6 +401,45 @@ def test_streaming_holds_lock_until_drained(tmp_path, monkeypatch, captured):assert not preview._preview_lock.locked()+def test_a_video_clip_is_refused_before_the_checkpoint_loads(tmp_path, monkeypatch, captured):+ """A preview target cannot read a clip, and the load it would precede evicts the resident model."""+ outputs = tmp_path / "outputs"+ _make_run(outputs)+ from utils.paths import storage_roots as _sr++ monkeypatch.setattr(_sr, "outputs_root", lambda: outputs)++ loaded = []++ async def _fake_load_model(load_req, request, subject):+ loaded.append(load_req)+ return None++ async def _fake_chat(payload, request, subject):+ return {"ok": True}++ monkeypatch.setattr(preview, "load_model_for_preview", _fake_load_model)+ monkeypatch.setattr(preview, "openai_chat_completions", _fake_chat)++ payload = ChatCompletionRequest(+ messages = [+ {+ "role": "user",+ "content": [+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJD"}},+ {"type": "text", "text": "what happens here?"},+ ],+ }+ ]+ )+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(preview._serve_chat("demorun", None, payload, request = None))+ assert excinfo.value.status_code == 400+ assert "Video input" in excinfo.value.detail+ assert loaded == []+ assert not preview._preview_lock.locked()++# ── Capability gating ────────────────────────────────────────────────────────@@ -547,8 +595,6 @@ def test_chat_rate_limited_returns_429(client, monkeypatch):import threadingfrom types import SimpleNamespace-from fastapi import HTTPException-import routes.inference as inferencefrom core.inference import llama_keepwarmfrom models.inference import LoadRequestdiff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.pyindex ca677fa4d76..4ec0ecc6ad4 100644--- a/studio/backend/tests/test_video_attachment_part.py+++ b/studio/backend/tests/test_video_attachment_part.py@@ -128,7 +128,7 @@ def test_video_is_refused_on_the_tool_passthrough_path():start = source.index("if using_gguf and _takes_tool_passthrough(payload, llama_backend):")branch = source[start : start + 2500]assert "payload.audio_base64" in branch- assert "payload.video_base64" in branch+ assert "_request_has_video(payload)" in branchassert "Video input is not supported together with guided decoding" in branch@@ -138,7 +138,7 @@ def test_the_size_check_runs_before_the_automatic_switch():source = _inference_source()# Anchor inside the chat-completions handler; other routes switch too.handler = source.index("_needs_image = bool(_pre_parsed[2])")- guard = source.index("_video_b64_rejection(payload.video_base64)", handler)+ guard = source.index("_request_video_rejection(payload)", handler)switch = source.index("await _maybe_auto_switch_model(", handler)assert guard < switch@@ -150,7 +150,7 @@ def test_video_joins_the_projector_requirement_before_switching():start = source.index("_needs_image = bool(_pre_parsed[2])")switch = source.index("await _maybe_auto_switch_model(", start)assert start < source.index("payload.audio_base64", start) < switch- assert start < source.index("payload.video_base64", start) < switch+ assert start < source.index("_request_has_video(payload)", start) < switchdef test_an_external_provider_refuses_video_rather_than_ignoring_it():@@ -159,7 +159,8 @@ def test_an_external_provider_refuses_video_rather_than_ignoring_it():source = _inference_source()start = source.index("if payload.provider_id or payload.provider_type:")branch = source[start : source.index("_proxy_to_external_provider(payload", start)]- assert "payload.video_base64" in branch+ # Both spellings through one predicate, so a video_url part cannot reach the proxy either.+ assert "_request_has_video(payload)" in branchassert "_VIDEO_INPUT_REFUSAL" in branch@@ -222,6 +223,8 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_generaconflict = source.index("if payload.audio_base64:", gate)assert conflict < speechassert "_AUDIO_VIDEO_INPUT_DETAIL" in source[conflict : conflict + 200]+ # The gate reads both spellings, so a video_url part reaches MLX exactly as the field does.+ assert "if _request_has_video(payload):\n _video_clip = _local_video_clip(" in sourcedef test_token_counting_refuses_video_like_image_and_audio():@@ -238,4 +241,267 @@ def test_both_video_checks_share_one_rule():"""Two size checks that drift let the pre-switch one pass what the post-loadone refuses, which is the model load this was meant to avoid."""source = _inference_source()- assert source.count("_video_b64_rejection(payload.video_base64)") == 3+ assert source.count("= _request_video_rejection(payload)") == 2+++def _video_url_request(+ *urls,+ text = "what happens here?",+ **kw,+):+ from models.inference import ChatCompletionRequest++ parts = [{"type": "video_url", "video_url": {"url": url}} for url in urls]+ parts.append({"type": "text", "text": text})+ return ChatCompletionRequest(messages = [{"role": "user", "content": parts}], **kw)+++def test_a_video_url_part_validates_as_the_openai_shape():+ from models.inference import VideoContentPart++ req = _video_url_request("data:video/mp4;base64,QUJD", "https://example.com/clip.mp4")+ assert all(isinstance(part, VideoContentPart) for part in req.messages[0].content[:2])+ assert req.messages[0].content[1].video_url.url == "https://example.com/clip.mp4"+++def test_a_message_carried_clip_counts_as_video_input():+ from models.inference import ChatCompletionRequest+ from routes.inference import _request_has_video++ assert _request_has_video(_video_url_request("data:video/mp4;base64,QUJD")) is True+ plain = ChatCompletionRequest(messages = [{"role": "user", "content": "hi"}])+ assert _request_has_video(plain) is False+++def test_a_data_uri_part_is_translated_in_place_without_its_header():+ from routes.inference import _openai_messages_for_gguf_chat, _translate_video_parts++ messages, _ = _openai_messages_for_gguf_chat(+ _video_url_request("data:video/mp4;base64,QUJD"), True+ )+ _translate_video_parts(messages)+ assert messages[0]["content"] == [+ {"type": "input_video", "input_video": {"data": "QUJD"}},+ {"type": "text", "text": "what happens here?"},+ ]+++def test_a_remote_url_is_refused_rather_than_forwarded():+ """llama-server would fetch it from this machine, and its downloader follows redirects+ (set_follow_location, common/http.h), so a public URL redirecting to a private address+ defeats any host check made here. input_video therefore only ever carries bytes."""+ import pytest+ from fastapi import HTTPException+ from routes.inference import _translate_video_parts++ messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "and this one?"},+ {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}},+ ],+ }+ ]+ with pytest.raises(HTTPException) as exc:+ _translate_video_parts(messages)+ assert exc.value.status_code == 400+ assert "Remote video URLs are not supported" in exc.value.detail+++def test_an_uppercase_data_uri_header_is_still_stripped():+ """RFC 2397 schemes are case-insensitive; a kept header would be decoded as media."""+ from routes.inference import _translate_video_parts++ messages = [+ {+ "role": "user",+ "content": [{"type": "video_url", "video_url": {"url": "DATA:video/mp4;base64,QUJD"}}],+ }+ ]+ _translate_video_parts(messages)+ assert messages[0]["content"][0] == {"type": "input_video", "input_video": {"data": "QUJD"}}+++def test_an_uppercase_scheme_is_refused_as_a_remote_url_too():+ """Schemes are case-insensitive, so HTTPS:// must be recognised as remote and refused+ rather than falling through and being decoded as base64."""+ import pytest+ from fastapi import HTTPException+ from routes.inference import _translate_video_parts++ for url in ("HTTPS://example.com/clip.mp4", "Http://example.com/other.mp4"):+ messages = [{"role": "user", "content": [{"type": "video_url", "video_url": {"url": url}}]}]+ with pytest.raises(HTTPException) as exc:+ _translate_video_parts(messages)+ assert exc.value.status_code == 400+++def test_message_parts_are_translated_where_the_legacy_clip_is_injected():+ """Both spellings reach llama-server from one place, after the capability gate.++ The legacy clip is injected first and the translation is the last thing before dispatch, so+ one call covers both. Asserted as order rather than adjacency: the destination guard runs+ between them, and pinning the exact neighbouring line only broke on that.+ """+ source = _inference_source()+ inject = source.index("_inject_video_part(gguf_messages, video_b64)")+ translate = source.index("_translate_video_parts(gguf_messages)")+ assert inject < translate+ assert source.count("_translate_video_parts(gguf_messages)") == 1+++def test_every_data_uri_part_is_sized_not_only_the_first():+ from routes.inference import _MAX_VIDEO_B64_CHARS, _request_video_rejection++ fits = _video_url_request("data:video/mp4;base64,QUJD", "data:video/mp4;base64,QUJD")+ assert _request_video_rejection(fits) is None+ oversized = "data:video/mp4;base64," + "A" * (_MAX_VIDEO_B64_CHARS + 1)+ too_big = _video_url_request("data:video/mp4;base64,QUJD", oversized)+ assert _request_video_rejection(too_big) == (413, "Video file is too large (max 64 MB).")+ assert _request_video_rejection(_video_url_request("")) == (+ 400,+ "Could not read the provided video file.",+ )+++def test_the_legacy_field_is_sized_beside_a_message_part():+ from routes.inference import _MAX_VIDEO_B64_CHARS, _request_video_rejection+ req = _video_url_request(+ "https://example.com/clip.mp4", video_base64 = "A" * (_MAX_VIDEO_B64_CHARS + 1)+ )+ assert _request_video_rejection(req) == (413, "Video file is too large (max 64 MB).")+++def test_admission_prices_a_message_clip_as_media_not_prompt_text():+ from routes.inference import (+ _openai_llama_admission_media_tokens,+ _openai_llama_admission_messages_for_estimate,+ )++ req = _video_url_request("data:video/mp4;base64," + "A" * 4000)+ estimate, image_parts = _openai_llama_admission_messages_for_estimate(req.messages)+ assert estimate[0]["content"][0] == {"type": "video_url", "video_url": {"url": "[video]"}}+ assert image_parts == 0+ assert _openai_llama_admission_media_tokens(req) >= 1000+++def test_the_rolling_context_does_not_price_a_video_url_part():+ from core.inference.context_window import estimate_message_tokens_without_unpriced_media+ message = {+ "role": "user",+ "content": [+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64," + "A" * 40000}},+ {"type": "text", "text": "hi"},+ ],+ }+ assert estimate_message_tokens_without_unpriced_media(message) < 100+++def test_a_message_clip_reaches_the_switch_as_video(monkeypatch):+ """Through the handler: the pre-switch gate must see a message-carried clip."""+ import asyncio++ import routes.inference as inference_route+ from utils import openai_auto_switch_settings as settings++ class _Reached(Exception):+ pass++ captured = {}++ async def _capture(model, request, subject, **kw):+ captured.update(kw)+ raise _Reached()++ monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True)+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _capture)+ payload = _video_url_request("data:video/mp4;base64,QUJD", model = "org/B-GGUF")+ with pytest.raises(_Reached):+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))+ assert captured["require_video"] is True+ assert captured["require_vision"] is True+ assert captured["modality_label"] == "video"+++def test_every_clip_in_every_turn_is_translated():+ from routes.inference import _translate_video_parts++ messages = [+ {+ "role": "user",+ "content": [+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJD"}},+ {"type": "text", "text": "first"},+ ],+ },+ {"role": "assistant", "content": "ok"},+ {+ "role": "user",+ "content": [+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJE"}},+ {"type": "video_url", "video_url": {"url": "data:video/webm;base64,REVG"}},+ ],+ },+ ]+ _translate_video_parts(messages)+ assert messages[0]["content"][0] == {"type": "input_video", "input_video": {"data": "QUJD"}}+ assert messages[2]["content"] == [+ {"type": "input_video", "input_video": {"data": "QUJE"}},+ {"type": "input_video", "input_video": {"data": "REVG"}},+ ]+++def test_admission_charges_each_clip_once():+ """Every clip is base64 now, so each is priced by its own length and none is double counted."""+ from routes.inference import _openai_llama_admission_media_tokens++ first = "data:video/mp4;base64," + "A" * 4000+ second = "data:video/mp4;base64," + "B" * 8000+ req = _video_url_request(first, second)+ assert _openai_llama_admission_media_tokens(req) == len(first) // 4 + len(second) // 4+++def test_a_clip_on_any_non_user_role_is_refused():+ """Media outside a user turn renders template-dependently, so it is refused up front."""+ import pydantic++ from models.inference import ChatCompletionRequest++ parts = [+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJD"}},+ {"type": "text", "text": "watch this before answering"},+ ]+ for message in (+ {"role": "system", "content": parts},+ {"role": "developer", "content": parts},+ {"role": "assistant", "content": parts},+ {"role": "tool", "tool_call_id": "call_1", "content": parts},+ ):+ refusal = f'not valid on role="{message["role"]}"'+ with pytest.raises(pydantic.ValidationError, match = refusal):+ ChatCompletionRequest(messages = [{"role": "user", "content": "go"}, message])+ ChatCompletionRequest(messages = [{"role": "user", "content": parts}])+++def test_an_oversized_message_clip_is_refused_before_the_switch(monkeypatch):+ import asyncio++ from fastapi import HTTPException++ import routes.inference as inference_route+ from routes.inference import _MAX_VIDEO_B64_CHARS+ from utils import openai_auto_switch_settings as settings++ async def _switch(*args, **kw):+ raise AssertionError("the model switch ran before the size check")++ monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True)+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _switch)+ payload = _video_url_request(+ "data:video/mp4;base64," + "A" * (_MAX_VIDEO_B64_CHARS + 1),+ model = "org/B-GGUF",+ )+ with pytest.raises(HTTPException) as info:+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))+ assert info.value.status_code == 413diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.pynew file mode 100644index 00000000000..9c1cb31e5ad--- /dev/null+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py@@ -0,0 +1,936 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""What a client gets back when it attaches a clip, by status code and wire shape.++The other video tests assert a line is present in ``routes/inference.py``, which is how the+non-GGUF regression went unnoticed: the source assertion passed while the request 400'd. These+take the ``messages`` kwarg of ``generate_chat_completion`` as the JSON llama-server receives.++Invariant, asserted per route: a ``video_url`` part and the legacy ``video_base64`` field behave+identically, so picking a spelling cannot pick a different set of refusals.++Wire shapes follow llama.cpp ``handle_media``: ``input_video`` is ``data`` else ``url``, remote+downloads cap at 10 MB.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++pytest.importorskip("torch")++from fastapi import FastAPI # noqa: E402+from fastapi.testclient import TestClient # noqa: E402++from auth.authentication import get_current_subject # noqa: E402+import routes.inference as inference_route # noqa: E402++from .llama_backend_double import FakeLlamaCppBackend # noqa: E402++_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header+_DATA_URI = f"data:video/mp4;base64,{_CLIP_B64}"+_REMOTE = "https://example.com/clip.mp4"+++class _VideoGguf(FakeLlamaCppBackend):+ """A loaded GGUF whose llama-server /props reported ``modalities.video``."""++ is_vision = True+ _has_video_input = True++ def __init__(self, *, has_video = True):+ self._has_video_input = has_video+ self.dispatched: list[dict] = []++ def generate_chat_completion(self, **kwargs):+ self.dispatched.append(kwargs)+ yield "ok"+ yield {"type": "metadata", "usage": {}, "timings": {}}+++def _no_gguf():+ """The resident backend is transformers or MLX, so ``using_gguf`` is False."""+ return SimpleNamespace(+ is_loaded = False, supports_tools = False, is_vision = False, context_length = None+ )+++def _client(+ monkeypatch,+ backend = None,+ *,+ prefix = "/v1",+):+ async def _no_switch(*_a, **_k):+ return None++ monkeypatch.setattr(+ inference_route, "get_llama_cpp_backend", lambda: backend if backend else _no_gguf()+ )+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_switch)++ app = FastAPI()+ app.include_router(inference_route.router, prefix = prefix)+ app.dependency_overrides[get_current_subject] = lambda: "tester"+ return TestClient(app, raise_server_exceptions = False)+++def _part_body(+ *urls,+ text = "what happens here?",+ **extra,+):+ parts = [{"type": "video_url", "video_url": {"url": u}} for u in urls]+ parts.append({"type": "text", "text": text})+ return {+ "model": "test/model.gguf",+ "stream": False,+ "messages": [{"role": "user", "content": parts}],+ **extra,+ }+++def _field_body(+ clip = _DATA_URI,+ text = "what happens here?",+ **extra,+):+ return {+ "model": "test/model.gguf",+ "stream": False,+ "messages": [{"role": "user", "content": [{"type": "text", "text": text}]}],+ "video_base64": clip,+ **extra,+ }+++def _detail(response) -> str:+ """The human message, from a bare detail or an OpenAI error body.++ Falls back to raw text, since a request that gets past the refusals answers with SSE or an+ empty body and a test asserting a refusal is *absent* must read those too.+ """+ try:+ body = response.json()+ except ValueError:+ return response.text+ if not isinstance(body, dict):+ return str(body)+ detail = body.get("detail", body)+ if isinstance(detail, dict):+ return str(detail.get("error", detail).get("message", detail))+ return str(detail)+++def _sent_parts(backend, index = -1):+ """Parts of a dispatched user turn. A system turn is prepended, so index from the user ones."""+ user_turns = [+ m+ for m in backend.dispatched[0]["messages"]+ if m.get("role") == "user" and isinstance(m.get("content"), list)+ ]+ return user_turns[index]["content"]+++def _sent_media(backend):+ """Every input_video part across every dispatched turn, in order."""+ return [+ part+ for message in backend.dispatched[0]["messages"]+ if isinstance(message.get("content"), list)+ for part in message["content"]+ if isinstance(part, dict) and part.get("type") == "input_video"+ ]+++ "url, expected",+ [+ (_DATA_URI, {"data": _CLIP_B64}),+ # The header is not payload; llama.cpp's own base64 decoder takes the body alone.+ (f"DATA:video/mp4;BASE64,{_CLIP_B64}", {"data": _CLIP_B64}),+ ("data:video/webm;base64,REVG", {"data": "REVG"}),+ # Bare base64 with no header: handle_media's final fallback treats it as payload.+ (_CLIP_B64, {"data": _CLIP_B64}),+ ],+)+def test_a_clip_reaches_llama_server_as_input_video(monkeypatch, url, expected):+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ response = client.post("/v1/chat/completions", json = _part_body(url))+ assert response.status_code == 200+ assert _sent_media(backend) == [{"type": "input_video", "input_video": expected}]+++def test_the_translated_part_is_the_only_video_key_left(monkeypatch):+ """A part left spelled video_url would be refused by llama-server as an unsupported type:+ it accepts image_url, input_audio and input_video, and nothing else."""+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI))+ types = [p.get("type") for p in _sent_parts(backend)]+ assert "video_url" not in types+ assert len(_sent_media(backend)) == 1+++def test_the_text_of_the_turn_survives_translation(monkeypatch):+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI, text = "what colour?"))+ assert {"type": "text", "text": "what colour?"} in _sent_parts(backend)+++def test_every_clip_in_a_turn_is_translated(monkeypatch):+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ client.post(+ "/v1/chat/completions", json = _part_body(_DATA_URI, "data:video/webm;base64,REVG")+ )+ assert [p["input_video"] for p in _sent_media(backend)] == [+ {"data": _CLIP_B64},+ {"data": "REVG"},+ ]+++def test_a_clip_in_an_older_turn_is_translated_too(monkeypatch):+ """_inject_video_part only ever touches the newest user turn, so a parts-carried clip on an+ earlier turn would otherwise reach llama-server still spelled video_url."""+ backend = _VideoGguf()+ body = {+ "model": "test/model.gguf",+ "stream": False,+ "messages": [+ {"role": "user", "content": [{"type": "video_url", "video_url": {"url": _DATA_URI}}]},+ {"role": "assistant", "content": "a red square"},+ {"role": "user", "content": [{"type": "text", "text": "and now?"}]},+ ],+ }+ with _client(monkeypatch, backend) as client:+ client.post("/v1/chat/completions", json = body)+ # Translated in place, on the turn that carried it.+ assert _sent_parts(backend, 0)[0]["type"] == "input_video"+ assert len(_sent_media(backend)) == 1+++def test_the_legacy_field_still_rides_the_newest_user_turn(monkeypatch):+ """Backwards compatibility: the spelling the Studio frontend sends is unchanged."""+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ response = client.post("/v1/chat/completions", json = _field_body())+ assert response.status_code == 200+ assert _sent_parts(backend)[-1] == {"type": "input_video", "input_video": {"data": _CLIP_B64}}+++def test_both_spellings_of_the_same_clip_produce_the_same_wire_shape(monkeypatch):+ """The invariant the whole change rests on, asserted on the dispatched body itself."""+ part_backend, field_backend = _VideoGguf(), _VideoGguf()+ with _client(monkeypatch, part_backend) as client:+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI))+ with _client(monkeypatch, field_backend) as client:+ client.post("/v1/chat/completions", json = _field_body())++ assert _sent_media(part_backend) == _sent_media(field_backend)++[email protected]("body", [_part_body(_DATA_URI), _field_body()])+def test_a_gguf_without_a_video_projector_refuses_either_spelling(monkeypatch, body):+ with _client(monkeypatch, _VideoGguf(has_video = False)) as client:+ response = client.post("/v1/chat/completions", json = body)+ assert response.status_code == 400+ assert "cannot take video input" in _detail(response)++[email protected]("body", [_part_body(_DATA_URI), _field_body()])+def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(monkeypatch, body):+ """The regression guard. A blanket 'not using_gguf' refusal ahead of _local_video_clip+ refuses video on transformers and MLX for BOTH spellings, including the legacy field the+ frontend sends, breaking working MLX video on macOS.+ """+ reached: list[dict] = []++ def _gate(payload, model_info):+ reached.append(model_info)+ return _CLIP_B64++ monkeypatch.setattr(+ inference_route,+ "get_inference_backend",+ lambda: SimpleNamespace(+ active_model_name = "mlx-model",+ models = {"mlx-model": {"is_vision": True, "has_video_input": True}},+ ),+ )+ monkeypatch.setattr(inference_route, "_local_video_clip", _gate)+ with _client(monkeypatch, None) as client:+ response = client.post("/v1/chat/completions", json = body)++ # Reached-the-gate alone cannot tell the regression from the fix: a blanket refusal placed+ # after the gate still lets the gate run.+ assert reached == [{"is_vision": True, "has_video_input": True}]+ assert "only supported on a local GGUF model" not in _detail(response)+ assert inference_route._VIDEO_INPUT_REFUSAL not in _detail(response)+++def test_the_non_gguf_gate_reads_both_spellings(monkeypatch):+ """_local_video_clip predates the part, so left reading payload.video_base64 alone it would+ hand an MLX model nothing while the legacy field worked."""+ from models.inference import ChatCompletionRequest++ info = {"is_vision": True, "has_video_input": True}+ field = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)+ part = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))+ assert inference_route._local_video_clip(field, info) == _CLIP_B64+ assert inference_route._local_video_clip(part, info) == _CLIP_B64+++def test_a_non_gguf_backend_without_video_refuses_by_name():+ from fastapi import HTTPException+ from models.inference import ChatCompletionRequest++ payload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))+ with pytest.raises(HTTPException) as exc:+ inference_route._local_video_clip(payload, {"is_vision": True})+ assert exc.value.status_code == 400+ assert exc.value.detail == inference_route._VIDEO_INPUT_REFUSAL+ # The refusal names MLX: that backend serves video, so a GGUF-only message misinforms.+ assert "MLX" in inference_route._VIDEO_INPUT_REFUSAL+++def test_a_second_clip_is_refused_rather_than_dropped_on_a_non_gguf_backend():+ """Generation takes one video kwarg. Taking clips[0] silently changed the prompt by+ backend: GGUF forwards every clip, so the same request meant two different things."""+ from fastapi import HTTPException+ from models.inference import ChatCompletionRequest++ info = {"is_vision": True, "has_video_input": True}+ two = ChatCompletionRequest.model_validate(_part_body(_DATA_URI, _DATA_URI))+ with pytest.raises(HTTPException) as exc:+ inference_route._local_video_clip(two, info)+ assert exc.value.status_code == 400+ assert "Only one video" in exc.value.detail++ # The legacy field beside a part is the same two-clip request in a different spelling.+ both = ChatCompletionRequest.model_validate(+ _part_body(_DATA_URI, **{"video_base64": _DATA_URI})+ )+ with pytest.raises(HTTPException) as exc:+ inference_route._local_video_clip(both, info)+ assert exc.value.status_code == 400++ # One clip still serves, so the guard did not swallow the ordinary case.+ one = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))+ assert inference_route._local_video_clip(one, info) == _CLIP_B64+++def test_an_unsupported_scheme_is_refused_on_the_non_gguf_path_too():+ """_request_video_rejection runs only when a pre-switch validation happens, so with+ auto-switch off a file:// clip reached the backend and was decoded as base64."""+ from fastapi import HTTPException+ from models.inference import ChatCompletionRequest++ payload = ChatCompletionRequest.model_validate(_part_body("file:///etc/passwd"))+ with pytest.raises(HTTPException) as exc:+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ assert exc.value.status_code == 400+ assert "Unsupported video URL scheme" in exc.value.detail+++def test_admission_compacts_a_clip_after_translation_too():+ """The server-side tool loop recosts the conversation after _translate_video_parts has+ renamed the part, so matching video_url alone priced the payload as dense prompt text."""+ import copy++ big = "data:video/mp4;base64," + "A" * 40_000+ msgs = [+ {+ "role": "user",+ "content": [+ {"type": "video_url", "video_url": {"url": big}},+ {"type": "text", "text": "hi"},+ ],+ }+ ]++ translated = copy.deepcopy(msgs)+ inference_route._translate_video_parts(translated)+ before, _ = inference_route._openai_llama_admission_messages_for_estimate(copy.deepcopy(msgs))+ after, _ = inference_route._openai_llama_admission_messages_for_estimate(translated)++ assert "A" * 40_000 not in str(after)+ # Both spellings compact to about the same size; the clip is a marker, not prompt text.+ assert abs(len(str(after)) - len(str(before))) < 200+++def test_a_remote_clip_is_refused_on_a_non_gguf_backend():+ """Only llama-server fetches a clip for itself. A transformers or MLX model is handed bytes,+ so forwarding the URL would feed it the text of the URL instead of the video."""+ from fastapi import HTTPException+ from models.inference import ChatCompletionRequest++ payload = ChatCompletionRequest.model_validate(_part_body(_REMOTE))+ with pytest.raises(HTTPException) as exc:+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ assert exc.value.status_code == 400+ assert "Remote video URLs are not supported" in exc.value.detail+++ "url",+ [+ "file:///etc/passwd",+ "FILE:///etc/passwd",+ "ftp://example.com/clip.mp4",+ "gopher://example.com/clip.mp4",+ ],+)+def test_an_unsupported_scheme_is_refused_by_name(monkeypatch, url):+ """handle_media reads input_video as data-or-url and treats the one string it finds the+ same way, honouring file:// under --media-path. Forwarding the clip as opaque payload does+ not neutralise the scheme, so refuse it here.+ """+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = _part_body(url))+ assert response.status_code == 400+ assert "Unsupported video URL scheme" in _detail(response)+++def test_a_bare_path_is_refused_without_naming_a_scheme(monkeypatch):+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = _part_body("/tmp/clip.mp4"))+ # No colon: indistinguishable from base64 payload, so llama-server owns the verdict.+ assert response.status_code == 200+++def test_bare_base64_is_not_mistaken_for_a_scheme(monkeypatch):+ """':' is not in the base64 alphabet, which is what separates a URL from payload. A scheme+ check that scanned the whole string would refuse legitimate clips."""+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ response = client.post("/v1/chat/completions", json = _part_body("QUJDREVGR0hJSktM"))+ assert response.status_code == 200+ assert _sent_media(backend) == [+ {"type": "input_video", "input_video": {"data": "QUJDREVGR0hJSktM"}}+ ]+++def test_an_oversized_clip_is_refused_in_either_spelling():+ """On the shared rule rather than over HTTP, since the body would be 85 MB of JSON. The+ ordering guarantee is pinned in test_video_attachment_part.py."""+ from models.inference import ChatCompletionRequest+ from routes.inference import _MAX_VIDEO_B64_CHARS++ over = "data:video/mp4;base64," + "A" * (_MAX_VIDEO_B64_CHARS + 1)+ for payload in (+ ChatCompletionRequest.model_validate(_part_body(over)),+ ChatCompletionRequest.model_validate(_field_body(over)),+ ):+ assert inference_route._request_video_rejection(payload)[0] == 413+++def test_a_clip_of_exactly_the_cap_is_admitted():+ """The boundary the cap exists to allow: refusing it would reject a file the composer offers."""+ from models.inference import ChatCompletionRequest+ from routes.inference import _MAX_VIDEO_B64_CHARS++ at_cap = "data:video/mp4;base64," + "A" * _MAX_VIDEO_B64_CHARS+ payload = ChatCompletionRequest.model_validate(_part_body(at_cap))+ assert inference_route._request_video_rejection(payload) is None+++def test_a_data_uri_with_no_payload_is_refused(monkeypatch):+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = _part_body("data:video/mp4;base64,"))+ assert response.status_code == 400+ assert "Could not read the provided video file" in _detail(response)++[email protected]("body", [_part_body(_DATA_URI), _field_body()])+def test_an_external_provider_refuses_either_spelling(monkeypatch, body):+ """_build_external_messages rebuilds from an allowlist, so a clip left standing is dropped+ and the provider answers a prompt the caller did not send."""+ body = {**body, "provider_type": "openai", "provider_api_key": "sk-test"}+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = body)+ assert response.status_code == 400+ assert response.json()["detail"] == inference_route._VIDEO_INPUT_REFUSAL++[email protected]("body", [_part_body(_DATA_URI), _field_body()])+def test_token_counting_refuses_either_spelling(monkeypatch, body):+ """llama-server samples the frames at completion time, so counting here undercounts."""+ body = {k: v for k, v in body.items() if k != "stream"}+ with _client(monkeypatch, _VideoGguf(), prefix = "") as client:+ response = client.post("/chat/count_tokens", json = body)+ assert response.status_code == 503+ assert "video" in _detail(response)++[email protected]("body", [_part_body(_DATA_URI), _field_body()])+def test_the_speech_route_refuses_a_clip_rather_than_speaking_past_it(monkeypatch, body):+ """/audio/generate keeps only text, so without this the clip was dropped in silence.+ Registering video_url as a known tag removed the unknown-part guard that covered it."""+ body = {k: v for k, v in body.items() if k != "stream"}+ with _client(monkeypatch, _VideoGguf(), prefix = "") as client:+ response = client.post("/audio/generate", json = body)+ assert response.status_code == 400+ assert "Video input is not supported here" in _detail(response)+++def test_the_speech_route_still_speaks_a_plain_text_turn(monkeypatch):+ """The control for the refusal above: it must not swallow ordinary requests."""+ body = {"model": "default", "messages": [{"role": "user", "content": "read this out"}]}+ with _client(monkeypatch, _VideoGguf(), prefix = "") as client:+ response = client.post("/audio/generate", json = body)+ assert "Video input is not supported here" not in _detail(response)++[email protected]("body", [_part_body(_DATA_URI), _field_body()])+def test_the_tool_passthrough_path_refuses_either_spelling(monkeypatch, body):+ """That branch forwards an explicit field list, so the clip would be dropped silently."""+ body = {+ **body,+ "tools": [+ {+ "type": "function",+ "function": {"name": "f", "parameters": {"type": "object", "properties": {}}},+ }+ ],+ }++ class _ToolGguf(_VideoGguf):+ supports_tools = True++ with _client(monkeypatch, _ToolGguf()) as client:+ response = client.post("/v1/chat/completions", json = body)+ assert response.status_code == 400+ assert "guided decoding" in _detail(response)+++def test_a_durable_chat_run_refuses_a_clip_in_either_spelling():+ """The request payload persists verbatim, so a data URI would sit in request_json for the+ life of the thread. The gate is field-shaped, so the part needs its own predicate."""+ from fastapi import HTTPException+ from routes.chat_generation_runs import CreateChatGenerationRun, _sanitize_request++ for body in (_part_body(_DATA_URI), _field_body()):+ run = CreateChatGenerationRun(+ runId = "run-1",+ threadId = "thread-1",+ userMessageId = "user-1",+ assistantMessageId = "assistant-1",+ requestPayload = {k: v for k, v in body.items() if k != "stream"},+ )+ with pytest.raises(HTTPException) as exc:+ _sanitize_request(run)+ assert exc.value.status_code == 400+ assert "Media chat runs" in str(exc.value.detail)+++def test_a_durable_chat_run_still_takes_a_plain_text_turn():+ """The control: the media gate must not start refusing ordinary durable runs."""+ from routes.chat_generation_runs import CreateChatGenerationRun, _sanitize_request++ run = CreateChatGenerationRun(+ runId = "run-1",+ threadId = "thread-1",+ userMessageId = "user-1",+ assistantMessageId = "assistant-1",+ requestPayload = {"model": "m", "messages": [{"role": "user", "content": "hi"}]},+ )+ assert _sanitize_request(run) is not None++[email protected]("role", ["system", "assistant"])+def test_a_clip_on_a_non_user_turn_is_refused(monkeypatch, role):+ """OpenAI places media on user turns only, and llama-server renders the marker into whichever+ turn carried it, so anywhere else the result is template-dependent."""+ body = {+ "model": "test/model.gguf",+ "stream": False,+ "messages": [+ {"role": role, "content": [{"type": "video_url", "video_url": {"url": _DATA_URI}}]},+ {"role": "user", "content": "go"},+ ],+ }+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = body)+ assert response.status_code == 422+++def test_a_clip_on_a_user_turn_is_accepted(monkeypatch):+ """The control: the role rule must not refuse the placement it exists to protect."""+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = _part_body(_DATA_URI))+ assert response.status_code == 200+++def test_admission_charges_each_clip_once_and_not_as_prompt_text():+ """Pricing a megabytes-long data URI as prompt text would swamp the estimate."""+ from models.inference import ChatCompletionRequest++ big = "data:video/mp4;base64," + "A" * 40_000+ one = ChatCompletionRequest.model_validate(_part_body(big))+ two = ChatCompletionRequest.model_validate(_part_body(big, big))+ charge_one = inference_route._openai_llama_admission_media_tokens(one)+ charge_two = inference_route._openai_llama_admission_media_tokens(two)+ assert charge_two - charge_one == pytest.approx(charge_one, rel = 0.05)++ estimate, _ = inference_route._openai_llama_admission_messages_for_estimate(+ [m.model_dump(exclude_none = True) for m in one.messages]+ )+ assert "A" * 40_000 not in str(estimate)+++def test_admission_prices_the_two_spellings_of_one_clip_alike():+ from models.inference import ChatCompletionRequest++ clip = "data:video/mp4;base64," + "A" * 40_000+ part = ChatCompletionRequest.model_validate(_part_body(clip))+ field = ChatCompletionRequest.model_validate(_field_body(clip))+ assert inference_route._openai_llama_admission_media_tokens(+ part+ ) == inference_route._openai_llama_admission_media_tokens(field)+++def test_the_rolling_context_does_not_price_a_clip_as_text():+ """Trimming counts a turn to decide what to drop; a clip counted as text would evict the+ conversation around it."""+ from core.inference.context_window import _UNPRICED_MEDIA_TYPES+ assert {"video_url", "input_video"} <= set(_UNPRICED_MEDIA_TYPES)+++def test_an_unknown_part_type_is_still_refused_by_name(monkeypatch):+ """video_url used to land here. Registering it must not have widened the catch-all: an+ unregistered type still has to be refused rather than silently dropped."""+ body = {+ "model": "test/model.gguf",+ "stream": False,+ "messages": [+ {"role": "user", "content": [{"type": "hologram_url", "hologram_url": {"url": "x"}}]}+ ],+ }+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = body)+ assert response.status_code == 400+ assert "hologram_url" in _detail(response)+++ "url",+ [+ "https://example.com/clip.mp4",+ "http://example.com/clip.mp4",+ "HTTPS://EXAMPLE.COM/Clip.MP4",+ "http://169.254.169.254/latest/meta-data/",+ "http://127.0.0.1:8080/clip.mp4",+ "http://[::1",+ "https://clips.example:bad/clip.mp4",+ ],+)+def test_a_remote_video_url_is_refused(monkeypatch, url):+ """llama-server, not Studio, would fetch the clip, and its downloader follows redirects+ (set_follow_location in common/http.h), so a public URL redirecting to a private address+ defeats any host check made here. The shape is refused rather than guarded."""+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = _part_body(url))+ assert response.status_code == 400+ assert "Remote video URLs are not supported" in _detail(response)+++def test_no_url_ever_reaches_llama_server_as_a_video(monkeypatch):+ """The refusal is the point: input_video must always carry bytes, never something to dial."""+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI))+ assert all("data" in part["input_video"] for part in _sent_media(backend))+ assert not any("url" in part["input_video"] for part in _sent_media(backend))+++def test_the_dispatch_boundary_refuses_a_remote_url_on_its_own():+ """_request_video_rejection only runs on a pre-switch validation, so translation has to+ refuse it too rather than trust that it was already checked."""+ from fastapi import HTTPException++ messages = [{"role": "user", "content": [{"type": "video_url", "video_url": {"url": _REMOTE}}]}]+ with pytest.raises(HTTPException) as exc:+ inference_route._translate_video_parts(messages)+ assert exc.value.status_code == 400+ assert "Remote video URLs are not supported" in exc.value.detail+++def test_both_spellings_refuse_a_remote_clip_alike(monkeypatch):+ with _client(monkeypatch, _VideoGguf()) as client:+ part = client.post("/v1/chat/completions", json = _part_body(_REMOTE))+ field = client.post("/v1/chat/completions", json = _field_body(_REMOTE))+ assert part.status_code == field.status_code == 400+ assert _detail(part) == _detail(field)+++def _older_turn_body(clip = _DATA_URI):+ return {+ "model": "test/model.gguf",+ "stream": False,+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "video_url", "video_url": {"url": clip}},+ {"type": "text", "text": "what is this?"},+ ],+ },+ {"role": "assistant", "content": "a clip"},+ {"role": "user", "content": [{"type": "text", "text": "and now?"}]},+ ],+ }+++def test_a_clip_on_an_older_turn_is_refused_on_a_non_gguf_backend():+ """GGUF keeps the part on the turn that carried it; the non-GGUF path flattens the parts and+ hands generation one video kwarg, which MLX attaches to the newest turn. Refusing keeps the+ two backends from reading the same request as different conversations."""+ from fastapi import HTTPException+ from models.inference import ChatCompletionRequest++ payload = ChatCompletionRequest.model_validate(_older_turn_body())+ with pytest.raises(HTTPException) as exc:+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ assert exc.value.status_code == 400+ assert "attached to the latest message" in exc.value.detail+++def test_a_clip_on_the_latest_turn_is_still_served_on_a_non_gguf_backend():+ from models.inference import ChatCompletionRequest+ payload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))+ assert (+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ == _CLIP_B64+ )+++def test_the_legacy_field_is_not_treated_as_an_older_turn():+ """video_base64 has no turn to belong to, so it is always the newest attachment."""+ from models.inference import ChatCompletionRequest++ payload = ChatCompletionRequest.model_validate(_field_body())+ assert inference_route._video_is_on_an_older_turn(payload) is False+ assert (+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})+ == _CLIP_B64+ )+++def test_gguf_still_keeps_a_clip_on_the_turn_that_carried_it(monkeypatch):+ """The refusal above is a non-GGUF limit, not a new rule for everyone."""+ backend = _VideoGguf()+ with _client(monkeypatch, backend) as client:+ response = client.post("/v1/chat/completions", json = _older_turn_body())+ assert response.status_code == 200+ turns = [+ m+ for m in backend.dispatched[0]["messages"]+ if m.get("role") == "user" and isinstance(m.get("content"), list)+ ]+ assert [p.get("type") for p in turns[0]["content"]] == ["input_video", "text"]+ assert [p.get("type") for p in turns[-1]["content"]] == ["text"]+++def test_validation_does_not_copy_the_clip():+ """Validation runs twice per request and only wants the verdict, so slicing the header off+ to get it copied the whole payload each time: 89 MB for a clip at the limit."""+ import tracemalloc++ from models.inference import ChatCompletionRequest++ clip = "data:video/mp4;base64," + "A" * (8 * 1024 * 1024)+ payload = ChatCompletionRequest.model_validate(_part_body(clip))+ tracemalloc.start()+ try:+ assert inference_route._request_video_rejection(payload) is None+ _, peak = tracemalloc.get_traced_memory()+ finally:+ tracemalloc.stop()+ assert peak < 1024 * 1024, f"validation allocated {peak} bytes for an 8 MB clip"+++ "clip, expected",+ [+ ("data:video/mp4;base64," + "A" * 40, None),+ ("A" * 40, None),+ ("", (400, "Could not read the provided video file.")),+ ("data:video/mp4;base64,", (400, "Could not read the provided video file.")),+ ("data:video/mp4;base64", (400, "Could not read the provided video file.")),+ ],+)+def test_the_measured_verdict_matches_the_sliced_one(clip, expected):+ """The fast path has to agree with _video_b64_rejection, header handling included."""+ assert inference_route._video_size_rejection(clip) == expected+ assert inference_route._video_b64_rejection(clip)[1] == expected+++def test_the_measured_verdict_agrees_on_the_cap_boundary():+ limit = inference_route._MAX_VIDEO_B64_CHARS+ for length in (limit, limit + 1):+ clip = "data:video/mp4;base64," + "A" * length+ assert (+ inference_route._video_size_rejection(clip)+ == (inference_route._video_b64_rejection(clip)[1])+ )+++def test_a_part_carried_clip_is_transcoded_like_the_legacy_field(monkeypatch):+ """main shrinks the clip before llama-server samples it, but only on the legacy field's+ path. Without the same pass over the parts, the same clip would reach llama-server at a+ different resolution depending on which spelling carried it."""+ seen = []++ def _shrink(+ clip,+ cap,+ *,+ sampled_fps = None,+ ):+ seen.append(clip)+ return "SHRUNK"++ monkeypatch.setattr(inference_route, "shrink_video_for_llama", _shrink)++ part_backend = _VideoGguf()+ with _client(monkeypatch, part_backend) as client:+ assert client.post("/v1/chat/completions", json = _part_body(_DATA_URI)).status_code == 200+ field_backend = _VideoGguf()+ with _client(monkeypatch, field_backend) as client:+ assert client.post("/v1/chat/completions", json = _field_body()).status_code == 200++ assert seen == [_CLIP_B64, _CLIP_B64], "both spellings must reach the transcoder"+ assert _sent_media(part_backend) == _sent_media(field_backend)+ assert _sent_media(part_backend) == [{"type": "input_video", "input_video": {"data": "SHRUNK"}}]+++def test_several_clips_are_capped_in_aggregate_not_only_per_clip():+ """Every clip is transcoded and shrink_video_for_llama allows 300s each, so N clips just+ under the per-clip limit is N*300s of ffmpeg for one request."""+ from models.inference import ChatCompletionRequest++ half = "data:video/mp4;base64," + "A" * (inference_route._MAX_VIDEO_B64_CHARS // 2 + 10)+ one = ChatCompletionRequest.model_validate(_part_body(half))+ assert inference_route._request_video_rejection(one) is None+ two = ChatCompletionRequest.model_validate(_part_body(half, half))+ rejection = inference_route._request_video_rejection(two)+ assert rejection is not None and rejection[0] == 413+ assert "per request" in rejection[1]+++def test_the_aggregate_cap_counts_the_legacy_field_too():+ """Otherwise the field plus a part could exceed it together."""+ from models.inference import ChatCompletionRequest++ half = "data:video/mp4;base64," + "A" * (inference_route._MAX_VIDEO_B64_CHARS // 2 + 10)+ body = _part_body(half)+ body["video_base64"] = half+ rejection = inference_route._request_video_rejection(ChatCompletionRequest.model_validate(body))+ assert rejection is not None and rejection[0] == 413+++def test_recosting_drops_a_clip_the_conversation_has_evicted():+ """truncate_oldest can evict the turn that carried the clip. Charging it from the opening+ payload kept every later round reserved at the full budget for media no longer sent."""+ from models.inference import ChatCompletionRequest++ clip = "data:video/mp4;base64," + "A" * 40_000+ payload = ChatCompletionRequest.model_validate(_part_body(clip))+ opening = inference_route._openai_llama_admission_media_tokens(payload)+ assert opening > 1000++ evicted = inference_route._openai_llama_admission_media_tokens(+ payload, message_video_clips = inference_route._conversation_video_clips([])+ )+ assert evicted == 0+++def test_recosting_still_charges_a_clip_the_conversation_kept():+ """Reading video_url alone would find nothing post-translation and charge no video at all."""+ from models.inference import ChatCompletionRequest++ clip = "A" * 40_000+ payload = ChatCompletionRequest.model_validate(_part_body("data:video/mp4;base64," + clip))+ conversation = [+ {"role": "user", "content": [{"type": "input_video", "input_video": {"data": clip}}]}+ ]+ kept = inference_route._openai_llama_admission_media_tokens(+ payload, message_video_clips = inference_route._conversation_video_clips(conversation)+ )+ assert kept == len(clip) // 4+++def test_the_legacy_field_is_not_charged_twice_during_recosting():+ """_inject_video_part splices the legacy clip into the conversation as input_video before+ the loop starts, so during a recost the conversation clips already include it. Charging the+ field as well priced it at exactly 2x and clamped the lease toward the whole KV budget."""+ from models.inference import ChatCompletionRequest++ clip = "A" * 40_000+ payload = ChatCompletionRequest.model_validate(+ _field_body("data:video/mp4;base64," + clip, text = "hi")+ )+ conversation = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]+ inference_route._inject_video_part(conversation, clip)++ opening = inference_route._openai_llama_admission_media_tokens(payload)+ recost = inference_route._openai_llama_admission_media_tokens(+ payload,+ message_video_clips = inference_route._conversation_video_clips(conversation),+ )+ assert recost == pytest.approx(opening, rel = 0.01)+++def test_audio_is_still_charged_from_the_field_during_recosting():+ """Only video is spliced into the conversation; dropping audio with it would undercharge."""+ from models.inference import ChatCompletionRequest++ body = _field_body("data:video/mp4;base64," + "A" * 400)+ body["audio_base64"] = "B" * 4000+ payload = ChatCompletionRequest.model_validate(body)+ assert (+ inference_route._openai_llama_admission_media_tokens(payload, message_video_clips = [])+ == 1000+ )+++def test_many_tiny_clips_are_refused_by_count_not_only_by_bytes():+ """Each clip is its own ffprobe and ffmpeg (30s and 300s ceilings), so thousands of tiny+ clips stay under the byte cap while holding the request and its lease for hours."""+ from models.inference import ChatCompletionRequest++ tiny = "data:video/mp4;base64,QUJD"+ many = ChatCompletionRequest.model_validate(+ _part_body(*[tiny] * (inference_route._MAX_VIDEO_CLIPS_PER_REQUEST + 1))+ )+ rejection = inference_route._request_video_rejection(many)+ assert rejection is not None and rejection[0] == 400+ assert "Too many videos" in rejection[1]+++def test_a_request_at_the_clip_limit_is_still_served():+ """The count cap must bound abuse, not ordinary multi-clip use."""+ from models.inference import ChatCompletionRequest++ tiny = "data:video/mp4;base64,QUJD"+ ok = ChatCompletionRequest.model_validate(+ _part_body(*[tiny] * inference_route._MAX_VIDEO_CLIPS_PER_REQUEST)+ )+ assert inference_route._request_video_rejection(ok) is None+++def test_the_clip_count_includes_the_legacy_field():+ """Otherwise the field plus the maximum parts exceeds the bound together."""+ from models.inference import ChatCompletionRequest++ tiny = "data:video/mp4;base64,QUJD"+ body = _part_body(*[tiny] * inference_route._MAX_VIDEO_CLIPS_PER_REQUEST)+ body["video_base64"] = tiny+ rejection = inference_route._request_video_rejection(ChatCompletionRequest.model_validate(body))+ assert rejection is not None and rejection[0] == 400diff --git a/studio/backend/tests/test_video_pr9057_simulation.py b/studio/backend/tests/test_video_pr9057_simulation.pyindex 8cb88eebe07..80996739343 100644--- a/studio/backend/tests/test_video_pr9057_simulation.py+++ b/studio/backend/tests/test_video_pr9057_simulation.py@@ -431,7 +431,7 @@ def test_every_path_that_cannot_serve_a_clip_refuses_out_loud(needle):def test_the_size_check_is_paid_before_the_model_switch_not_after():src = _routes_source()handler = src.index("_needs_image = bool(_pre_parsed[2])")- assert src.index("_video_b64_rejection(payload.video_base64)", handler) < src.index(+ assert src.index("_request_video_rejection(payload)", handler) < src.index("await _maybe_auto_switch_model(", handler)@@ -440,4 +440,4 @@ def test_the_external_provider_refusal_precedes_the_proxy_call():src = _routes_source()start = src.index("if payload.provider_id or payload.provider_type:")branch = src[start : src.index("_proxy_to_external_provider(payload", start)]- assert "payload.video_base64" in branch+ assert "_request_has_video(payload)" in branch