Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: accept OpenAI video_url content parts on /v1/chat/completions (#10439)

Long Yixingf7ab2098fa27
diff --git a/studio/backend/core/inference/context_window.py b/studio/backend/core/inference/context_window.py
index 840a02a6dc8..de15047f953 100644
--- a/studio/backend/core/inference/context_window.py
+++ b/studio/backend/core/inference/context_window.py
@@ -13,7 +13,7 @@ from typing import Any, Optional
_OMITTED_TOOL_EXCHANGE = "[Earlier tool exchange omitted from the rolling context window.]"
_UNPRICED_MEDIA_TYPES = frozenset(
- ("image_url", "input_audio", "audio", "input_image", "input_video")
+ ("image_url", "input_audio", "audio", "input_image", "input_video", "video_url")
)
# Trim BELOW the budget: trimming to exactly it puts the next turn over again, so the boundary creeps every turn and
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 92fa69e607a..c610a07f1ef 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1708,6 +1708,19 @@ class ImageContentPart(BaseModel):
image_url: ImageUrl
+class VideoUrl(BaseModel):
+ """Video URL object: an inline data URI. Remote URLs are not fetched."""
+
+ url: str = Field(..., description = "data:video/mp4;base64,... (inline only)")
+
+
+class VideoContentPart(BaseModel):
+ """Video content part; served only through llama-server's ``input_video``."""
+
+ type: Literal["video_url"]
+ video_url: VideoUrl
+
+
class InputDocumentContentPart(BaseModel):
"""Document (PDF / file) content part in a multimodal message.
@@ -1804,6 +1817,7 @@ _KNOWN_CONTENT_PART_TAGS = frozenset(
{
"text",
"image_url",
+ "video_url",
"input_audio",
"input_document",
"reasoning",
@@ -1825,6 +1839,7 @@ ContentPart = Annotated[
Union[
Annotated[TextContentPart, Tag("text")],
Annotated[ImageContentPart, Tag("image_url")],
+ Annotated[VideoContentPart, Tag("video_url")],
Annotated[InputAudioContentPart, Tag("input_audio")],
Annotated[InputDocumentContentPart, Tag("input_document")],
Annotated[OpenAIReasoningContentPart, Tag("reasoning")],
@@ -1892,6 +1907,14 @@ class ChatMessage(BaseModel):
raise ValueError('"tool_calls" is only valid on role="assistant" messages.')
if self.tool_call_id is not None and self.role != "tool":
raise ValueError('"tool_call_id" is only valid on role="tool" messages.')
+ # llama-server renders the marker into whatever turn carried it, so off a user turn the
+ # result is template-dependent.
+ if (
+ self.role != "user"
+ and isinstance(self.content, list)
+ and any(isinstance(part, VideoContentPart) for part in self.content)
+ ):
+ raise ValueError(f'"video_url" parts are not valid on role="{self.role}" messages.')
if self.role == "tool":
# tool_call_id resolution happens at ChatCompletionRequest scope. OpenAI accepts empty tool
diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.py
index 9b80743b0dc..8a2bfc87ba0 100644
--- a/studio/backend/routes/chat_generation_runs.py
+++ b/studio/backend/routes/chat_generation_runs.py
@@ -153,7 +153,11 @@ def _sanitize_request(payload: CreateChatGenerationRun) -> dict[str, Any]:
detail = safe_validation_errors(exc.errors()),
) from exc
# Without this an unservable part is queued at 202 and fails where the caller cannot see it.
- from routes.inference import _messages_have_input_audio, _reject_unsupported_content_parts
+ from routes.inference import (
+ _messages_have_input_audio,
+ _reject_unsupported_content_parts,
+ _request_has_video,
+ )
_reject_unsupported_content_parts(request)
@@ -181,9 +185,13 @@ def _sanitize_request(payload: CreateChatGenerationRun) -> dict[str, Any]:
detail = "Durable chat runs are available only for local inference",
)
# A media turn has no replayable transcript and its payload persists verbatim, so a base64 blob would live in
- if any(
- raw.get(field) not in (None, "") for field in _MEDIA_FIELDS
- ) or _messages_have_input_audio(request.messages):
+ # request_json for the life of the thread. _MEDIA_FIELDS is field-shaped, so a video_url part
+ # needs _request_has_video.
+ if (
+ any(raw.get(field) not in (None, "") for field in _MEDIA_FIELDS)
+ or _messages_have_input_audio(request.messages)
+ or _request_has_video(request)
+ ):
raise HTTPException(
status_code = 400,
detail = "Media chat runs use the legacy streaming path",
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index b5858f01c29..d9fac49a316 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1979,6 +1979,10 @@ def _openai_llama_admission_image_tokens(llama_backend) -> int:
_ADMISSION_IMAGE_PART_TYPES = ("image_url", "image")
+# Both spellings: the server-side tool loop recosts the conversation AFTER
+# _translate_video_parts has renamed the part, so matching video_url alone priced the
+# whole base64 payload as prompt text and inflated the lease toward the KV budget.
+_ADMISSION_VIDEO_PART_TYPES = ("video_url", "input_video")
def _openai_llama_admission_compact_image_part(part: dict) -> dict:
@@ -2044,6 +2048,11 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
estimate_content.append(part)
continue
+ # Priced by its own byte allowance, not as an image and not as prompt text.
+ if part_type in _ADMISSION_VIDEO_PART_TYPES:
+ estimate_content.append({"type": part_type, part_type: {"url": "[video]"}})
+ continue
+
if part_type not in _ADMISSION_IMAGE_PART_TYPES:
estimate_content.append(part)
continue
@@ -2055,11 +2064,35 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
return estimate_messages, image_parts
+def _conversation_video_clips(messages) -> list[str]:
+ """Clips still present in a tool-loop conversation, in either spelling.
+
+ The loop recosts AFTER ``_translate_video_parts`` has renamed the part, so reading
+ ``video_url`` alone would find nothing and charge no video at all.
+ """
+ clips: list[str] = []
+ for msg in messages or []:
+ content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if not isinstance(part, dict):
+ continue
+ media = part.get(part.get("type") or "")
+ if part.get("type") not in _ADMISSION_VIDEO_PART_TYPES or not isinstance(media, dict):
+ continue
+ value = media.get("data") or media.get("url") or ""
+ if isinstance(value, str) and value:
+ clips.append(value)
+ return clips
+
+
def _openai_llama_admission_media_tokens(
payload,
*,
message_image_parts: int = 0,
image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,
+ message_video_clips: Optional[list[str]] = None,
) -> int:
"""Estimate media KV usage without charging transport bytes as prompt tokens.
@@ -2076,10 +2109,26 @@ def _openai_llama_admission_media_tokens(
extra += max(0, message_image_parts) * image_tokens
if _legacy_image_is_distinct(payload):
extra += image_tokens
- for attribute in ("audio_base64", "video_base64"):
+ # _inject_video_part splices the legacy clip into the conversation as input_video before the
+ # loop starts, so during a recost the clips below already include it and charging the field
+ # too priced it exactly twice.
+ fields = (
+ ("audio_base64",) if message_video_clips is not None else ("audio_base64", "video_base64")
+ )
+ for attribute in fields:
value = getattr(payload, attribute, None)
if isinstance(value, str) and value:
extra += max(1, len(value) // 4)
+ # Recosting passes the CURRENT conversation: a clip on a turn that truncate_oldest has since
+ # evicted is no longer sent, and charging it from the opening payload kept every later round
+ # reserved at the full budget. Images already come from the conversation via message_image_parts.
+ clips = (
+ message_video_clips
+ if message_video_clips is not None
+ else _message_video_urls(getattr(payload, "messages", None))
+ )
+ for clip in clips:
+ extra += max(1, len(clip) // 4)
return extra
@@ -2294,6 +2343,7 @@ def _openai_llama_admission_recost(
payload,
message_image_parts = message_image_parts,
image_tokens = _openai_llama_admission_image_tokens(llama_backend),
+ message_video_clips = _conversation_video_clips(conversation),
)
# Reading "Max" literally here would put the run back on the whole cache at its
# first round boundary.
@@ -3206,6 +3256,7 @@ from models.inference import (
InputAudioContentPart,
UnknownContentPart,
ImageUrl,
+ VideoContentPart,
ResponsesRequest,
ResponsesInputTextPart,
ResponsesInputImagePart,
@@ -18571,6 +18622,13 @@ async def generate_audio(
"messages",
"Audio input is not supported here; this route speaks the message text.",
)
+ # A known tag now, so the guard above no longer covers it: the clip would be dropped and
+ # the text spoken alone.
+ if _request_has_video(payload):
+ _raise_unsupported_openai_parameter(
+ "messages",
+ "Video input is not supported here; this route speaks the message text.",
+ )
# Extract text from the last user message
_, chat_messages, _ = _extract_content_parts(payload.messages)
@@ -19923,6 +19981,20 @@ _MAX_AUDIO_B64_CHARS = STT_AUDIO_B64_MAX_CHARS
# Flooring instead refused a file of exactly the size the composer allows.
_MAX_VIDEO_BYTES = 64 * 1024 * 1024
_MAX_VIDEO_B64_CHARS = 4 * math.ceil(_MAX_VIDEO_BYTES / 3)
+# Longest scheme named back to the caller; bounds the scan over a megabytes-long clip.
+_MAX_VIDEO_SCHEME_CHARS = 16
+# llama-server, not Studio, would fetch a remote clip, and its downloader sets
+# set_follow_location(true) (common/http.h), so a public URL redirecting to a private address
+# defeats any host check made here. Refuse the whole shape; #11010 tracks a pinned fetch that
+# would let images and video both accept one.
+# Each clip is a separate ffprobe + ffmpeg pair, so this bounds process launches per request.
+# Four rather than one: GGUF genuinely serves several clips, and the non-GGUF path already
+# refuses more than one on its own.
+_MAX_VIDEO_CLIPS_PER_REQUEST = 4
+_REMOTE_VIDEO_REFUSAL = (
+ 400,
+ "Remote video URLs are not supported. Send the clip as a data URI instead.",
+)
_MAX_AUDIO_SECONDS = 30 * 60
# The duration cap alone is rate-relative, so a high-rate container retains far
# more memory for the same 30 minutes: at 48 kHz that is 86M float32 samples,
@@ -20584,15 +20656,66 @@ _VIDEO_INPUT_REFUSAL = (
def _local_video_clip(payload, model_info) -> str:
- """The clip, as bare base64, that a non-GGUF backend is handed, else a refusal by name."""
+ """The clip, as bare base64, that a non-GGUF backend is handed, else a refusal by name.
+
+ Reads both spellings, so MLX serves a ``video_url`` part exactly as it serves the field.
+ """
if not model_info.get("has_video_input"):
raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)
- video_b64, rejection = _video_b64_rejection(payload.video_base64)
+ clips = _request_video_clips(payload)
+ if not clips:
+ raise HTTPException(status_code = 400, detail = "Could not read the provided video file.")
+ # Generation takes one video kwarg, so a second clip would be dropped in silence and the
+ # same request would mean different things on this backend and on GGUF, which forwards all.
+ if len(clips) > 1:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Only one video is supported per request on this model.",
+ )
+ # Generation is handed one clip and no turn, so MLX would move it onto the newest turn and
+ # answer as though it had just been attached.
+ if _video_is_on_an_older_turn(payload):
+ raise HTTPException(
+ status_code = 400,
+ detail = "A video must be attached to the latest message on this model.",
+ )
+ # Refused everywhere, so the message must not send the caller to GGUF for it.
+ if _is_remote_video(clips[0]):
+ raise HTTPException(status_code = _REMOTE_VIDEO_REFUSAL[0], detail = _REMOTE_VIDEO_REFUSAL[1])
+ # _request_video_rejection runs only when a pre-switch validation happens, so an unsupported
+ # scheme reached here and was decoded as base64 instead of earning the 400 GGUF returns.
+ scheme_rejection = _video_scheme_rejection(clips[0])
+ if scheme_rejection is not None:
+ raise HTTPException(status_code = scheme_rejection[0], detail = scheme_rejection[1])
+ video_b64, rejection = _video_b64_rejection(clips[0])
if rejection is not None:
raise HTTPException(status_code = rejection[0], detail = rejection[1])
return video_b64
+def _video_payload_chars(clip: str) -> int:
+ """Base64 length without the data URI header, measured rather than sliced."""
+ if clip[:5].lower() != "data:":
+ return len(clip)
+ comma = clip.find(",")
+ return len(clip) - comma - 1 if comma != -1 else 0
+
+
+def _video_size_rejection(clip: str) -> Optional[tuple[int, str]]:
+ """``_video_b64_rejection``'s verdict, measured rather than sliced.
+
+ Validation runs twice per request, before the switch and again after the load, and only
+ wants the verdict. Slicing the header off to get it copied the whole payload each time:
+ 89 MB and 47 ms for a clip at the 64 MB limit, against 0 MB here.
+ """
+ payload_len = _video_payload_chars(clip)
+ if not payload_len:
+ return (400, "Could not read the provided video file.")
+ if payload_len > _MAX_VIDEO_B64_CHARS:
+ return (413, "Video file is too large (max 64 MB).")
+ return None
+
+
def _video_b64_rejection(video_b64: str) -> tuple[str, Optional[tuple[int, str]]]:
"""The clip's base64 without its data URI header, plus why it is refused.
@@ -20600,7 +20723,7 @@ def _video_b64_rejection(video_b64: str) -> tuple[str, Optional[tuple[int, str]]
size the composer allows. Returned rather than raised so the pre-switch and
post-load checks share one rule while raising their own way.
"""
- if video_b64.startswith("data:"):
+ if video_b64[:5].lower() == "data:":
video_b64 = video_b64.split(",", 1)[1] if "," in video_b64 else ""
if not video_b64:
return "", (400, "Could not read the provided video file.")
@@ -20725,6 +20848,172 @@ def _normalise_chat_content_parts(payload) -> None:
payload.audio_base64 = lifted
+def _message_video_urls(messages) -> list[str]:
+ urls: list[str] = []
+ for msg in messages or []:
+ content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if isinstance(part, VideoContentPart):
+ urls.append(part.video_url.url)
+ elif isinstance(part, dict) and part.get("type") == "video_url":
+ video_url = part.get("video_url")
+ url = video_url.get("url") if isinstance(video_url, dict) else video_url
+ urls.append(url if isinstance(url, str) else "")
+ return urls
+
+
+def _video_is_on_an_older_turn(payload) -> bool:
+ """True when a ``video_url`` part sits anywhere before the newest user turn.
+
+ GGUF keeps a part on the turn that carried it. The non-GGUF path flattens the parts away and
+ hands generation one video kwarg, which MLX attaches to the newest user turn, so the same
+ request would put the clip on a different turn per backend.
+ """
+ messages = getattr(payload, "messages", None) or []
+ roles = [m.get("role") if isinstance(m, dict) else getattr(m, "role", None) for m in messages]
+ if "user" not in roles:
+ return False
+ return bool(_message_video_urls(messages[: len(roles) - 1 - roles[::-1].index("user")]))
+
+
+def _request_video_clips(payload) -> list[str]:
+ legacy = getattr(payload, "video_base64", None)
+ return ([legacy] if legacy else []) + _message_video_urls(getattr(payload, "messages", None))
+
+
+def _request_has_video(payload) -> bool:
+ return bool(_request_video_clips(payload))
+
+
+def _is_remote_video(url: str) -> bool:
+ # Schemes are case-insensitive and a clip is megabytes: read the prefix, not a copy.
+ return url[:8].lower().startswith(("http://", "https://"))
+
+
+def _video_scheme_rejection(clip: str) -> Optional[tuple[int, str]]:
+ """Refuse a clip whose URL names a scheme neither we nor llama-server should honour.
+
+ ``handle_media`` reads ``input_video`` as ``data`` else ``url`` and treats the one string it
+ finds the same way, honouring ``file://`` under ``--media-path``. Forwarding such a clip as
+ opaque payload is therefore still a local file read, so refuse it by name instead.
+ """
+ if not clip or clip[:5].lower() == "data:" or _is_remote_video(clip):
+ return None
+ # ':' is not in the base64 alphabet, so an early colon marks a URL, not payload.
+ head = clip[: _MAX_VIDEO_SCHEME_CHARS + 1]
+ if ":" not in head:
+ return None
+ return (
+ 400,
+ f"Unsupported video URL scheme ('{head.split(':', 1)[0]}:'). "
+ "Send the clip as a data URI.",
+ )
+
+
+def _request_video_rejection(payload) -> Optional[tuple[int, str]]:
+ clips = _request_video_clips(payload)
+ # Bytes alone do not bound the work: each clip costs its own ffprobe and ffmpeg (30s and
+ # 300s ceilings), so thousands of tiny clips stay under the byte cap while holding the
+ # request, and its admission lease, for hours. Count is the bound on process launches.
+ if len(clips) > _MAX_VIDEO_CLIPS_PER_REQUEST:
+ return (
+ 400,
+ f"Too many videos in one request (max {_MAX_VIDEO_CLIPS_PER_REQUEST}).",
+ )
+ total = 0
+ for clip in clips:
+ if _is_remote_video(clip):
+ return _REMOTE_VIDEO_REFUSAL
+ rejection = _video_scheme_rejection(clip)
+ if rejection is not None:
+ return rejection
+ rejection = _video_size_rejection(clip)
+ if rejection is not None:
+ return rejection
+ total += _video_payload_chars(clip)
+ # And bytes bound the per-clip transcode cost the count cannot see.
+ if total > _MAX_VIDEO_B64_CHARS:
+ return (413, "Videos are too large (max 64 MB per request).")
+ return None
+
+
+async def _shrink_clip_for_llama(video_b64: str, llama_backend) -> str:
+ """Transcode one clip down to the cap, as the legacy field's path does.
+
+ The transcode runs BEFORE llama-server samples and the server can only duplicate what was
+ dropped, so an explicit rate must reach it.
+ """
+ try:
+ return await asyncio.to_thread(
+ shrink_video_for_llama,
+ video_b64,
+ _MAX_VIDEO_BYTES,
+ sampled_fps = requested_video_fps(getattr(llama_backend, "extra_args", None)),
+ )
+ except Exception as e:
+ # The helper absorbs its own ffmpeg failures, so reaching here means something it does
+ # not model went wrong. Reject like the audio path, rather than letting a raw 500 leak
+ # the monitor entry with it.
+ logger.warning("Video preprocessing failed: %s", e, exc_info = True)
+ raise HTTPException(
+ status_code = 400, detail = "Could not process the provided video file."
+ ) from None
+
+
+async def _shrink_video_parts(messages: list[dict], llama_backend) -> None:
+ """Transcode every ``video_url`` part in place, before translation renames them.
+
+ The legacy field is shrunk on its own path. Doing it here as well is what keeps the two
+ spellings identical: a part-carried clip would otherwise reach llama-server untranscoded and
+ at a different resolution than the same clip sent as the field.
+ """
+ for msg in messages:
+ content = msg.get("content")
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if not isinstance(part, dict) or part.get("type") != "video_url":
+ continue
+ video_url = part.get("video_url")
+ url = video_url.get("url") if isinstance(video_url, dict) else video_url
+ url = url if isinstance(url, str) else ""
+ if _is_remote_video(url) or _video_scheme_rejection(url) is not None:
+ continue # _translate_video_parts refuses these by name a moment later.
+ clip, rejection = _video_b64_rejection(url)
+ if rejection is not None:
+ continue
+ part["video_url"] = {"url": await _shrink_clip_for_llama(clip, llama_backend)}
+
+
+def _translate_video_parts(messages: list[dict]) -> None:
+ """Rewrite ``video_url`` parts in place as llama-server's ``input_video``, always as bytes."""
+ for msg in messages:
+ content = msg.get("content")
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if not isinstance(part, dict) or part.get("type") != "video_url":
+ continue
+ video_url = part.get("video_url")
+ url = video_url.get("url") if isinstance(video_url, dict) else video_url
+ url = url if isinstance(url, str) else ""
+ # Restated: _request_video_rejection only runs when a pre-switch validation happens,
+ # so no caller reaching here by another route may hand llama-server a URL to dial.
+ if _is_remote_video(url):
+ raise HTTPException(
+ status_code = _REMOTE_VIDEO_REFUSAL[0], detail = _REMOTE_VIDEO_REFUSAL[1]
+ )
+ scheme_rejection = _video_scheme_rejection(url)
+ if scheme_rejection is not None:
+ raise HTTPException(status_code = scheme_rejection[0], detail = scheme_rejection[1])
+ part.clear()
+ part.update(
+ {"type": "input_video", "input_video": {"data": _video_b64_rejection(url)[0]}}
+ )
+
+
def _inject_audio_part(messages: list[dict], audio_b64: str, audio_format: str) -> None:
"""Append an input_audio part to the last user message, in place.
@@ -22698,8 +22987,9 @@ async def produce_openai_chat_completions(
untrack_current_request(request.scope)
if _wants_multiple_choices(payload):
_raise_unsupported_n("external provider chat completions")
- # The proxy has nowhere to put the clip; say so rather than answering without it.
- if payload.video_base64:
+ # The proxy has nowhere to put the clip; say so rather than answering without it. Both
+ # spellings, so a video_url part is refused here the way the legacy field is.
+ if _request_has_video(payload):
raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)
# _build_external_messages carries no input_audio case, so the recording would be
# stripped and the provider would answer the text alone -- a plausible reply to a
@@ -22867,7 +23157,7 @@ async def produce_openai_chat_completions(
# Video rides that projector too. Its own /props gate can only run after
# the load, so this at least keeps a text-only target from evicting a
# working model to serve a clip it could never take.
- _needs_video = bool(payload.video_base64)
+ _needs_video = _request_has_video(payload)
_needs_vision = _needs_image or bool(payload.audio_base64) or _needs_video
# Name what is actually attached, so the refusal does not report a
# modality the request never carried.
@@ -22877,7 +23167,7 @@ async def produce_openai_chat_completions(
for name, present in (
("image", _needs_image),
("audio", bool(payload.audio_base64)),
- ("video", bool(payload.video_base64)),
+ ("video", _needs_video),
)
if present
)
@@ -22885,10 +23175,9 @@ async def produce_openai_chat_completions(
)
# Size is knowable now and the switch is not cheap: refuse an oversized
# clip before it costs a model load.
- if payload.video_base64:
- _, _video_rejection = _video_b64_rejection(payload.video_base64)
- if _video_rejection is not None:
- raise HTTPException(status_code = _video_rejection[0], detail = _video_rejection[1])
+ _video_rejection = _request_video_rejection(payload)
+ if _video_rejection is not None:
+ raise HTTPException(status_code = _video_rejection[0], detail = _video_rejection[1])
_needs_audio_input = bool(payload.audio_base64)
# the rest of the audio checks depend on the target's format, so the switch runs them.
@@ -23046,7 +23335,7 @@ async def produce_openai_chat_completions(
model_info = backend.models.get(backend.active_model_name, {})
# Before the speech and audio-input dispatches, which return before the clip is attached.
_video_clip = None
- if payload.video_base64:
+ if _request_has_video(payload):
_video_clip = _local_video_clip(payload, model_info)
# Settled here: a model without audio input never enters the audio-input path.
if payload.audio_base64:
@@ -23507,7 +23796,7 @@ async def produce_openai_chat_completions(
400,
"Audio input is not supported together with guided decoding or client-supplied tools yet.",
)
- if payload.video_base64:
+ if _request_has_video(payload):
# Same shape: _build_openai_passthrough_body forwards an explicit
# field list, so the clip would be dropped and the model would
# answer without it.
@@ -23609,31 +23898,19 @@ async def produce_openai_chat_completions(
# llama-server samples frames but encodes each at the clip's resolution.
video_b64 = None
- if payload.video_base64:
+ if _request_has_video(payload):
if not getattr(llama_backend, "_has_video_input", False):
raise _reject(
400,
"Video provided but the current GGUF model cannot take video input. "
"It needs an mmproj with video support, and ffmpeg/ffprobe installed.",
)
- video_b64, video_rejection = _video_b64_rejection(payload.video_base64)
+ video_rejection = _request_video_rejection(payload)
if video_rejection is not None:
raise _reject(*video_rejection)
- try:
- # The transcode runs BEFORE llama-server samples and the server can
- # only duplicate what was dropped, so an explicit rate must reach it.
- video_b64 = await asyncio.to_thread(
- shrink_video_for_llama,
- video_b64,
- _MAX_VIDEO_BYTES,
- sampled_fps = requested_video_fps(getattr(llama_backend, "extra_args", None)),
- )
- except Exception as e:
- # The helper absorbs its own ffmpeg failures, so reaching here means
- # something it does not model went wrong. Reject like the audio path
- # above, rather than letting a raw 500 leak the monitor entry with it.
- logger.warning("Video preprocessing failed: %s", e, exc_info = True)
- raise _reject(400, "Could not process the provided video file.")
+ if payload.video_base64:
+ video_b64, _ = _video_b64_rejection(payload.video_base64)
+ video_b64 = await _shrink_clip_for_llama(video_b64, llama_backend)
gguf_messages, _ = await _openai_messages_for_gguf_chat_async(
payload,
@@ -23645,6 +23922,8 @@ async def produce_openai_chat_completions(
_inject_audio_part(gguf_messages, audio_b64, audio_format)
if video_b64:
_inject_video_part(gguf_messages, video_b64)
+ await _shrink_video_parts(gguf_messages, llama_backend)
+ _translate_video_parts(gguf_messages)
cancel_event = _chat_cancel_event(request)
@@ -31754,7 +32033,7 @@ async def chat_count_tokens(
detail = "Cannot count tokens for messages containing audio.",
)
# And video, whose frames llama-server samples at completion time.
- if getattr(payload, "video_base64", None):
+ if _request_has_video(payload):
raise HTTPException(
status_code = 503,
detail = "Cannot count tokens for messages containing video.",
diff --git a/studio/backend/routes/preview.py b/studio/backend/routes/preview.py
index 018fadfd895..719756e696e 100644
--- a/studio/backend/routes/preview.py
+++ b/studio/backend/routes/preview.py
@@ -19,6 +19,7 @@ from auth.storage import DEFAULT_ADMIN_USERNAME
from models.inference import ChatCompletionRequest, LoadRequest
from routes.inference import (
_reject_unsupported_content_parts,
+ _request_has_video,
disable_openai_auto_switch_for_request,
load_model_for_preview,
openai_chat_completions,
@@ -159,6 +160,12 @@ async def _serve_chat(
):
path = _resolve_or_4xx(run, checkpoint)
_reject_unsupported_content_parts(payload)
+ # A checkpoint cannot read a clip: refuse before the load the refusal would otherwise follow.
+ if _request_has_video(payload):
+ raise HTTPException(
+ status_code = 400,
+ detail = "Video input is only supported on a local GGUF model with video support.",
+ )
is_lora = (path / "adapter_config.json").exists()
payload = _sanitize_preview_payload(payload, is_lora)
scope = getattr(request, "scope", None)
diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
index bf4537cc0fe..d409e4000a1 100644
--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
@@ -901,3 +901,100 @@ class TestARoundIsCostedTheSameWayTheReservationWas:
f"round zero shrank the lease from {opened} to {committed}, dropping the "
f"non-message prompt the reservation charged"
)
+
+
+class TestARoundStopsPayingForAnEvictedClip:
+ """truncate_oldest can drop the turn that carried a clip. The re-cost reads the CURRENT
+ conversation for text and images, so reading the opening payload for video kept every later
+ round reserved at the full budget for media llama-server is no longer sent.
+ """
+
+ class _Backend:
+ base_url = "http://llama"
+ effective_parallel_slots = 4
+ _kv_cache_context_total = 4096
+ context_length = 4096
+ _mmproj_projector_type = None
+ _extra_args = None
+
+ class _Reservation:
+ def __init__(self, lease):
+ self._lease = lease
+
+ def lease_nowait(self):
+ return self._lease
+
+ def _recost(self, payload, conversation):
+ from routes.inference import (
+ _openai_llama_admission_recost,
+ _openai_llama_admission_tokens,
+ )
+ async def _run():
+ queue = LlamaAdmissionQueue("test")
+ opened = _openai_llama_admission_tokens(
+ payload, budget = 4096, capacity = 4, tool_loop = True
+ )
+ reservation = queue.reserve(
+ capacity = 4,
+ config = LlamaAdmissionConfig(),
+ tokens = opened,
+ budget = 4096,
+ )
+ lease = reservation.lease_nowait()
+ assert lease is not None
+ _openai_llama_admission_recost(
+ self._Reservation(lease),
+ conversation,
+ request = None,
+ llama_backend = self._Backend(),
+ payload = payload,
+ output_tokens = 64,
+ )
+ return queue.snapshot().committed
+
+ return asyncio.run(_run())
+
+ def _payload(self, clip_b64):
+ return _Payload(
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "video_url",
+ "video_url": {"url": "data:video/mp4;base64," + clip_b64},
+ }
+ ],
+ }
+ ],
+ video_base64 = None,
+ audio_base64 = None,
+ image_base64 = None,
+ )
+
+ def test_the_clip_is_charged_while_the_conversation_still_carries_it(self):
+ clip = "A" * 40_000
+ kept = [
+ {
+ "role": "user",
+ "content": [{"type": "input_video", "input_video": {"data": clip}}],
+ }
+ ]
+ assert self._recost(self._payload(clip), kept) > self._recost(
+ self._payload(clip), [{"role": "user", "content": "text only"}]
+ )
+
+ def test_an_evicted_clip_stops_being_charged(self):
+ """The whole point: once the turn is gone the round must not still reserve for it."""
+ clip = "A" * 40_000
+ evicted = self._recost(self._payload(clip), [{"role": "user", "content": "text only"}])
+ no_video_at_all = self._recost(
+ _Payload(
+ messages = [{"role": "user", "content": "text only"}],
+ video_base64 = None,
+ audio_base64 = None,
+ image_base64 = None,
+ ),
+ [{"role": "user", "content": "text only"}],
+ )
+ assert evicted == no_video_at_all
diff --git a/studio/backend/tests/test_llama_video_frame_budget.py b/studio/backend/tests/test_llama_video_frame_budget.py
index f8929a767ae..89ef7750b44 100644
--- a/studio/backend/tests/test_llama_video_frame_budget.py
+++ b/studio/backend/tests/test_llama_video_frame_budget.py
@@ -412,9 +412,10 @@ def test_the_gguf_route_shrinks_the_clip_after_the_size_check_and_before_injecti
)
start = source.index('"Video provided but the current GGUF model cannot take video input. "')
check = source.index("_video_b64_rejection(payload.video_base64)", start)
- # Bare name: the call has been reflowed twice, and each time an anchor on the
- # full call raised ValueError instead of reporting an ordering problem.
- shrink = source.index("shrink_video_for_llama,", start)
+ # The transcode moved into _shrink_clip_for_llama so the video_url parts can reuse it;
+ # anchor on the route's call to that helper, not on shrink_video_for_llama, which now
+ # only appears inside it (and earlier in the file).
+ shrink = source.index("_shrink_clip_for_llama(video_b64, llama_backend)", start)
inject = source.index("_inject_video_part(gguf_messages, video_b64)", start)
assert check < shrink < inject
@@ -439,6 +440,9 @@ def test_the_conversions_result_is_what_gets_injected():
call = node.value if isinstance(node, ast.Await) else node
if not isinstance(call, ast.Call):
return False
+ # Either spelling: the route calls the helper, and the helper calls the real thing.
+ if isinstance(call.func, ast.Name) and call.func.id == "_shrink_clip_for_llama":
+ return True
return any(isinstance(a, ast.Name) and a.id == "shrink_video_for_llama" for a in call.args)
bound_to = {
diff --git a/studio/backend/tests/test_preview_routes.py b/studio/backend/tests/test_preview_routes.py
index ad0643ac531..77774ab91a0 100644
--- a/studio/backend/tests/test_preview_routes.py
+++ b/studio/backend/tests/test_preview_routes.py
@@ -37,9 +37,13 @@ if _BACKEND_DIR not in sys.path:
# Mirror test_preview.py: the real `loggers` package pulls in heavy handlers.
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
+# __path__ so `loggers.media_progress` still resolves to the real submodule. Without it the stub
+# shadows the package rather than its __init__, and any module importing a submodule dies with
+# "'loggers' is not a package" as soon as this file shares a pytest process with it (#10995).
+_loggers_stub.__path__ = [str(Path(_BACKEND_DIR) / "loggers")]
sys.modules.setdefault("loggers", _loggers_stub)
-from fastapi import FastAPI
+from fastapi import FastAPI, HTTPException
from fastapi.responses import StreamingResponse
from fastapi.testclient import TestClient
@@ -159,6 +163,11 @@ def test_page_recovers_from_empty_reply(client):
assert "Reply cut off before the model returned an answer." in text
+ sys.platform == "win32",
+ reason = "'<' is not a legal Windows filename character, so a run dir named a<b cannot "
+ "exist there and the escaping this covers is unreachable",
+)
def test_page_escapes_title(tmp_path, monkeypatch, captured):
outputs = tmp_path / "outputs"
# Run dir name carries an HTML-special char; the page must escape it.
@@ -392,6 +401,45 @@ def test_streaming_holds_lock_until_drained(tmp_path, monkeypatch, captured):
assert not preview._preview_lock.locked()
+def test_a_video_clip_is_refused_before_the_checkpoint_loads(tmp_path, monkeypatch, captured):
+ """A preview target cannot read a clip, and the load it would precede evicts the resident model."""
+ outputs = tmp_path / "outputs"
+ _make_run(outputs)
+ from utils.paths import storage_roots as _sr
+
+ monkeypatch.setattr(_sr, "outputs_root", lambda: outputs)
+
+ loaded = []
+
+ async def _fake_load_model(load_req, request, subject):
+ loaded.append(load_req)
+ return None
+
+ async def _fake_chat(payload, request, subject):
+ return {"ok": True}
+
+ monkeypatch.setattr(preview, "load_model_for_preview", _fake_load_model)
+ monkeypatch.setattr(preview, "openai_chat_completions", _fake_chat)
+
+ payload = ChatCompletionRequest(
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJD"}},
+ {"type": "text", "text": "what happens here?"},
+ ],
+ }
+ ]
+ )
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(preview._serve_chat("demorun", None, payload, request = None))
+ assert excinfo.value.status_code == 400
+ assert "Video input" in excinfo.value.detail
+ assert loaded == []
+ assert not preview._preview_lock.locked()
+
+
# ── Capability gating ────────────────────────────────────────────────────────
@@ -547,8 +595,6 @@ def test_chat_rate_limited_returns_429(client, monkeypatch):
import threading
from types import SimpleNamespace
-from fastapi import HTTPException
-
import routes.inference as inference
from core.inference import llama_keepwarm
from models.inference import LoadRequest
diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.py
index ca677fa4d76..4ec0ecc6ad4 100644
--- a/studio/backend/tests/test_video_attachment_part.py
+++ b/studio/backend/tests/test_video_attachment_part.py
@@ -128,7 +128,7 @@ def test_video_is_refused_on_the_tool_passthrough_path():
start = source.index("if using_gguf and _takes_tool_passthrough(payload, llama_backend):")
branch = source[start : start + 2500]
assert "payload.audio_base64" in branch
- assert "payload.video_base64" in branch
+ assert "_request_has_video(payload)" in branch
assert "Video input is not supported together with guided decoding" in branch
@@ -138,7 +138,7 @@ def test_the_size_check_runs_before_the_automatic_switch():
source = _inference_source()
# Anchor inside the chat-completions handler; other routes switch too.
handler = source.index("_needs_image = bool(_pre_parsed[2])")
- guard = source.index("_video_b64_rejection(payload.video_base64)", handler)
+ guard = source.index("_request_video_rejection(payload)", handler)
switch = source.index("await _maybe_auto_switch_model(", handler)
assert guard < switch
@@ -150,7 +150,7 @@ def test_video_joins_the_projector_requirement_before_switching():
start = source.index("_needs_image = bool(_pre_parsed[2])")
switch = source.index("await _maybe_auto_switch_model(", start)
assert start < source.index("payload.audio_base64", start) < switch
- assert start < source.index("payload.video_base64", start) < switch
+ assert start < source.index("_request_has_video(payload)", start) < switch
def test_an_external_provider_refuses_video_rather_than_ignoring_it():
@@ -159,7 +159,8 @@ def test_an_external_provider_refuses_video_rather_than_ignoring_it():
source = _inference_source()
start = source.index("if payload.provider_id or payload.provider_type:")
branch = source[start : source.index("_proxy_to_external_provider(payload", start)]
- assert "payload.video_base64" in branch
+ # Both spellings through one predicate, so a video_url part cannot reach the proxy either.
+ assert "_request_has_video(payload)" in branch
assert "_VIDEO_INPUT_REFUSAL" in branch
@@ -222,6 +223,8 @@ def test_a_non_gguf_model_takes_the_clip_through_one_gate_and_hands_it_to_genera
conflict = source.index("if payload.audio_base64:", gate)
assert conflict < speech
assert "_AUDIO_VIDEO_INPUT_DETAIL" in source[conflict : conflict + 200]
+ # The gate reads both spellings, so a video_url part reaches MLX exactly as the field does.
+ assert "if _request_has_video(payload):\n _video_clip = _local_video_clip(" in source
def test_token_counting_refuses_video_like_image_and_audio():
@@ -238,4 +241,267 @@ def test_both_video_checks_share_one_rule():
"""Two size checks that drift let the pre-switch one pass what the post-load
one refuses, which is the model load this was meant to avoid."""
source = _inference_source()
- assert source.count("_video_b64_rejection(payload.video_base64)") == 3
+ assert source.count("= _request_video_rejection(payload)") == 2
+
+
+def _video_url_request(
+ *urls,
+ text = "what happens here?",
+ **kw,
+):
+ from models.inference import ChatCompletionRequest
+
+ parts = [{"type": "video_url", "video_url": {"url": url}} for url in urls]
+ parts.append({"type": "text", "text": text})
+ return ChatCompletionRequest(messages = [{"role": "user", "content": parts}], **kw)
+
+
+def test_a_video_url_part_validates_as_the_openai_shape():
+ from models.inference import VideoContentPart
+
+ req = _video_url_request("data:video/mp4;base64,QUJD", "https://example.com/clip.mp4")
+ assert all(isinstance(part, VideoContentPart) for part in req.messages[0].content[:2])
+ assert req.messages[0].content[1].video_url.url == "https://example.com/clip.mp4"
+
+
+def test_a_message_carried_clip_counts_as_video_input():
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _request_has_video
+
+ assert _request_has_video(_video_url_request("data:video/mp4;base64,QUJD")) is True
+ plain = ChatCompletionRequest(messages = [{"role": "user", "content": "hi"}])
+ assert _request_has_video(plain) is False
+
+
+def test_a_data_uri_part_is_translated_in_place_without_its_header():
+ from routes.inference import _openai_messages_for_gguf_chat, _translate_video_parts
+
+ messages, _ = _openai_messages_for_gguf_chat(
+ _video_url_request("data:video/mp4;base64,QUJD"), True
+ )
+ _translate_video_parts(messages)
+ assert messages[0]["content"] == [
+ {"type": "input_video", "input_video": {"data": "QUJD"}},
+ {"type": "text", "text": "what happens here?"},
+ ]
+
+
+def test_a_remote_url_is_refused_rather_than_forwarded():
+ """llama-server would fetch it from this machine, and its downloader follows redirects
+ (set_follow_location, common/http.h), so a public URL redirecting to a private address
+ defeats any host check made here. input_video therefore only ever carries bytes."""
+ import pytest
+ from fastapi import HTTPException
+ from routes.inference import _translate_video_parts
+
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "and this one?"},
+ {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}},
+ ],
+ }
+ ]
+ with pytest.raises(HTTPException) as exc:
+ _translate_video_parts(messages)
+ assert exc.value.status_code == 400
+ assert "Remote video URLs are not supported" in exc.value.detail
+
+
+def test_an_uppercase_data_uri_header_is_still_stripped():
+ """RFC 2397 schemes are case-insensitive; a kept header would be decoded as media."""
+ from routes.inference import _translate_video_parts
+
+ messages = [
+ {
+ "role": "user",
+ "content": [{"type": "video_url", "video_url": {"url": "DATA:video/mp4;base64,QUJD"}}],
+ }
+ ]
+ _translate_video_parts(messages)
+ assert messages[0]["content"][0] == {"type": "input_video", "input_video": {"data": "QUJD"}}
+
+
+def test_an_uppercase_scheme_is_refused_as_a_remote_url_too():
+ """Schemes are case-insensitive, so HTTPS:// must be recognised as remote and refused
+ rather than falling through and being decoded as base64."""
+ import pytest
+ from fastapi import HTTPException
+ from routes.inference import _translate_video_parts
+
+ for url in ("HTTPS://example.com/clip.mp4", "Http://example.com/other.mp4"):
+ messages = [{"role": "user", "content": [{"type": "video_url", "video_url": {"url": url}}]}]
+ with pytest.raises(HTTPException) as exc:
+ _translate_video_parts(messages)
+ assert exc.value.status_code == 400
+
+
+def test_message_parts_are_translated_where_the_legacy_clip_is_injected():
+ """Both spellings reach llama-server from one place, after the capability gate.
+
+ The legacy clip is injected first and the translation is the last thing before dispatch, so
+ one call covers both. Asserted as order rather than adjacency: the destination guard runs
+ between them, and pinning the exact neighbouring line only broke on that.
+ """
+ source = _inference_source()
+ inject = source.index("_inject_video_part(gguf_messages, video_b64)")
+ translate = source.index("_translate_video_parts(gguf_messages)")
+ assert inject < translate
+ assert source.count("_translate_video_parts(gguf_messages)") == 1
+
+
+def test_every_data_uri_part_is_sized_not_only_the_first():
+ from routes.inference import _MAX_VIDEO_B64_CHARS, _request_video_rejection
+
+ fits = _video_url_request("data:video/mp4;base64,QUJD", "data:video/mp4;base64,QUJD")
+ assert _request_video_rejection(fits) is None
+ oversized = "data:video/mp4;base64," + "A" * (_MAX_VIDEO_B64_CHARS + 1)
+ too_big = _video_url_request("data:video/mp4;base64,QUJD", oversized)
+ assert _request_video_rejection(too_big) == (413, "Video file is too large (max 64 MB).")
+ assert _request_video_rejection(_video_url_request("")) == (
+ 400,
+ "Could not read the provided video file.",
+ )
+
+
+def test_the_legacy_field_is_sized_beside_a_message_part():
+ from routes.inference import _MAX_VIDEO_B64_CHARS, _request_video_rejection
+ req = _video_url_request(
+ "https://example.com/clip.mp4", video_base64 = "A" * (_MAX_VIDEO_B64_CHARS + 1)
+ )
+ assert _request_video_rejection(req) == (413, "Video file is too large (max 64 MB).")
+
+
+def test_admission_prices_a_message_clip_as_media_not_prompt_text():
+ from routes.inference import (
+ _openai_llama_admission_media_tokens,
+ _openai_llama_admission_messages_for_estimate,
+ )
+
+ req = _video_url_request("data:video/mp4;base64," + "A" * 4000)
+ estimate, image_parts = _openai_llama_admission_messages_for_estimate(req.messages)
+ assert estimate[0]["content"][0] == {"type": "video_url", "video_url": {"url": "[video]"}}
+ assert image_parts == 0
+ assert _openai_llama_admission_media_tokens(req) >= 1000
+
+
+def test_the_rolling_context_does_not_price_a_video_url_part():
+ from core.inference.context_window import estimate_message_tokens_without_unpriced_media
+ message = {
+ "role": "user",
+ "content": [
+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64," + "A" * 40000}},
+ {"type": "text", "text": "hi"},
+ ],
+ }
+ assert estimate_message_tokens_without_unpriced_media(message) < 100
+
+
+def test_a_message_clip_reaches_the_switch_as_video(monkeypatch):
+ """Through the handler: the pre-switch gate must see a message-carried clip."""
+ import asyncio
+
+ import routes.inference as inference_route
+ from utils import openai_auto_switch_settings as settings
+
+ class _Reached(Exception):
+ pass
+
+ captured = {}
+
+ async def _capture(model, request, subject, **kw):
+ captured.update(kw)
+ raise _Reached()
+
+ monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True)
+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _capture)
+ payload = _video_url_request("data:video/mp4;base64,QUJD", model = "org/B-GGUF")
+ with pytest.raises(_Reached):
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+ assert captured["require_video"] is True
+ assert captured["require_vision"] is True
+ assert captured["modality_label"] == "video"
+
+
+def test_every_clip_in_every_turn_is_translated():
+ from routes.inference import _translate_video_parts
+
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJD"}},
+ {"type": "text", "text": "first"},
+ ],
+ },
+ {"role": "assistant", "content": "ok"},
+ {
+ "role": "user",
+ "content": [
+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJE"}},
+ {"type": "video_url", "video_url": {"url": "data:video/webm;base64,REVG"}},
+ ],
+ },
+ ]
+ _translate_video_parts(messages)
+ assert messages[0]["content"][0] == {"type": "input_video", "input_video": {"data": "QUJD"}}
+ assert messages[2]["content"] == [
+ {"type": "input_video", "input_video": {"data": "QUJE"}},
+ {"type": "input_video", "input_video": {"data": "REVG"}},
+ ]
+
+
+def test_admission_charges_each_clip_once():
+ """Every clip is base64 now, so each is priced by its own length and none is double counted."""
+ from routes.inference import _openai_llama_admission_media_tokens
+
+ first = "data:video/mp4;base64," + "A" * 4000
+ second = "data:video/mp4;base64," + "B" * 8000
+ req = _video_url_request(first, second)
+ assert _openai_llama_admission_media_tokens(req) == len(first) // 4 + len(second) // 4
+
+
+def test_a_clip_on_any_non_user_role_is_refused():
+ """Media outside a user turn renders template-dependently, so it is refused up front."""
+ import pydantic
+
+ from models.inference import ChatCompletionRequest
+
+ parts = [
+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,QUJD"}},
+ {"type": "text", "text": "watch this before answering"},
+ ]
+ for message in (
+ {"role": "system", "content": parts},
+ {"role": "developer", "content": parts},
+ {"role": "assistant", "content": parts},
+ {"role": "tool", "tool_call_id": "call_1", "content": parts},
+ ):
+ refusal = f'not valid on role="{message["role"]}"'
+ with pytest.raises(pydantic.ValidationError, match = refusal):
+ ChatCompletionRequest(messages = [{"role": "user", "content": "go"}, message])
+ ChatCompletionRequest(messages = [{"role": "user", "content": parts}])
+
+
+def test_an_oversized_message_clip_is_refused_before_the_switch(monkeypatch):
+ import asyncio
+
+ from fastapi import HTTPException
+
+ import routes.inference as inference_route
+ from routes.inference import _MAX_VIDEO_B64_CHARS
+ from utils import openai_auto_switch_settings as settings
+
+ async def _switch(*args, **kw):
+ raise AssertionError("the model switch ran before the size check")
+
+ monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True)
+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _switch)
+ payload = _video_url_request(
+ "data:video/mp4;base64," + "A" * (_MAX_VIDEO_B64_CHARS + 1),
+ model = "org/B-GGUF",
+ )
+ with pytest.raises(HTTPException) as info:
+ asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester"))
+ assert info.value.status_code == 413
diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.py
new file mode 100644
index 00000000000..9c1cb31e5ad
--- /dev/null
+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py
@@ -0,0 +1,936 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""What a client gets back when it attaches a clip, by status code and wire shape.
+
+The other video tests assert a line is present in ``routes/inference.py``, which is how the
+non-GGUF regression went unnoticed: the source assertion passed while the request 400'd. These
+take the ``messages`` kwarg of ``generate_chat_completion`` as the JSON llama-server receives.
+
+Invariant, asserted per route: a ``video_url`` part and the legacy ``video_base64`` field behave
+identically, so picking a spelling cannot pick a different set of refusals.
+
+Wire shapes follow llama.cpp ``handle_media``: ``input_video`` is ``data`` else ``url``, remote
+downloads cap at 10 MB.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+pytest.importorskip("torch")
+
+from fastapi import FastAPI # noqa: E402
+from fastapi.testclient import TestClient # noqa: E402
+
+from auth.authentication import get_current_subject # noqa: E402
+import routes.inference as inference_route # noqa: E402
+
+from .llama_backend_double import FakeLlamaCppBackend # noqa: E402
+
+_CLIP_B64 = "AAAAGGZ0eXBtcDQy" # a bare mp4 box header
+_DATA_URI = f"data:video/mp4;base64,{_CLIP_B64}"
+_REMOTE = "https://example.com/clip.mp4"
+
+
+class _VideoGguf(FakeLlamaCppBackend):
+ """A loaded GGUF whose llama-server /props reported ``modalities.video``."""
+
+ is_vision = True
+ _has_video_input = True
+
+ def __init__(self, *, has_video = True):
+ self._has_video_input = has_video
+ self.dispatched: list[dict] = []
+
+ def generate_chat_completion(self, **kwargs):
+ self.dispatched.append(kwargs)
+ yield "ok"
+ yield {"type": "metadata", "usage": {}, "timings": {}}
+
+
+def _no_gguf():
+ """The resident backend is transformers or MLX, so ``using_gguf`` is False."""
+ return SimpleNamespace(
+ is_loaded = False, supports_tools = False, is_vision = False, context_length = None
+ )
+
+
+def _client(
+ monkeypatch,
+ backend = None,
+ *,
+ prefix = "/v1",
+):
+ async def _no_switch(*_a, **_k):
+ return None
+
+ monkeypatch.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: backend if backend else _no_gguf()
+ )
+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_switch)
+
+ app = FastAPI()
+ app.include_router(inference_route.router, prefix = prefix)
+ app.dependency_overrides[get_current_subject] = lambda: "tester"
+ return TestClient(app, raise_server_exceptions = False)
+
+
+def _part_body(
+ *urls,
+ text = "what happens here?",
+ **extra,
+):
+ parts = [{"type": "video_url", "video_url": {"url": u}} for u in urls]
+ parts.append({"type": "text", "text": text})
+ return {
+ "model": "test/model.gguf",
+ "stream": False,
+ "messages": [{"role": "user", "content": parts}],
+ **extra,
+ }
+
+
+def _field_body(
+ clip = _DATA_URI,
+ text = "what happens here?",
+ **extra,
+):
+ return {
+ "model": "test/model.gguf",
+ "stream": False,
+ "messages": [{"role": "user", "content": [{"type": "text", "text": text}]}],
+ "video_base64": clip,
+ **extra,
+ }
+
+
+def _detail(response) -> str:
+ """The human message, from a bare detail or an OpenAI error body.
+
+ Falls back to raw text, since a request that gets past the refusals answers with SSE or an
+ empty body and a test asserting a refusal is *absent* must read those too.
+ """
+ try:
+ body = response.json()
+ except ValueError:
+ return response.text
+ if not isinstance(body, dict):
+ return str(body)
+ detail = body.get("detail", body)
+ if isinstance(detail, dict):
+ return str(detail.get("error", detail).get("message", detail))
+ return str(detail)
+
+
+def _sent_parts(backend, index = -1):
+ """Parts of a dispatched user turn. A system turn is prepended, so index from the user ones."""
+ user_turns = [
+ m
+ for m in backend.dispatched[0]["messages"]
+ if m.get("role") == "user" and isinstance(m.get("content"), list)
+ ]
+ return user_turns[index]["content"]
+
+
+def _sent_media(backend):
+ """Every input_video part across every dispatched turn, in order."""
+ return [
+ part
+ for message in backend.dispatched[0]["messages"]
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ if isinstance(part, dict) and part.get("type") == "input_video"
+ ]
+
+
+ "url, expected",
+ [
+ (_DATA_URI, {"data": _CLIP_B64}),
+ # The header is not payload; llama.cpp's own base64 decoder takes the body alone.
+ (f"DATA:video/mp4;BASE64,{_CLIP_B64}", {"data": _CLIP_B64}),
+ ("data:video/webm;base64,REVG", {"data": "REVG"}),
+ # Bare base64 with no header: handle_media's final fallback treats it as payload.
+ (_CLIP_B64, {"data": _CLIP_B64}),
+ ],
+)
+def test_a_clip_reaches_llama_server_as_input_video(monkeypatch, url, expected):
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ response = client.post("/v1/chat/completions", json = _part_body(url))
+ assert response.status_code == 200
+ assert _sent_media(backend) == [{"type": "input_video", "input_video": expected}]
+
+
+def test_the_translated_part_is_the_only_video_key_left(monkeypatch):
+ """A part left spelled video_url would be refused by llama-server as an unsupported type:
+ it accepts image_url, input_audio and input_video, and nothing else."""
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI))
+ types = [p.get("type") for p in _sent_parts(backend)]
+ assert "video_url" not in types
+ assert len(_sent_media(backend)) == 1
+
+
+def test_the_text_of_the_turn_survives_translation(monkeypatch):
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI, text = "what colour?"))
+ assert {"type": "text", "text": "what colour?"} in _sent_parts(backend)
+
+
+def test_every_clip_in_a_turn_is_translated(monkeypatch):
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ client.post(
+ "/v1/chat/completions", json = _part_body(_DATA_URI, "data:video/webm;base64,REVG")
+ )
+ assert [p["input_video"] for p in _sent_media(backend)] == [
+ {"data": _CLIP_B64},
+ {"data": "REVG"},
+ ]
+
+
+def test_a_clip_in_an_older_turn_is_translated_too(monkeypatch):
+ """_inject_video_part only ever touches the newest user turn, so a parts-carried clip on an
+ earlier turn would otherwise reach llama-server still spelled video_url."""
+ backend = _VideoGguf()
+ body = {
+ "model": "test/model.gguf",
+ "stream": False,
+ "messages": [
+ {"role": "user", "content": [{"type": "video_url", "video_url": {"url": _DATA_URI}}]},
+ {"role": "assistant", "content": "a red square"},
+ {"role": "user", "content": [{"type": "text", "text": "and now?"}]},
+ ],
+ }
+ with _client(monkeypatch, backend) as client:
+ client.post("/v1/chat/completions", json = body)
+ # Translated in place, on the turn that carried it.
+ assert _sent_parts(backend, 0)[0]["type"] == "input_video"
+ assert len(_sent_media(backend)) == 1
+
+
+def test_the_legacy_field_still_rides_the_newest_user_turn(monkeypatch):
+ """Backwards compatibility: the spelling the Studio frontend sends is unchanged."""
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ response = client.post("/v1/chat/completions", json = _field_body())
+ assert response.status_code == 200
+ assert _sent_parts(backend)[-1] == {"type": "input_video", "input_video": {"data": _CLIP_B64}}
+
+
+def test_both_spellings_of_the_same_clip_produce_the_same_wire_shape(monkeypatch):
+ """The invariant the whole change rests on, asserted on the dispatched body itself."""
+ part_backend, field_backend = _VideoGguf(), _VideoGguf()
+ with _client(monkeypatch, part_backend) as client:
+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI))
+ with _client(monkeypatch, field_backend) as client:
+ client.post("/v1/chat/completions", json = _field_body())
+
+ assert _sent_media(part_backend) == _sent_media(field_backend)
+
+
[email protected]("body", [_part_body(_DATA_URI), _field_body()])
+def test_a_gguf_without_a_video_projector_refuses_either_spelling(monkeypatch, body):
+ with _client(monkeypatch, _VideoGguf(has_video = False)) as client:
+ response = client.post("/v1/chat/completions", json = body)
+ assert response.status_code == 400
+ assert "cannot take video input" in _detail(response)
+
+
[email protected]("body", [_part_body(_DATA_URI), _field_body()])
+def test_a_non_gguf_backend_is_offered_the_clip_rather_than_refused_outright(monkeypatch, body):
+ """The regression guard. A blanket 'not using_gguf' refusal ahead of _local_video_clip
+ refuses video on transformers and MLX for BOTH spellings, including the legacy field the
+ frontend sends, breaking working MLX video on macOS.
+ """
+ reached: list[dict] = []
+
+ def _gate(payload, model_info):
+ reached.append(model_info)
+ return _CLIP_B64
+
+ monkeypatch.setattr(
+ inference_route,
+ "get_inference_backend",
+ lambda: SimpleNamespace(
+ active_model_name = "mlx-model",
+ models = {"mlx-model": {"is_vision": True, "has_video_input": True}},
+ ),
+ )
+ monkeypatch.setattr(inference_route, "_local_video_clip", _gate)
+ with _client(monkeypatch, None) as client:
+ response = client.post("/v1/chat/completions", json = body)
+
+ # Reached-the-gate alone cannot tell the regression from the fix: a blanket refusal placed
+ # after the gate still lets the gate run.
+ assert reached == [{"is_vision": True, "has_video_input": True}]
+ assert "only supported on a local GGUF model" not in _detail(response)
+ assert inference_route._VIDEO_INPUT_REFUSAL not in _detail(response)
+
+
+def test_the_non_gguf_gate_reads_both_spellings(monkeypatch):
+ """_local_video_clip predates the part, so left reading payload.video_base64 alone it would
+ hand an MLX model nothing while the legacy field worked."""
+ from models.inference import ChatCompletionRequest
+
+ info = {"is_vision": True, "has_video_input": True}
+ field = ChatCompletionRequest(model = "m", messages = [], video_base64 = _DATA_URI)
+ part = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
+ assert inference_route._local_video_clip(field, info) == _CLIP_B64
+ assert inference_route._local_video_clip(part, info) == _CLIP_B64
+
+
+def test_a_non_gguf_backend_without_video_refuses_by_name():
+ from fastapi import HTTPException
+ from models.inference import ChatCompletionRequest
+
+ payload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
+ with pytest.raises(HTTPException) as exc:
+ inference_route._local_video_clip(payload, {"is_vision": True})
+ assert exc.value.status_code == 400
+ assert exc.value.detail == inference_route._VIDEO_INPUT_REFUSAL
+ # The refusal names MLX: that backend serves video, so a GGUF-only message misinforms.
+ assert "MLX" in inference_route._VIDEO_INPUT_REFUSAL
+
+
+def test_a_second_clip_is_refused_rather_than_dropped_on_a_non_gguf_backend():
+ """Generation takes one video kwarg. Taking clips[0] silently changed the prompt by
+ backend: GGUF forwards every clip, so the same request meant two different things."""
+ from fastapi import HTTPException
+ from models.inference import ChatCompletionRequest
+
+ info = {"is_vision": True, "has_video_input": True}
+ two = ChatCompletionRequest.model_validate(_part_body(_DATA_URI, _DATA_URI))
+ with pytest.raises(HTTPException) as exc:
+ inference_route._local_video_clip(two, info)
+ assert exc.value.status_code == 400
+ assert "Only one video" in exc.value.detail
+
+ # The legacy field beside a part is the same two-clip request in a different spelling.
+ both = ChatCompletionRequest.model_validate(
+ _part_body(_DATA_URI, **{"video_base64": _DATA_URI})
+ )
+ with pytest.raises(HTTPException) as exc:
+ inference_route._local_video_clip(both, info)
+ assert exc.value.status_code == 400
+
+ # One clip still serves, so the guard did not swallow the ordinary case.
+ one = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
+ assert inference_route._local_video_clip(one, info) == _CLIP_B64
+
+
+def test_an_unsupported_scheme_is_refused_on_the_non_gguf_path_too():
+ """_request_video_rejection runs only when a pre-switch validation happens, so with
+ auto-switch off a file:// clip reached the backend and was decoded as base64."""
+ from fastapi import HTTPException
+ from models.inference import ChatCompletionRequest
+
+ payload = ChatCompletionRequest.model_validate(_part_body("file:///etc/passwd"))
+ with pytest.raises(HTTPException) as exc:
+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ assert exc.value.status_code == 400
+ assert "Unsupported video URL scheme" in exc.value.detail
+
+
+def test_admission_compacts_a_clip_after_translation_too():
+ """The server-side tool loop recosts the conversation after _translate_video_parts has
+ renamed the part, so matching video_url alone priced the payload as dense prompt text."""
+ import copy
+
+ big = "data:video/mp4;base64," + "A" * 40_000
+ msgs = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "video_url", "video_url": {"url": big}},
+ {"type": "text", "text": "hi"},
+ ],
+ }
+ ]
+
+ translated = copy.deepcopy(msgs)
+ inference_route._translate_video_parts(translated)
+ before, _ = inference_route._openai_llama_admission_messages_for_estimate(copy.deepcopy(msgs))
+ after, _ = inference_route._openai_llama_admission_messages_for_estimate(translated)
+
+ assert "A" * 40_000 not in str(after)
+ # Both spellings compact to about the same size; the clip is a marker, not prompt text.
+ assert abs(len(str(after)) - len(str(before))) < 200
+
+
+def test_a_remote_clip_is_refused_on_a_non_gguf_backend():
+ """Only llama-server fetches a clip for itself. A transformers or MLX model is handed bytes,
+ so forwarding the URL would feed it the text of the URL instead of the video."""
+ from fastapi import HTTPException
+ from models.inference import ChatCompletionRequest
+
+ payload = ChatCompletionRequest.model_validate(_part_body(_REMOTE))
+ with pytest.raises(HTTPException) as exc:
+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ assert exc.value.status_code == 400
+ assert "Remote video URLs are not supported" in exc.value.detail
+
+
+ "url",
+ [
+ "file:///etc/passwd",
+ "FILE:///etc/passwd",
+ "ftp://example.com/clip.mp4",
+ "gopher://example.com/clip.mp4",
+ ],
+)
+def test_an_unsupported_scheme_is_refused_by_name(monkeypatch, url):
+ """handle_media reads input_video as data-or-url and treats the one string it finds the
+ same way, honouring file:// under --media-path. Forwarding the clip as opaque payload does
+ not neutralise the scheme, so refuse it here.
+ """
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = _part_body(url))
+ assert response.status_code == 400
+ assert "Unsupported video URL scheme" in _detail(response)
+
+
+def test_a_bare_path_is_refused_without_naming_a_scheme(monkeypatch):
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = _part_body("/tmp/clip.mp4"))
+ # No colon: indistinguishable from base64 payload, so llama-server owns the verdict.
+ assert response.status_code == 200
+
+
+def test_bare_base64_is_not_mistaken_for_a_scheme(monkeypatch):
+ """':' is not in the base64 alphabet, which is what separates a URL from payload. A scheme
+ check that scanned the whole string would refuse legitimate clips."""
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ response = client.post("/v1/chat/completions", json = _part_body("QUJDREVGR0hJSktM"))
+ assert response.status_code == 200
+ assert _sent_media(backend) == [
+ {"type": "input_video", "input_video": {"data": "QUJDREVGR0hJSktM"}}
+ ]
+
+
+def test_an_oversized_clip_is_refused_in_either_spelling():
+ """On the shared rule rather than over HTTP, since the body would be 85 MB of JSON. The
+ ordering guarantee is pinned in test_video_attachment_part.py."""
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _MAX_VIDEO_B64_CHARS
+
+ over = "data:video/mp4;base64," + "A" * (_MAX_VIDEO_B64_CHARS + 1)
+ for payload in (
+ ChatCompletionRequest.model_validate(_part_body(over)),
+ ChatCompletionRequest.model_validate(_field_body(over)),
+ ):
+ assert inference_route._request_video_rejection(payload)[0] == 413
+
+
+def test_a_clip_of_exactly_the_cap_is_admitted():
+ """The boundary the cap exists to allow: refusing it would reject a file the composer offers."""
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _MAX_VIDEO_B64_CHARS
+
+ at_cap = "data:video/mp4;base64," + "A" * _MAX_VIDEO_B64_CHARS
+ payload = ChatCompletionRequest.model_validate(_part_body(at_cap))
+ assert inference_route._request_video_rejection(payload) is None
+
+
+def test_a_data_uri_with_no_payload_is_refused(monkeypatch):
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = _part_body("data:video/mp4;base64,"))
+ assert response.status_code == 400
+ assert "Could not read the provided video file" in _detail(response)
+
+
[email protected]("body", [_part_body(_DATA_URI), _field_body()])
+def test_an_external_provider_refuses_either_spelling(monkeypatch, body):
+ """_build_external_messages rebuilds from an allowlist, so a clip left standing is dropped
+ and the provider answers a prompt the caller did not send."""
+ body = {**body, "provider_type": "openai", "provider_api_key": "sk-test"}
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = body)
+ assert response.status_code == 400
+ assert response.json()["detail"] == inference_route._VIDEO_INPUT_REFUSAL
+
+
[email protected]("body", [_part_body(_DATA_URI), _field_body()])
+def test_token_counting_refuses_either_spelling(monkeypatch, body):
+ """llama-server samples the frames at completion time, so counting here undercounts."""
+ body = {k: v for k, v in body.items() if k != "stream"}
+ with _client(monkeypatch, _VideoGguf(), prefix = "") as client:
+ response = client.post("/chat/count_tokens", json = body)
+ assert response.status_code == 503
+ assert "video" in _detail(response)
+
+
[email protected]("body", [_part_body(_DATA_URI), _field_body()])
+def test_the_speech_route_refuses_a_clip_rather_than_speaking_past_it(monkeypatch, body):
+ """/audio/generate keeps only text, so without this the clip was dropped in silence.
+ Registering video_url as a known tag removed the unknown-part guard that covered it."""
+ body = {k: v for k, v in body.items() if k != "stream"}
+ with _client(monkeypatch, _VideoGguf(), prefix = "") as client:
+ response = client.post("/audio/generate", json = body)
+ assert response.status_code == 400
+ assert "Video input is not supported here" in _detail(response)
+
+
+def test_the_speech_route_still_speaks_a_plain_text_turn(monkeypatch):
+ """The control for the refusal above: it must not swallow ordinary requests."""
+ body = {"model": "default", "messages": [{"role": "user", "content": "read this out"}]}
+ with _client(monkeypatch, _VideoGguf(), prefix = "") as client:
+ response = client.post("/audio/generate", json = body)
+ assert "Video input is not supported here" not in _detail(response)
+
+
[email protected]("body", [_part_body(_DATA_URI), _field_body()])
+def test_the_tool_passthrough_path_refuses_either_spelling(monkeypatch, body):
+ """That branch forwards an explicit field list, so the clip would be dropped silently."""
+ body = {
+ **body,
+ "tools": [
+ {
+ "type": "function",
+ "function": {"name": "f", "parameters": {"type": "object", "properties": {}}},
+ }
+ ],
+ }
+
+ class _ToolGguf(_VideoGguf):
+ supports_tools = True
+
+ with _client(monkeypatch, _ToolGguf()) as client:
+ response = client.post("/v1/chat/completions", json = body)
+ assert response.status_code == 400
+ assert "guided decoding" in _detail(response)
+
+
+def test_a_durable_chat_run_refuses_a_clip_in_either_spelling():
+ """The request payload persists verbatim, so a data URI would sit in request_json for the
+ life of the thread. The gate is field-shaped, so the part needs its own predicate."""
+ from fastapi import HTTPException
+ from routes.chat_generation_runs import CreateChatGenerationRun, _sanitize_request
+
+ for body in (_part_body(_DATA_URI), _field_body()):
+ run = CreateChatGenerationRun(
+ runId = "run-1",
+ threadId = "thread-1",
+ userMessageId = "user-1",
+ assistantMessageId = "assistant-1",
+ requestPayload = {k: v for k, v in body.items() if k != "stream"},
+ )
+ with pytest.raises(HTTPException) as exc:
+ _sanitize_request(run)
+ assert exc.value.status_code == 400
+ assert "Media chat runs" in str(exc.value.detail)
+
+
+def test_a_durable_chat_run_still_takes_a_plain_text_turn():
+ """The control: the media gate must not start refusing ordinary durable runs."""
+ from routes.chat_generation_runs import CreateChatGenerationRun, _sanitize_request
+
+ run = CreateChatGenerationRun(
+ runId = "run-1",
+ threadId = "thread-1",
+ userMessageId = "user-1",
+ assistantMessageId = "assistant-1",
+ requestPayload = {"model": "m", "messages": [{"role": "user", "content": "hi"}]},
+ )
+ assert _sanitize_request(run) is not None
+
+
[email protected]("role", ["system", "assistant"])
+def test_a_clip_on_a_non_user_turn_is_refused(monkeypatch, role):
+ """OpenAI places media on user turns only, and llama-server renders the marker into whichever
+ turn carried it, so anywhere else the result is template-dependent."""
+ body = {
+ "model": "test/model.gguf",
+ "stream": False,
+ "messages": [
+ {"role": role, "content": [{"type": "video_url", "video_url": {"url": _DATA_URI}}]},
+ {"role": "user", "content": "go"},
+ ],
+ }
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = body)
+ assert response.status_code == 422
+
+
+def test_a_clip_on_a_user_turn_is_accepted(monkeypatch):
+ """The control: the role rule must not refuse the placement it exists to protect."""
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = _part_body(_DATA_URI))
+ assert response.status_code == 200
+
+
+def test_admission_charges_each_clip_once_and_not_as_prompt_text():
+ """Pricing a megabytes-long data URI as prompt text would swamp the estimate."""
+ from models.inference import ChatCompletionRequest
+
+ big = "data:video/mp4;base64," + "A" * 40_000
+ one = ChatCompletionRequest.model_validate(_part_body(big))
+ two = ChatCompletionRequest.model_validate(_part_body(big, big))
+ charge_one = inference_route._openai_llama_admission_media_tokens(one)
+ charge_two = inference_route._openai_llama_admission_media_tokens(two)
+ assert charge_two - charge_one == pytest.approx(charge_one, rel = 0.05)
+
+ estimate, _ = inference_route._openai_llama_admission_messages_for_estimate(
+ [m.model_dump(exclude_none = True) for m in one.messages]
+ )
+ assert "A" * 40_000 not in str(estimate)
+
+
+def test_admission_prices_the_two_spellings_of_one_clip_alike():
+ from models.inference import ChatCompletionRequest
+
+ clip = "data:video/mp4;base64," + "A" * 40_000
+ part = ChatCompletionRequest.model_validate(_part_body(clip))
+ field = ChatCompletionRequest.model_validate(_field_body(clip))
+ assert inference_route._openai_llama_admission_media_tokens(
+ part
+ ) == inference_route._openai_llama_admission_media_tokens(field)
+
+
+def test_the_rolling_context_does_not_price_a_clip_as_text():
+ """Trimming counts a turn to decide what to drop; a clip counted as text would evict the
+ conversation around it."""
+ from core.inference.context_window import _UNPRICED_MEDIA_TYPES
+ assert {"video_url", "input_video"} <= set(_UNPRICED_MEDIA_TYPES)
+
+
+def test_an_unknown_part_type_is_still_refused_by_name(monkeypatch):
+ """video_url used to land here. Registering it must not have widened the catch-all: an
+ unregistered type still has to be refused rather than silently dropped."""
+ body = {
+ "model": "test/model.gguf",
+ "stream": False,
+ "messages": [
+ {"role": "user", "content": [{"type": "hologram_url", "hologram_url": {"url": "x"}}]}
+ ],
+ }
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = body)
+ assert response.status_code == 400
+ assert "hologram_url" in _detail(response)
+
+
+ "url",
+ [
+ "https://example.com/clip.mp4",
+ "http://example.com/clip.mp4",
+ "HTTPS://EXAMPLE.COM/Clip.MP4",
+ "http://169.254.169.254/latest/meta-data/",
+ "http://127.0.0.1:8080/clip.mp4",
+ "http://[::1",
+ "https://clips.example:bad/clip.mp4",
+ ],
+)
+def test_a_remote_video_url_is_refused(monkeypatch, url):
+ """llama-server, not Studio, would fetch the clip, and its downloader follows redirects
+ (set_follow_location in common/http.h), so a public URL redirecting to a private address
+ defeats any host check made here. The shape is refused rather than guarded."""
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = _part_body(url))
+ assert response.status_code == 400
+ assert "Remote video URLs are not supported" in _detail(response)
+
+
+def test_no_url_ever_reaches_llama_server_as_a_video(monkeypatch):
+ """The refusal is the point: input_video must always carry bytes, never something to dial."""
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ client.post("/v1/chat/completions", json = _part_body(_DATA_URI))
+ assert all("data" in part["input_video"] for part in _sent_media(backend))
+ assert not any("url" in part["input_video"] for part in _sent_media(backend))
+
+
+def test_the_dispatch_boundary_refuses_a_remote_url_on_its_own():
+ """_request_video_rejection only runs on a pre-switch validation, so translation has to
+ refuse it too rather than trust that it was already checked."""
+ from fastapi import HTTPException
+
+ messages = [{"role": "user", "content": [{"type": "video_url", "video_url": {"url": _REMOTE}}]}]
+ with pytest.raises(HTTPException) as exc:
+ inference_route._translate_video_parts(messages)
+ assert exc.value.status_code == 400
+ assert "Remote video URLs are not supported" in exc.value.detail
+
+
+def test_both_spellings_refuse_a_remote_clip_alike(monkeypatch):
+ with _client(monkeypatch, _VideoGguf()) as client:
+ part = client.post("/v1/chat/completions", json = _part_body(_REMOTE))
+ field = client.post("/v1/chat/completions", json = _field_body(_REMOTE))
+ assert part.status_code == field.status_code == 400
+ assert _detail(part) == _detail(field)
+
+
+def _older_turn_body(clip = _DATA_URI):
+ return {
+ "model": "test/model.gguf",
+ "stream": False,
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "video_url", "video_url": {"url": clip}},
+ {"type": "text", "text": "what is this?"},
+ ],
+ },
+ {"role": "assistant", "content": "a clip"},
+ {"role": "user", "content": [{"type": "text", "text": "and now?"}]},
+ ],
+ }
+
+
+def test_a_clip_on_an_older_turn_is_refused_on_a_non_gguf_backend():
+ """GGUF keeps the part on the turn that carried it; the non-GGUF path flattens the parts and
+ hands generation one video kwarg, which MLX attaches to the newest turn. Refusing keeps the
+ two backends from reading the same request as different conversations."""
+ from fastapi import HTTPException
+ from models.inference import ChatCompletionRequest
+
+ payload = ChatCompletionRequest.model_validate(_older_turn_body())
+ with pytest.raises(HTTPException) as exc:
+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ assert exc.value.status_code == 400
+ assert "attached to the latest message" in exc.value.detail
+
+
+def test_a_clip_on_the_latest_turn_is_still_served_on_a_non_gguf_backend():
+ from models.inference import ChatCompletionRequest
+ payload = ChatCompletionRequest.model_validate(_part_body(_DATA_URI))
+ assert (
+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ == _CLIP_B64
+ )
+
+
+def test_the_legacy_field_is_not_treated_as_an_older_turn():
+ """video_base64 has no turn to belong to, so it is always the newest attachment."""
+ from models.inference import ChatCompletionRequest
+
+ payload = ChatCompletionRequest.model_validate(_field_body())
+ assert inference_route._video_is_on_an_older_turn(payload) is False
+ assert (
+ inference_route._local_video_clip(payload, {"is_vision": True, "has_video_input": True})
+ == _CLIP_B64
+ )
+
+
+def test_gguf_still_keeps_a_clip_on_the_turn_that_carried_it(monkeypatch):
+ """The refusal above is a non-GGUF limit, not a new rule for everyone."""
+ backend = _VideoGguf()
+ with _client(monkeypatch, backend) as client:
+ response = client.post("/v1/chat/completions", json = _older_turn_body())
+ assert response.status_code == 200
+ turns = [
+ m
+ for m in backend.dispatched[0]["messages"]
+ if m.get("role") == "user" and isinstance(m.get("content"), list)
+ ]
+ assert [p.get("type") for p in turns[0]["content"]] == ["input_video", "text"]
+ assert [p.get("type") for p in turns[-1]["content"]] == ["text"]
+
+
+def test_validation_does_not_copy_the_clip():
+ """Validation runs twice per request and only wants the verdict, so slicing the header off
+ to get it copied the whole payload each time: 89 MB for a clip at the limit."""
+ import tracemalloc
+
+ from models.inference import ChatCompletionRequest
+
+ clip = "data:video/mp4;base64," + "A" * (8 * 1024 * 1024)
+ payload = ChatCompletionRequest.model_validate(_part_body(clip))
+ tracemalloc.start()
+ try:
+ assert inference_route._request_video_rejection(payload) is None
+ _, peak = tracemalloc.get_traced_memory()
+ finally:
+ tracemalloc.stop()
+ assert peak < 1024 * 1024, f"validation allocated {peak} bytes for an 8 MB clip"
+
+
+ "clip, expected",
+ [
+ ("data:video/mp4;base64," + "A" * 40, None),
+ ("A" * 40, None),
+ ("", (400, "Could not read the provided video file.")),
+ ("data:video/mp4;base64,", (400, "Could not read the provided video file.")),
+ ("data:video/mp4;base64", (400, "Could not read the provided video file.")),
+ ],
+)
+def test_the_measured_verdict_matches_the_sliced_one(clip, expected):
+ """The fast path has to agree with _video_b64_rejection, header handling included."""
+ assert inference_route._video_size_rejection(clip) == expected
+ assert inference_route._video_b64_rejection(clip)[1] == expected
+
+
+def test_the_measured_verdict_agrees_on_the_cap_boundary():
+ limit = inference_route._MAX_VIDEO_B64_CHARS
+ for length in (limit, limit + 1):
+ clip = "data:video/mp4;base64," + "A" * length
+ assert (
+ inference_route._video_size_rejection(clip)
+ == (inference_route._video_b64_rejection(clip)[1])
+ )
+
+
+def test_a_part_carried_clip_is_transcoded_like_the_legacy_field(monkeypatch):
+ """main shrinks the clip before llama-server samples it, but only on the legacy field's
+ path. Without the same pass over the parts, the same clip would reach llama-server at a
+ different resolution depending on which spelling carried it."""
+ seen = []
+
+ def _shrink(
+ clip,
+ cap,
+ *,
+ sampled_fps = None,
+ ):
+ seen.append(clip)
+ return "SHRUNK"
+
+ monkeypatch.setattr(inference_route, "shrink_video_for_llama", _shrink)
+
+ part_backend = _VideoGguf()
+ with _client(monkeypatch, part_backend) as client:
+ assert client.post("/v1/chat/completions", json = _part_body(_DATA_URI)).status_code == 200
+ field_backend = _VideoGguf()
+ with _client(monkeypatch, field_backend) as client:
+ assert client.post("/v1/chat/completions", json = _field_body()).status_code == 200
+
+ assert seen == [_CLIP_B64, _CLIP_B64], "both spellings must reach the transcoder"
+ assert _sent_media(part_backend) == _sent_media(field_backend)
+ assert _sent_media(part_backend) == [{"type": "input_video", "input_video": {"data": "SHRUNK"}}]
+
+
+def test_several_clips_are_capped_in_aggregate_not_only_per_clip():
+ """Every clip is transcoded and shrink_video_for_llama allows 300s each, so N clips just
+ under the per-clip limit is N*300s of ffmpeg for one request."""
+ from models.inference import ChatCompletionRequest
+
+ half = "data:video/mp4;base64," + "A" * (inference_route._MAX_VIDEO_B64_CHARS // 2 + 10)
+ one = ChatCompletionRequest.model_validate(_part_body(half))
+ assert inference_route._request_video_rejection(one) is None
+ two = ChatCompletionRequest.model_validate(_part_body(half, half))
+ rejection = inference_route._request_video_rejection(two)
+ assert rejection is not None and rejection[0] == 413
+ assert "per request" in rejection[1]
+
+
+def test_the_aggregate_cap_counts_the_legacy_field_too():
+ """Otherwise the field plus a part could exceed it together."""
+ from models.inference import ChatCompletionRequest
+
+ half = "data:video/mp4;base64," + "A" * (inference_route._MAX_VIDEO_B64_CHARS // 2 + 10)
+ body = _part_body(half)
+ body["video_base64"] = half
+ rejection = inference_route._request_video_rejection(ChatCompletionRequest.model_validate(body))
+ assert rejection is not None and rejection[0] == 413
+
+
+def test_recosting_drops_a_clip_the_conversation_has_evicted():
+ """truncate_oldest can evict the turn that carried the clip. Charging it from the opening
+ payload kept every later round reserved at the full budget for media no longer sent."""
+ from models.inference import ChatCompletionRequest
+
+ clip = "data:video/mp4;base64," + "A" * 40_000
+ payload = ChatCompletionRequest.model_validate(_part_body(clip))
+ opening = inference_route._openai_llama_admission_media_tokens(payload)
+ assert opening > 1000
+
+ evicted = inference_route._openai_llama_admission_media_tokens(
+ payload, message_video_clips = inference_route._conversation_video_clips([])
+ )
+ assert evicted == 0
+
+
+def test_recosting_still_charges_a_clip_the_conversation_kept():
+ """Reading video_url alone would find nothing post-translation and charge no video at all."""
+ from models.inference import ChatCompletionRequest
+
+ clip = "A" * 40_000
+ payload = ChatCompletionRequest.model_validate(_part_body("data:video/mp4;base64," + clip))
+ conversation = [
+ {"role": "user", "content": [{"type": "input_video", "input_video": {"data": clip}}]}
+ ]
+ kept = inference_route._openai_llama_admission_media_tokens(
+ payload, message_video_clips = inference_route._conversation_video_clips(conversation)
+ )
+ assert kept == len(clip) // 4
+
+
+def test_the_legacy_field_is_not_charged_twice_during_recosting():
+ """_inject_video_part splices the legacy clip into the conversation as input_video before
+ the loop starts, so during a recost the conversation clips already include it. Charging the
+ field as well priced it at exactly 2x and clamped the lease toward the whole KV budget."""
+ from models.inference import ChatCompletionRequest
+
+ clip = "A" * 40_000
+ payload = ChatCompletionRequest.model_validate(
+ _field_body("data:video/mp4;base64," + clip, text = "hi")
+ )
+ conversation = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
+ inference_route._inject_video_part(conversation, clip)
+
+ opening = inference_route._openai_llama_admission_media_tokens(payload)
+ recost = inference_route._openai_llama_admission_media_tokens(
+ payload,
+ message_video_clips = inference_route._conversation_video_clips(conversation),
+ )
+ assert recost == pytest.approx(opening, rel = 0.01)
+
+
+def test_audio_is_still_charged_from_the_field_during_recosting():
+ """Only video is spliced into the conversation; dropping audio with it would undercharge."""
+ from models.inference import ChatCompletionRequest
+
+ body = _field_body("data:video/mp4;base64," + "A" * 400)
+ body["audio_base64"] = "B" * 4000
+ payload = ChatCompletionRequest.model_validate(body)
+ assert (
+ inference_route._openai_llama_admission_media_tokens(payload, message_video_clips = [])
+ == 1000
+ )
+
+
+def test_many_tiny_clips_are_refused_by_count_not_only_by_bytes():
+ """Each clip is its own ffprobe and ffmpeg (30s and 300s ceilings), so thousands of tiny
+ clips stay under the byte cap while holding the request and its lease for hours."""
+ from models.inference import ChatCompletionRequest
+
+ tiny = "data:video/mp4;base64,QUJD"
+ many = ChatCompletionRequest.model_validate(
+ _part_body(*[tiny] * (inference_route._MAX_VIDEO_CLIPS_PER_REQUEST + 1))
+ )
+ rejection = inference_route._request_video_rejection(many)
+ assert rejection is not None and rejection[0] == 400
+ assert "Too many videos" in rejection[1]
+
+
+def test_a_request_at_the_clip_limit_is_still_served():
+ """The count cap must bound abuse, not ordinary multi-clip use."""
+ from models.inference import ChatCompletionRequest
+
+ tiny = "data:video/mp4;base64,QUJD"
+ ok = ChatCompletionRequest.model_validate(
+ _part_body(*[tiny] * inference_route._MAX_VIDEO_CLIPS_PER_REQUEST)
+ )
+ assert inference_route._request_video_rejection(ok) is None
+
+
+def test_the_clip_count_includes_the_legacy_field():
+ """Otherwise the field plus the maximum parts exceeds the bound together."""
+ from models.inference import ChatCompletionRequest
+
+ tiny = "data:video/mp4;base64,QUJD"
+ body = _part_body(*[tiny] * inference_route._MAX_VIDEO_CLIPS_PER_REQUEST)
+ body["video_base64"] = tiny
+ rejection = inference_route._request_video_rejection(ChatCompletionRequest.model_validate(body))
+ assert rejection is not None and rejection[0] == 400
diff --git a/studio/backend/tests/test_video_pr9057_simulation.py b/studio/backend/tests/test_video_pr9057_simulation.py
index 8cb88eebe07..80996739343 100644
--- a/studio/backend/tests/test_video_pr9057_simulation.py
+++ b/studio/backend/tests/test_video_pr9057_simulation.py
@@ -431,7 +431,7 @@ def test_every_path_that_cannot_serve_a_clip_refuses_out_loud(needle):
def test_the_size_check_is_paid_before_the_model_switch_not_after():
src = _routes_source()
handler = src.index("_needs_image = bool(_pre_parsed[2])")
- assert src.index("_video_b64_rejection(payload.video_base64)", handler) < src.index(
+ assert src.index("_request_video_rejection(payload)", handler) < src.index(
"await _maybe_auto_switch_model(", handler
)
@@ -440,4 +440,4 @@ def test_the_external_provider_refusal_precedes_the_proxy_call():
src = _routes_source()
start = src.index("if payload.provider_id or payload.provider_type:")
branch = src[start : src.index("_proxy_to_external_provider(payload", start)]
- assert "payload.video_base64" in branch
+ assert "_request_has_video(payload)" in branch