Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: send video to custom OpenAI-compatible connections (#17322)

Nilayd3cc5bca23ec
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 651e2275574..abdf1d07a9c 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -2574,7 +2574,7 @@ class VideoUrl(BaseModel):
class VideoContentPart(BaseModel):
- """Video content part; served only through llama-server's ``input_video``."""
+ """Video content part for llama-server or a custom OpenAI-compatible connection."""
type: Literal["video_url"]
video_url: VideoUrl
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 4a16c025c26..f1f99811fd2 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -27608,7 +27608,8 @@ def _llama_passthrough_audio(raw: bytes) -> Optional[tuple[str, float]]:
_VIDEO_INPUT_REFUSAL = (
- "Video input is only supported on a local GGUF or MLX model with video support."
+ "Video input is only supported on a local GGUF or MLX model with video support, "
+ "or a custom OpenAI-compatible connection."
)
_VIDEO_NEEDS_FFMPEG_DETAIL = (
"This model takes video, but ffmpeg and ffprobe were not found in llama-server's search path. "
@@ -27928,6 +27929,32 @@ async def _shrink_video_parts(messages: list[dict], llama_backend) -> None:
part["video_url"] = {"url": await _shrink_clip_for_llama(clip, llama_backend)}
+_VIDEO_URL_PROVIDER_TYPES = frozenset({"custom", "vllm", "llama_cpp", "ollama"})
+
+
+async def _external_video_url(clip: str) -> str:
+ bare = _video_b64_rejection(clip)[0]
+ shrunk = await _shrink_clip_for_llama(bare, None)
+ if shrunk != bare:
+ return f"data:video/x-matroska;base64,{shrunk}"
+ return clip if clip[:5].lower() == "data:" else f"data:video/mp4;base64,{bare}"
+
+
+async def _forward_external_video(messages: list[dict], legacy_clip: Optional[str]) -> None:
+ if legacy_clip:
+ for msg in reversed(messages):
+ if msg.get("role") == "user":
+ content = msg.get("content")
+ if not isinstance(content, list):
+ content = msg["content"] = [{"type": "text", "text": content or ""}]
+ content.append({"type": "video_url", "video_url": {"url": legacy_clip}})
+ break
+ for msg in messages:
+ for part in msg.get("content") if isinstance(msg.get("content"), list) else []:
+ if isinstance(part, dict) and part.get("type") == "video_url":
+ part["video_url"] = {"url": await _external_video_url(part["video_url"]["url"])}
+
+
def _translate_video_parts(messages: list[dict]) -> None:
"""Rewrite ``video_url`` parts in place as llama-server's ``input_video``, always as bytes."""
for msg in messages:
@@ -28588,6 +28615,10 @@ def _build_external_messages(
"image_url": {"url": part.image_url.url},
}
)
+ elif part.type == "video_url":
+ parts.append(
+ {"type": "video_url", "video_url": {"url": part.video_url.url}}
+ )
elif (
responses_native_parts
and msg.role == "assistant"
@@ -29063,6 +29094,14 @@ async def _proxy_to_external_provider(
detail = "Either provider_id or provider_type is required for external provider routing.",
)
_refuse_decision_connection(provider_type, api_type)
+ if _request_has_video(payload):
+ if provider_type not in _VIDEO_URL_PROVIDER_TYPES or (
+ provider_type == "custom" and api_type == "responses"
+ ):
+ raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)
+ rejection = _request_video_rejection(payload)
+ if rejection is not None:
+ raise HTTPException(status_code = rejection[0], detail = rejection[1])
if provider_type == "custom":
try:
validate_provider_reasoning_contract(provider_type, api_type, reasoning_config)
@@ -29620,6 +29659,8 @@ async def _proxy_to_external_provider(
model = model,
preserve_thinking = payload.preserve_thinking,
)
+ if _request_has_video(payload):
+ await _forward_external_video(chat_messages, payload.video_base64)
monitor_id = None
if not getattr(request.state, "skip_api_monitor", False):
monitor_id = api_monitor.start(
@@ -31352,9 +31393,6 @@ async def produce_openai_chat_completions(
untrack_current_request(request.scope)
if _wants_multiple_choices(payload):
_raise_unsupported_n("external provider chat completions")
- # Refuse video here: the proxy cannot place a clip, and video_url is refused as the legacy field is.
- if _request_has_video(payload):
- raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)
# _build_external_messages has no input_audio case; refuse rather than drop it.
if _messages_have_input_audio(payload.messages):
raise HTTPException(
diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.py
index e950a8e46aa..3d82007027c 100644
--- a/studio/backend/tests/test_video_attachment_part.py
+++ b/studio/backend/tests/test_video_attachment_part.py
@@ -141,16 +141,6 @@ def test_video_joins_the_projector_requirement_before_switching():
assert start < source.index("_request_has_video(payload)", start) < switch
-def test_an_external_provider_refuses_video_rather_than_ignoring_it():
- """input_video is llama.cpp's own part type, so the proxy cannot carry the clip."""
- source = _inference_source()
- start = source.index("if payload.provider_id or payload.provider_type:")
- branch = source[start : source.index("_proxy_to_external_provider(payload", start)]
- # Both spellings through one predicate, so a video_url part cannot reach the proxy either.
- assert "_request_has_video(payload)" in branch
- assert "_VIDEO_INPUT_REFUSAL" in branch
-
-
def _model_info(**fields):
return {"is_vision": True, **fields}
@@ -221,7 +211,7 @@ def test_token_counting_refuses_video_like_image_and_audio():
def test_both_video_checks_share_one_rule():
"""Pre-switch and post-load size checks must agree, or a load is wasted on a refused clip."""
source = _inference_source()
- assert source.count("= _request_video_rejection(payload)") == 3
+ assert source.count("= _request_video_rejection(payload)") == 4
def _video_url_request(
diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.py
index 33333043f92..3d2e774f2b1 100644
--- a/studio/backend/tests/test_video_chat_completions_behaviour.py
+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py
@@ -458,6 +458,43 @@ def test_an_external_provider_refuses_either_spelling(monkeypatch, body):
assert response.json()["detail"] == inference_route._VIDEO_INPUT_REFUSAL
[email protected]("body", [_part_body(_DATA_URI), _field_body()])
[email protected]("provider_type", ["custom", "vllm"])
+def test_a_custom_connection_is_handed_the_clip_like_an_image(monkeypatch, body, provider_type):
+ import httpx
+ from core.inference import external_provider as ep_mod
+ from .test_external_provider_sampling_over_the_wire import _Server
+
+ monkeypatch.setattr(ep_mod, "_http_client", httpx.AsyncClient(trust_env = False))
+ with _Server() as server, _client(monkeypatch, _VideoGguf()) as client:
+ body = {
+ **body,
+ "stream": True,
+ "provider_type": provider_type,
+ "provider_base_url": server.base_url,
+ }
+ response = client.post("/v1/chat/completions", json = body)
+ assert response.status_code == 200, response.text
+ sent = server.bodies[-1]["messages"][-1]["content"]
+ assert {"type": "video_url", "video_url": {"url": _DATA_URI}} in sent
+ assert {"type": "text", "text": "what happens here?"} in sent
+
+
+ "connection",
+ [
+ {"provider_type": "anthropic"},
+ {"provider_type": "custom", "provider_api_type": "responses"},
+ ],
+)
+def test_a_connection_that_cannot_place_a_clip_still_refuses_it(monkeypatch, connection):
+ body = {**_field_body(), **connection, "provider_base_url": "http://127.0.0.1:9/v1"}
+ with _client(monkeypatch, _VideoGguf()) as client:
+ response = client.post("/v1/chat/completions", json = body)
+ assert response.status_code == 400
+ assert response.json()["detail"] == inference_route._VIDEO_INPUT_REFUSAL
+
+
@pytest.mark.parametrize("body", [_part_body(_DATA_URI), _field_body()])
def test_token_counting_refuses_either_spelling(monkeypatch, body):
"""llama-server samples the frames at completion time, so counting here undercounts."""
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index d22fff1e0e1..fc297d7acc4 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -176,6 +176,7 @@ import {
import type { ChatModelAdapter } from "@assistant-ui/react";
import { parsePartialJsonObject } from "assistant-stream/utils";
import {
+ externalModelTakesVideo,
isExternalModelId,
isPromptCacheTtl,
loadExternalProviders,
@@ -5540,6 +5541,7 @@ export function createOpenAIStreamAdapter(
audio: Boolean(findLatestUserAudioBase64(survivingMessages, false)),
audioCount: latestUserAudioClips(survivingMessages).length,
video: Boolean(videoBase64),
+ externalTakesVideo: externalModelTakesVideo(params.checkpoint),
});
if (attachedMediaReason) {
blockAttachmentRun(attachedMediaReason);
@@ -6727,6 +6729,7 @@ export function createOpenAIStreamAdapter(
: {}),
stream: true,
stream_options: { include_usage: true },
+ video_base64: findLatestUserVideoBase64(currentTurnMessages),
// Never forwarded upstream; the trailing assistant turn is what requests continuation.
...(continuation ? { continue_final_message: true } : {}),
// Reasoning-class models (OpenAI gpt-5.x / o3) reject temperature and top_p; forward only when supported.
diff --git a/studio/frontend/src/features/chat/external-providers.ts b/studio/frontend/src/features/chat/external-providers.ts
index a4c11b79a5e..4f2d0376809 100644
--- a/studio/frontend/src/features/chat/external-providers.ts
+++ b/studio/frontend/src/features/chat/external-providers.ts
@@ -346,6 +346,22 @@ export function externalModelSupportsStudioTools(
);
}
+export function externalModelTakesVideo(
+ checkpoint: string | null | undefined,
+): boolean {
+ const selection = parseExternalModelId(checkpoint);
+ if (!selection) return false;
+ const provider = loadExternalProviders().find(
+ (candidate) => candidate.id === selection.providerId,
+ );
+ if (!provider) return false;
+ const providerType = provider.backendProviderType ?? provider.providerType;
+ return (
+ isCustomProviderType(providerType) &&
+ !(providerType === LEGACY_CUSTOM_PROVIDER_TYPE && provider.apiType === "responses")
+ );
+}
+
export const CUSTOM_BACKEND_PROVIDER_TYPE = "openai";
export const LEGACY_CUSTOM_PROVIDER_TYPE = "custom";
export const CUSTOM_PROVIDER_DISPLAY_NAME = "Custom";
diff --git a/studio/frontend/src/features/chat/lib/attached-media-gate.ts b/studio/frontend/src/features/chat/lib/attached-media-gate.ts
index 8b56307267f..b0ff5c1f729 100644
--- a/studio/frontend/src/features/chat/lib/attached-media-gate.ts
+++ b/studio/frontend/src/features/chat/lib/attached-media-gate.ts
@@ -8,6 +8,7 @@ export function attachedMediaUnavailableReason({
audio,
audioCount = audio ? 1 : 0,
video,
+ externalTakesVideo = false,
}: {
activeModel:
| { hasAudioInput?: boolean; hasVideoInput?: boolean; isMlx?: boolean }
@@ -17,6 +18,7 @@ export function attachedMediaUnavailableReason({
audio: boolean;
audioCount?: number;
video: boolean;
+ externalTakesVideo?: boolean;
}): string | null {
if (!checkpoint || (!audio && !video)) return null;
if (audio && !activeModel?.hasAudioInput) {
@@ -25,7 +27,7 @@ export function attachedMediaUnavailableReason({
if (audio && audioCount > 1 && activeModel?.isMlx) {
return `${modelLabel} takes one audio file per message. Remove the extra audio files, or load a GGUF model to send several.`;
}
- if (video && !activeModel?.hasVideoInput) {
+ if (video && !activeModel?.hasVideoInput && !externalTakesVideo) {
return `${modelLabel} cannot accept video. Load a model that reads video, or remove the video.`;
}
return null;
diff --git a/studio/frontend/src/features/chat/video-attachment-adapter.ts b/studio/frontend/src/features/chat/video-attachment-adapter.ts
index 5bd4b970f63..9b327943d73 100644
--- a/studio/frontend/src/features/chat/video-attachment-adapter.ts
+++ b/studio/frontend/src/features/chat/video-attachment-adapter.ts
@@ -14,6 +14,7 @@ import type {
PendingAttachment,
} from "@assistant-ui/react";
import { toast } from "sonner";
+import { externalModelTakesVideo } from "./external-providers";
import { externalModelLabel } from "./lib/external-model-label";
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
@@ -36,7 +37,11 @@ export class VideoAttachmentAdapter implements AttachmentAdapter {
const activeModel = state.models.find((m) => m.id === checkpoint);
const modelLoaded = !!checkpoint && !state.modelLoading;
let unavailableReason: string | null = null;
- if (modelLoaded && !activeModel?.hasVideoInput) {
+ if (
+ modelLoaded &&
+ !activeModel?.hasVideoInput &&
+ !externalModelTakesVideo(checkpoint)
+ ) {
const label =
activeModel?.name ||
externalModelLabel(checkpoint) ||
diff --git a/studio/frontend/tests/chat-attachment-display.test.ts b/studio/frontend/tests/chat-attachment-display.test.ts
index 986498cead9..07f47c6a507 100644
--- a/studio/frontend/tests/chat-attachment-display.test.ts
+++ b/studio/frontend/tests/chat-attachment-display.test.ts
@@ -213,6 +213,31 @@ test("composer attachment cards are keyed and scoped by attachment id", async ()
assert.match(scopes, /aui\.composer\(\)\.attachment\(\{ id \}\)/);
});
+test("a custom OpenAI-compatible connection is sent video like an image", async () => {
+ const base = {
+ checkpoint: "external::conn::vision-model",
+ modelLabel: "vision-model",
+ activeModel: undefined,
+ audio: false,
+ video: true,
+ };
+ assert.equal(
+ attachedMediaUnavailableReason({ ...base, externalTakesVideo: true }),
+ null,
+ );
+ assert.match(
+ attachedMediaUnavailableReason(base) ?? "",
+ /^vision-model cannot accept video\./,
+ );
+ const adapter = await readSrcAsync("features/chat/api/chat-adapter.ts");
+ const start = adapter.indexOf("model: externalSelection.modelId,");
+ const external = adapter.slice(
+ start,
+ adapter.indexOf("...externalReasoningFields,", start),
+ );
+ assert.match(external, /video_base64: findLatestUserVideoBase64\(currentTurnMessages\)/);
+});
+
test("audio or video attached before a model loaded is checked when sent", () => {
const listener = { hasAudioInput: true, hasVideoInput: false };
const textOnly = { hasAudioInput: false, hasVideoInput: false };