Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: send video to custom OpenAI-compatible connections (#17322)
Nilayd3cc5bca23ec
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 651e2275574..abdf1d07a9c 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -2574,7 +2574,7 @@ class VideoUrl(BaseModel):class VideoContentPart(BaseModel):- """Video content part; served only through llama-server's ``input_video``."""+ """Video content part for llama-server or a custom OpenAI-compatible connection."""type: Literal["video_url"]video_url: VideoUrldiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 4a16c025c26..f1f99811fd2 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -27608,7 +27608,8 @@ def _llama_passthrough_audio(raw: bytes) -> Optional[tuple[str, float]]:_VIDEO_INPUT_REFUSAL = (- "Video input is only supported on a local GGUF or MLX model with video support."+ "Video input is only supported on a local GGUF or MLX model with video support, "+ "or a custom OpenAI-compatible connection.")_VIDEO_NEEDS_FFMPEG_DETAIL = ("This model takes video, but ffmpeg and ffprobe were not found in llama-server's search path. "@@ -27928,6 +27929,32 @@ async def _shrink_video_parts(messages: list[dict], llama_backend) -> None:part["video_url"] = {"url": await _shrink_clip_for_llama(clip, llama_backend)}+_VIDEO_URL_PROVIDER_TYPES = frozenset({"custom", "vllm", "llama_cpp", "ollama"})+++async def _external_video_url(clip: str) -> str:+ bare = _video_b64_rejection(clip)[0]+ shrunk = await _shrink_clip_for_llama(bare, None)+ if shrunk != bare:+ return f"data:video/x-matroska;base64,{shrunk}"+ return clip if clip[:5].lower() == "data:" else f"data:video/mp4;base64,{bare}"+++async def _forward_external_video(messages: list[dict], legacy_clip: Optional[str]) -> None:+ if legacy_clip:+ for msg in reversed(messages):+ if msg.get("role") == "user":+ content = msg.get("content")+ if not isinstance(content, list):+ content = msg["content"] = [{"type": "text", "text": content or ""}]+ content.append({"type": "video_url", "video_url": {"url": legacy_clip}})+ break+ for msg in messages:+ for part in msg.get("content") if isinstance(msg.get("content"), list) else []:+ if isinstance(part, dict) and part.get("type") == "video_url":+ part["video_url"] = {"url": await _external_video_url(part["video_url"]["url"])}++def _translate_video_parts(messages: list[dict]) -> None:"""Rewrite ``video_url`` parts in place as llama-server's ``input_video``, always as bytes."""for msg in messages:@@ -28588,6 +28615,10 @@ def _build_external_messages("image_url": {"url": part.image_url.url},})+ elif part.type == "video_url":+ parts.append(+ {"type": "video_url", "video_url": {"url": part.video_url.url}}+ )elif (responses_native_partsand msg.role == "assistant"@@ -29063,6 +29094,14 @@ async def _proxy_to_external_provider(detail = "Either provider_id or provider_type is required for external provider routing.",)_refuse_decision_connection(provider_type, api_type)+ if _request_has_video(payload):+ if provider_type not in _VIDEO_URL_PROVIDER_TYPES or (+ provider_type == "custom" and api_type == "responses"+ ):+ raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)+ rejection = _request_video_rejection(payload)+ if rejection is not None:+ raise HTTPException(status_code = rejection[0], detail = rejection[1])if provider_type == "custom":try:validate_provider_reasoning_contract(provider_type, api_type, reasoning_config)@@ -29620,6 +29659,8 @@ async def _proxy_to_external_provider(model = model,preserve_thinking = payload.preserve_thinking,)+ if _request_has_video(payload):+ await _forward_external_video(chat_messages, payload.video_base64)monitor_id = Noneif not getattr(request.state, "skip_api_monitor", False):monitor_id = api_monitor.start(@@ -31352,9 +31393,6 @@ async def produce_openai_chat_completions(untrack_current_request(request.scope)if _wants_multiple_choices(payload):_raise_unsupported_n("external provider chat completions")- # Refuse video here: the proxy cannot place a clip, and video_url is refused as the legacy field is.- if _request_has_video(payload):- raise HTTPException(status_code = 400, detail = _VIDEO_INPUT_REFUSAL)# _build_external_messages has no input_audio case; refuse rather than drop it.if _messages_have_input_audio(payload.messages):raise HTTPException(diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.pyindex e950a8e46aa..3d82007027c 100644--- a/studio/backend/tests/test_video_attachment_part.py+++ b/studio/backend/tests/test_video_attachment_part.py@@ -141,16 +141,6 @@ def test_video_joins_the_projector_requirement_before_switching():assert start < source.index("_request_has_video(payload)", start) < switch-def test_an_external_provider_refuses_video_rather_than_ignoring_it():- """input_video is llama.cpp's own part type, so the proxy cannot carry the clip."""- source = _inference_source()- start = source.index("if payload.provider_id or payload.provider_type:")- branch = source[start : source.index("_proxy_to_external_provider(payload", start)]- # Both spellings through one predicate, so a video_url part cannot reach the proxy either.- assert "_request_has_video(payload)" in branch- assert "_VIDEO_INPUT_REFUSAL" in branch--def _model_info(**fields):return {"is_vision": True, **fields}@@ -221,7 +211,7 @@ def test_token_counting_refuses_video_like_image_and_audio():def test_both_video_checks_share_one_rule():"""Pre-switch and post-load size checks must agree, or a load is wasted on a refused clip."""source = _inference_source()- assert source.count("= _request_video_rejection(payload)") == 3+ assert source.count("= _request_video_rejection(payload)") == 4def _video_url_request(diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.pyindex 33333043f92..3d2e774f2b1 100644--- a/studio/backend/tests/test_video_chat_completions_behaviour.py+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py@@ -458,6 +458,43 @@ def test_an_external_provider_refuses_either_spelling(monkeypatch, body):assert response.json()["detail"] == inference_route._VIDEO_INPUT_REFUSAL[email protected]("body", [_part_body(_DATA_URI), _field_body()])[email protected]("provider_type", ["custom", "vllm"])+def test_a_custom_connection_is_handed_the_clip_like_an_image(monkeypatch, body, provider_type):+ import httpx+ from core.inference import external_provider as ep_mod+ from .test_external_provider_sampling_over_the_wire import _Server++ monkeypatch.setattr(ep_mod, "_http_client", httpx.AsyncClient(trust_env = False))+ with _Server() as server, _client(monkeypatch, _VideoGguf()) as client:+ body = {+ **body,+ "stream": True,+ "provider_type": provider_type,+ "provider_base_url": server.base_url,+ }+ response = client.post("/v1/chat/completions", json = body)+ assert response.status_code == 200, response.text+ sent = server.bodies[-1]["messages"][-1]["content"]+ assert {"type": "video_url", "video_url": {"url": _DATA_URI}} in sent+ assert {"type": "text", "text": "what happens here?"} in sent+++ "connection",+ [+ {"provider_type": "anthropic"},+ {"provider_type": "custom", "provider_api_type": "responses"},+ ],+)+def test_a_connection_that_cannot_place_a_clip_still_refuses_it(monkeypatch, connection):+ body = {**_field_body(), **connection, "provider_base_url": "http://127.0.0.1:9/v1"}+ with _client(monkeypatch, _VideoGguf()) as client:+ response = client.post("/v1/chat/completions", json = body)+ assert response.status_code == 400+ assert response.json()["detail"] == inference_route._VIDEO_INPUT_REFUSAL++@pytest.mark.parametrize("body", [_part_body(_DATA_URI), _field_body()])def test_token_counting_refuses_either_spelling(monkeypatch, body):"""llama-server samples the frames at completion time, so counting here undercounts."""diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex d22fff1e0e1..fc297d7acc4 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -176,6 +176,7 @@ import {import type { ChatModelAdapter } from "@assistant-ui/react";import { parsePartialJsonObject } from "assistant-stream/utils";import {+ externalModelTakesVideo,isExternalModelId,isPromptCacheTtl,loadExternalProviders,@@ -5540,6 +5541,7 @@ export function createOpenAIStreamAdapter(audio: Boolean(findLatestUserAudioBase64(survivingMessages, false)),audioCount: latestUserAudioClips(survivingMessages).length,video: Boolean(videoBase64),+ externalTakesVideo: externalModelTakesVideo(params.checkpoint),});if (attachedMediaReason) {blockAttachmentRun(attachedMediaReason);@@ -6727,6 +6729,7 @@ export function createOpenAIStreamAdapter(: {}),stream: true,stream_options: { include_usage: true },+ video_base64: findLatestUserVideoBase64(currentTurnMessages),// Never forwarded upstream; the trailing assistant turn is what requests continuation....(continuation ? { continue_final_message: true } : {}),// Reasoning-class models (OpenAI gpt-5.x / o3) reject temperature and top_p; forward only when supported.diff --git a/studio/frontend/src/features/chat/external-providers.ts b/studio/frontend/src/features/chat/external-providers.tsindex a4c11b79a5e..4f2d0376809 100644--- a/studio/frontend/src/features/chat/external-providers.ts+++ b/studio/frontend/src/features/chat/external-providers.ts@@ -346,6 +346,22 @@ export function externalModelSupportsStudioTools();}+export function externalModelTakesVideo(+ checkpoint: string | null | undefined,+): boolean {+ const selection = parseExternalModelId(checkpoint);+ if (!selection) return false;+ const provider = loadExternalProviders().find(+ (candidate) => candidate.id === selection.providerId,+ );+ if (!provider) return false;+ const providerType = provider.backendProviderType ?? provider.providerType;+ return (+ isCustomProviderType(providerType) &&+ !(providerType === LEGACY_CUSTOM_PROVIDER_TYPE && provider.apiType === "responses")+ );+}+export const CUSTOM_BACKEND_PROVIDER_TYPE = "openai";export const LEGACY_CUSTOM_PROVIDER_TYPE = "custom";export const CUSTOM_PROVIDER_DISPLAY_NAME = "Custom";diff --git a/studio/frontend/src/features/chat/lib/attached-media-gate.ts b/studio/frontend/src/features/chat/lib/attached-media-gate.tsindex 8b56307267f..b0ff5c1f729 100644--- a/studio/frontend/src/features/chat/lib/attached-media-gate.ts+++ b/studio/frontend/src/features/chat/lib/attached-media-gate.ts@@ -8,6 +8,7 @@ export function attachedMediaUnavailableReason({audio,audioCount = audio ? 1 : 0,video,+ externalTakesVideo = false,}: {activeModel:| { hasAudioInput?: boolean; hasVideoInput?: boolean; isMlx?: boolean }@@ -17,6 +18,7 @@ export function attachedMediaUnavailableReason({audio: boolean;audioCount?: number;video: boolean;+ externalTakesVideo?: boolean;}): string | null {if (!checkpoint || (!audio && !video)) return null;if (audio && !activeModel?.hasAudioInput) {@@ -25,7 +27,7 @@ export function attachedMediaUnavailableReason({if (audio && audioCount > 1 && activeModel?.isMlx) {return `${modelLabel} takes one audio file per message. Remove the extra audio files, or load a GGUF model to send several.`;}- if (video && !activeModel?.hasVideoInput) {+ if (video && !activeModel?.hasVideoInput && !externalTakesVideo) {return `${modelLabel} cannot accept video. Load a model that reads video, or remove the video.`;}return null;diff --git a/studio/frontend/src/features/chat/video-attachment-adapter.ts b/studio/frontend/src/features/chat/video-attachment-adapter.tsindex 5bd4b970f63..9b327943d73 100644--- a/studio/frontend/src/features/chat/video-attachment-adapter.ts+++ b/studio/frontend/src/features/chat/video-attachment-adapter.ts@@ -14,6 +14,7 @@ import type {PendingAttachment,} from "@assistant-ui/react";import { toast } from "sonner";+import { externalModelTakesVideo } from "./external-providers";import { externalModelLabel } from "./lib/external-model-label";import { useChatRuntimeStore } from "./stores/chat-runtime-store";@@ -36,7 +37,11 @@ export class VideoAttachmentAdapter implements AttachmentAdapter {const activeModel = state.models.find((m) => m.id === checkpoint);const modelLoaded = !!checkpoint && !state.modelLoading;let unavailableReason: string | null = null;- if (modelLoaded && !activeModel?.hasVideoInput) {+ if (+ modelLoaded &&+ !activeModel?.hasVideoInput &&+ !externalModelTakesVideo(checkpoint)+ ) {const label =activeModel?.name ||externalModelLabel(checkpoint) ||diff --git a/studio/frontend/tests/chat-attachment-display.test.ts b/studio/frontend/tests/chat-attachment-display.test.tsindex 986498cead9..07f47c6a507 100644--- a/studio/frontend/tests/chat-attachment-display.test.ts+++ b/studio/frontend/tests/chat-attachment-display.test.ts@@ -213,6 +213,31 @@ test("composer attachment cards are keyed and scoped by attachment id", async ()assert.match(scopes, /aui\.composer\(\)\.attachment\(\{ id \}\)/);});+test("a custom OpenAI-compatible connection is sent video like an image", async () => {+ const base = {+ checkpoint: "external::conn::vision-model",+ modelLabel: "vision-model",+ activeModel: undefined,+ audio: false,+ video: true,+ };+ assert.equal(+ attachedMediaUnavailableReason({ ...base, externalTakesVideo: true }),+ null,+ );+ assert.match(+ attachedMediaUnavailableReason(base) ?? "",+ /^vision-model cannot accept video\./,+ );+ const adapter = await readSrcAsync("features/chat/api/chat-adapter.ts");+ const start = adapter.indexOf("model: externalSelection.modelId,");+ const external = adapter.slice(+ start,+ adapter.indexOf("...externalReasoningFields,", start),+ );+ assert.match(external, /video_base64: findLatestUserVideoBase64\(currentTurnMessages\)/);+});+test("audio or video attached before a model loaded is checked when sent", () => {const listener = { hasAudioInput: true, hasVideoInput: false };const textOnly = { hasAudioInput: false, hasVideoInput: false };