Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { withBackgroundLoadNotice } from "@/lib/model-lifecycle-events";
import { authFetch } from "@/features/auth";
// Both /download-plan routes share a response model.
import type { DiffusionDownloadPlan } from "@/features/images/api";
import { apiUrl } from "@/lib/api-base";
import { readFastApiError } from "@/lib/format-fastapi-error";
import { parseLoadJson } from "@/lib/multi-model-refusal";
import { withAllowOversizedLoad } from "@/lib/unified-memory-load-refusal";
// `requested` null = left to the backend; `source` is "auto" or "explicit"; `reason` is the tooltip.
export interface VideoResolvedControl {
value: string | boolean | null;
// Absent on backends predating the requested/actual split.
requested?: string | boolean | null;
source: "auto" | "explicit";
// "applied" also covers nothing asked. Absent on older backends.
status?: "applied" | "fell_back" | "unsupported";
reason: string;
// "prequant:<repo>/<file>" when a hosted checkpoint was seeded; absent on a runtime quantise.
artifact?: string | null;
}
export interface VideoGenerationDefaults {
steps: number;
guidance: number;
num_frames: number;
fps: number;
fps_range?: [number, number] | null;
fps_choices?: number[];
// Absent on older backends.
min_num_frames?: number | null;
max_num_frames?: number | null;
// Valid frame counts are k * frame_step + frame_offset.
frame_step: number;
frame_offset: number;
duration_presets: number[];
resolution_multiple: number;
// (width, height) presets the UI offers; the default comes first.
resolution_presets: Array<[number, number]>;
// Bounds a size off the presets must fit; absent on backends that accept only the presets.
custom_size_min_pixels?: number | null;
custom_size_max_pixels?: number | null;
custom_size_max_aspect?: number | null;
// Backend-owned keyframe canvas rule, or null when unsupported.
canvas_short_edge?: number | null;
canvas_max_pixels?: number | null;
// Image-to-video families without that rule take the preset nearest the first frame's aspect.
canvas_from_presets?: boolean;
// Released schedule shifts, or null when unsupported.
flow_shift?: number | null;
audio_flow_shift?: number | null;
supports_audio_flow_shift?: boolean;
}
export interface VideoStatus {
loaded: boolean;
repo_id: string | null;
display_repo_id?: string | null;
resident_mib?: number | null;
family: string | null;
supported_families?: string[];
modular_families?: string[];
base_repo: string | null;
device: string | null;
dtype: string | null;
model_kind?: string | null;
engine?: "diffusers" | "sd_cpp" | null;
// Selected GGUF quant; newer backends report it separately from the compute dtype.
gguf_variant?: string | null;
offload_policy?: string | null;
vae_tiling: boolean;
memory_mode?: string | null;
speed_mode?: string | null;
speed_optims: string[];
attention_backend?: string | null;
transformer_cache?: string | null;
transformer_cache_stats?: {
mode?: string;
every?: number;
planned_skips?: number;
stats?: { calls?: number; computed?: number; skipped?: number };
} | null;
// null for bf16.
transformer_quant?: string | null;
transformer_quant_backend?: string | null;
transformer_quant_backend_reason?: string | null;
// null for dense bf16.
text_encoder_quant?: string | null;
has_audio: boolean;
supports_cfg: boolean;
supports_keyframes?: boolean;
// Per keyframe: every image-to-video family takes a first frame; only MiniMax-H3 a last one.
// Absent on older backends, which read supports_keyframes for both.
supports_first_frame?: boolean;
supports_last_frame?: boolean;
supports_references?: boolean;
supports_video_continuation?: boolean;
h3_task?: string | null;
defaults?: VideoGenerationDefaults | null;
// One or two sentences on how the loaded model reads prompts; null when it has no tip.
prompt_hint?: string | null;
// Per-control provenance keyed by control name, read by the "Auto: X" badges. Null when
// nothing is loaded or the backend does not record it.
// The names are memory_mode, speed_mode, attention_backend and transformer_cache.
resolved?: Record<string, VideoResolvedControl> | null;
}
export interface VideoGenerateProgress {
active: boolean;
// Terminal phases carry the background job's outcome.
phase?: string | null;
step: number;
total: number;
eta_seconds?: number | null;
// Small JPEG data URL of the first frame, plus a counter that moves with each one.
preview?: string | null;
preview_seq?: number;
video?: GalleryVideo | null;
error?: string | null;
}
export interface VideoLoadProgress {
phase:
| "downloading"
| "finalizing"
| "saving_chat_context"
| "ready"
| "error"
| null;
downloaded_bytes: number;
expected_bytes?: number | null;
error?: string | null;
/** Absent on an older backend. */
error_code?: string | null;
}
export interface VideoLoadRequest {
model_path: string;
display_repo_id?: string;
// Required for gguf / single_file; omitted for a from_pretrained pipeline.
gguf_filename?: string;
// Omit to auto-detect from gguf_filename: gguf, single_file (safetensors transformer, any repo)
// or pipeline (restricted to unsloth/ repos or family bases).
model_kind?: "gguf" | "single_file" | "pipeline";
base_repo?: string;
family_override?: string;
hf_token?: string;
// Advanced load-time tuning; omit for the backend's auto defaults.
memory_mode?: "auto" | "fast" | "balanced" | "low_vram";
speed_mode?: "off" | "eager" | "default" | "max";
attention_backend?:
| "auto"
| "native"
| "sdpa"
| "cudnn"
| "flash"
| "flash2"
| "flash3"
| "flash4"
| "sage"
| "xformers"
| "aiter";
transformer_cache?: "off" | "fbcache" | "static";
transformer_cache_threshold?: number;
// Omit for the hardware ladder; "none" pins bf16. GGUF / single-file carry their own.
transformer_quant?: "none" | "fp8" | "int8" | "nvfp4" | "mxfp8";
h3_task?: "fl2va" | "ref2va";
// Omit for automatic. Neither engine shards, so several cards resolve to the one with most free VRAM.
gpu_ids?: number[];
// Text-encoder precision. Omit for the family default (the hosted quantized encoder ComfyUI's template matches);
// "none" keeps the dense bf16 encoder. An explicit scheme is refused with a 409 when the host cannot run it.
text_encoder_quant?: "none" | "fp8" | "fp8_dynamic" | "int8" | "nvfp4";
// Omitted = refuse as usual; withAllowOversizedLoad fills it from the persisted setting.
allow_oversized_load?: boolean;
}
export interface VideoReferenceVideo {
// Base64/data-URL video file, 2 to 15 seconds.
video: string;
// Omitted takes the soundtrack embedded in the file.
audio?: string;
// Both endpoints are required together; duration must be 2 to 15s.
trim_start_seconds?: number;
trim_end_seconds?: number;
}
export interface VideoGenerateRequest {
prompt: string;
// Omitted = the server default (on).
live_preview?: boolean;
negative_prompt?: string;
// Width, height, num_frames and fps default per family; when sent they must match its rules:
// num_frames on the k times frame_step plus 1 lattice, and width and height a preset or on
// resolution_multiple within the custom_size bounds, else 422.
width?: number;
height?: number;
num_frames?: number;
fps?: number;
steps?: number;
guidance?: number;
seed?: number;
// Omit both dimensions to match the source aspect.
first_frame?: string;
last_frame?: string;
// Grouped in the model's image, video, then audio order.
reference_images?: string[];
reference_videos?: VideoReferenceVideo[];
reference_audios?: string[];
// "max" uses Diffusers' 2048px short-edge policy; "match" uses the clip area.
reference_image_size?: "match" | "max";
// Video schedule shift, and the audio one (Diffusers engine only).
flow_shift?: number;
audio_flow_shift?: number;
continuation_video?: string;
continuation_keep?: VideoContinuationKeep;
}
export type VideoContinuationKeep = "last" | "first";
export interface GalleryVideo {
id: string;
url: string;
prompt: string;
negative_prompt?: string | null;
width: number;
height: number;
num_frames: number;
fps: number;
duration_s: number;
steps: number;
guidance: number;
seed: number;
has_audio: boolean;
// Absent on older clips.
conditioning?: string | null;
flow_shift?: number | null;
audio_flow_shift?: number | null;
model?: string | null;
// ENGAGED load-time values, so the recipe names its precision after unload. Absent on older sidecars.
model_kind?: string | null;
gguf_filename?: string | null;
transformer_quant?: string | null;
text_encoder_quant?: string | null;
memory_mode?: string | null;
offload_policy?: string | null;
created_at: string;
// Library state, not recipe; absent on older sidecars.
pinned?: boolean;
archived?: boolean;
/** The server's unpinned sort key: the drag key, else the file mtime. */
order_at?: number | null;
}
// The saved record arrives via getVideoGenerateProgress at phase "completed".
export interface VideoGenerateResponse {
status: "started";
// Always null (kept for response-shape compatibility).
video?: GalleryVideo | null;
}
async function parseJson<T>(response: Response): Promise<T> {
if (!response.ok) {
throw new Error(await readFastApiError(response));
}
return (await response.json()) as T;
}
export async function getVideoStatus(
signal?: AbortSignal,
): Promise<VideoStatus> {
return parseJson(await authFetch("/api/inference/video/status", { signal }));
}
export async function getVideoLoadProgress(
signal?: AbortSignal,
): Promise<VideoLoadProgress> {
return parseJson(
await authFetch("/api/inference/video/load-progress", { signal }),
);
}
export async function getVideoGenerateProgress(): Promise<VideoGenerateProgress> {
return parseJson(await authFetch("/api/inference/video/generate-progress"));
}
export async function loadVideoModel(body: VideoLoadRequest): Promise<VideoStatus> {
// This POST only starts the load; load-progress settles the notice. See images.
return withBackgroundLoadNotice(
"video",
body.model_path,
async () =>
parseLoadJson<VideoStatus>(
await authFetch("/api/inference/video/load", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(withAllowOversizedLoad(body)),
}),
),
async (signal) => (await getVideoLoadProgress(signal)).phase,
);
}
export async function getVideoDownloadPlan(
body: VideoLoadRequest,
): Promise<DiffusionDownloadPlan> {
return parseJson(
await authFetch("/api/inference/video/download-plan", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(withAllowOversizedLoad(body)),
}),
);
}
/** Returns once accepted: a clip takes minutes and the secure tunnel caps responses near 100s. */
export async function generateVideo(
body: VideoGenerateRequest,
): Promise<VideoGenerateResponse> {
return parseJson(
await authFetch("/api/inference/video/generate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
}),
);
}
/** Best-effort: stops at the next step boundary; the caller maps the sentinel to a 409. */
export async function cancelVideoGeneration(): Promise<{ cancelled: boolean }> {
return parseJson(
await authFetch("/api/inference/video/generate/cancel", { method: "POST" }),
);
}
export async function unloadVideoModel(): Promise<VideoStatus> {
return parseJson(await authFetch("/api/inference/video/unload", { method: "POST" }));
}
export interface VideoGalleryPage {
videos: GalleryVideo[];
has_more: boolean;
}
/** `archived`: false is the strip, true is the archive. */
export async function getVideoGallery(
offset = 0,
limit = 50,
archived = false,
): Promise<VideoGalleryPage> {
return parseJson(
await authFetch(
`/api/inference/video/gallery?offset=${offset}&limit=${limit}&archived=${archived}`,
),
);
}
/** Pin/unpin or archive/restore one clip; omitted flags are left alone. Returns the new record. */
/** `afterId` null = front. The server also decides the pin. */
export async function moveGalleryVideo(id: string, afterId: string | null): Promise<GalleryVideo> {
return parseJson(
await authFetch(`/api/inference/video/gallery/${id}/move`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ after_id: afterId }),
}),
);
}
export async function addGalleryVideoToProject(
id: string,
projectId: string,
): Promise<{ path: string; already: boolean }> {
return parseJson(
await authFetch(`/api/inference/video/gallery/${id}/project`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ project_id: projectId }),
}),
);
}
export async function setGalleryVideoFlags(
id: string,
flags: { pinned?: boolean; archived?: boolean },
): Promise<GalleryVideo> {
return parseJson(
await authFetch(`/api/inference/video/gallery/${id}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(flags),
}),
);
}
export async function deleteGalleryVideo(id: string): Promise<void> {
const res = await authFetch(`/api/inference/video/gallery/${id}`, { method: "DELETE" });
if (!res.ok) throw new Error(await readFastApiError(res));
}
export async function clearVideoGallery(): Promise<void> {
const res = await authFetch("/api/inference/video/gallery", { method: "DELETE" });
if (!res.ok) throw new Error(await readFastApiError(res));
}
/** res.blob() would download a whole MP4 before playback and pin it; the file route streams
* ranges but is bearer-gated, so mint a short-lived signed link. */
export async function fetchGalleryVideoSignedUrl(id: string): Promise<string> {
const res = await authFetch(
`/api/inference/video/gallery/${encodeURIComponent(id)}/signed-url`,
);
if (!res.ok) throw new Error(await readFastApiError(res));
const body = (await res.json()) as { url?: string };
if (!body.url) throw new Error("The server returned no video link.");
// Absolute because consumers bypass authFetch and Tauri resolves relative paths to the webview.
return apiUrl(body.url);
}
/** Bearer-gated, so hold the bytes in a revocable object URL. */
export async function fetchGalleryVideoThumbnail(
id: string,
): Promise<{ url: string; bytes: number }> {
const res = await authFetch(
`/v1/videos/${encodeURIComponent(id)}/content?variant=thumbnail`,
);
if (!res.ok) throw new Error(await readFastApiError(res));
const blob = await res.blob();
// An empty 200 would cache a card that never renders and end every retry.
if (blob.size === 0) throw new Error("The thumbnail response was empty.");
return { url: URL.createObjectURL(blob), bytes: blob.size };
}
/** Server-side transcode for the Download menu (WebM / GIF). The backend 501s with a readable
* message when the codec is unavailable. */
export async function fetchGalleryVideoExport(
id: string,
format: "webm" | "gif",
): Promise<Blob> {
const res = await authFetch(
`/api/inference/video/gallery/${id}/export?format=${format}`,
);
if (!res.ok) throw new Error(await readFastApiError(res));
return res.blob();
}
export interface VideoRestyleProgress {
active: boolean;
pending?: boolean;
id?: string;
phase: string | null;
// Latent frames done out of total; Cosmos3 makes one per 4 output frames.
step?: number;
total?: number;
num_frames?: number;
eta_seconds?: number | null;
error?: string | null;
video?: GalleryVideo | null;
seed?: number | null;
}
export interface VideoRestyleStatus {
available: boolean;
reason: string | null;
}
export interface VideoRestylePreview {
frames: number;
seconds: number;
source_seconds: number | null;
width: number;
height: number;
frame_jpeg: string;
edges_jpeg: string;
}
export async function fetchVideoRestyleStatus(): Promise<VideoRestyleStatus> {
return parseJson(await authFetch("/api/inference/video/restyle/status"));
}
function restyleForm(video?: File | null, frames?: File[]): FormData {
const form = new FormData();
if (video) form.append("video", video);
for (const frame of frames ?? []) form.append("frames", frame, frame.name);
return form;
}
/** The frames a restyle would use and one of them with its edge control. Runs no model. */
export async function previewVideoRestyle(input: {
video?: File | null;
frames?: File[];
}): Promise<VideoRestylePreview> {
return parseJson(
await authFetch("/api/inference/video/restyle/preview", {
method: "POST",
body: restyleForm(input.video, input.frames),
}),
);
}
export async function startVideoRestyle(input: {
prompt: string;
seed?: number | null;
video?: File | null;
frames?: File[];
}): Promise<VideoRestyleProgress> {
const form = restyleForm(input.video, input.frames);
form.append("prompt", input.prompt);
if (input.seed != null) form.append("seed", String(input.seed));
return parseJson(
await authFetch("/api/inference/video/restyle", { method: "POST", body: form }),
);
}
export async function fetchVideoRestyleProgress(): Promise<VideoRestyleProgress> {
return parseJson(await authFetch("/api/inference/video/restyle/progress"));
}
export async function cancelVideoRestyle(): Promise<{ cancelled: boolean }> {
return parseJson(
await authFetch("/api/inference/video/restyle/cancel", { method: "POST" }),
);
}
export async function fetchVideoRestyleClip(
id: string,
kind: "input" | "control" | "output",
): Promise<Blob> {
const response = await authFetch(
`/api/inference/video/restyle/${encodeURIComponent(id)}/${kind}.mp4`,
);
if (!response.ok) throw new Error(await readFastApiError(response));
return response.blob();
}