Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: queue /v1/videos while a video renders and batch waiting /v1/images/generations requests (#15613)

oobabooga85744ff385b0
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 1467722540e..5c738266870 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -14405,6 +14405,22 @@ class DiffusionBackend:
# Sole clear of published progress, on every exit.
self._gen = None
+ @contextmanager
+ def generation_waiter(self, cancel: threading.Event):
+ with self._generation_cancel_lock:
+ self._queued_generate_cancels.add(cancel)
+ try:
+ yield
+ finally:
+ with self._generation_cancel_lock:
+ self._queued_generate_cancels.discard(cancel)
+
+ def generation_holder(self) -> Optional[object]:
+ with self._generation_cancel_lock:
+ if self._active_generate_cancel is not None:
+ return self._active_generate_cancel
+ return self._generate_lock if self._generate_lock.locked() else None
+
def generate_progress(self) -> dict[str, Any]:
"""Live per-step progress for an in-flight generation (lock-free read)."""
gen = self._gen
diff --git a/studio/backend/core/inference/generation_ticket.py b/studio/backend/core/inference/generation_ticket.py
index 06b6359e1d3..c734bc06086 100644
--- a/studio/backend/core/inference/generation_ticket.py
+++ b/studio/backend/core/inference/generation_ticket.py
@@ -24,6 +24,10 @@ class GenerationTicket:
if stopped:
event.set()
+ def cancelled(self) -> bool:
+ with self._lock:
+ return self._stopped or any(event.is_set() for event in self._events)
+
def stop(self) -> None:
with self._lock:
self._stopped = True
@@ -32,6 +36,16 @@ class GenerationTicket:
event.set()
+class TicketStopEvent(threading.Event):
+ def __init__(self, ticket: GenerationTicket) -> None:
+ super().__init__()
+ self._ticket = ticket
+
+ def set(self) -> None:
+ super().set()
+ self._ticket.stop()
+
+
_ticket: contextvars.ContextVar[Optional[GenerationTicket]] = contextvars.ContextVar(
"generation_ticket", default = None
)
diff --git a/studio/backend/core/inference/image_request_batching.py b/studio/backend/core/inference/image_request_batching.py
new file mode 100644
index 00000000000..14abba776b8
--- /dev/null
+++ b/studio/backend/core/inference/image_request_batching.py
@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Batches waiting /v1/images/generations requests; pixels vary with batch size (diffusion_batched)."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import secrets
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Hashable, Optional
+
+from loggers import get_logger
+
+from .diffusion_batched import SEED_MASK
+
+logger = get_logger(__name__)
+
+BATCH_MAX_DEFAULT = 4
+POLL_S = 0.05
+
+
+class _RunAlone(Exception):
+ """Retry this request individually."""
+
+
+@dataclass
+class _Member:
+ prompt: str
+ seeds: list[int]
+ future: "asyncio.Future[dict[str, Any]]"
+
+
+@dataclass
+class _Batch:
+ key: Hashable
+ members: list[_Member] = field(default_factory = list)
+ images: int = 0
+
+
+_OPEN: dict[Hashable, _Batch] = {}
+
+
+def batch_max() -> int:
+ try:
+ return max(1, int(os.environ.get("UNSLOTH_STUDIO_IMAGE_BATCH_MAX", BATCH_MAX_DEFAULT)))
+ except ValueError:
+ return BATCH_MAX_DEFAULT
+
+
+def request_seeds(seed: Optional[int], n: int) -> list[int]:
+ base = seed if seed is not None else secrets.randbelow(SEED_MASK + 1)
+ return [(base + i) & SEED_MASK for i in range(n)]
+
+
+def _split(result: dict[str, Any], members: list[_Member]) -> list[dict[str, Any]]:
+ images = list(result.get("images") or [])
+ seeds = list(result.get("seeds") or [])
+ wanted = sum(len(member.seeds) for member in members)
+ if len(images) != wanted or (seeds and len(seeds) != wanted):
+ raise ValueError(f"a batch of {wanted} images returned {len(images)}")
+ parts = []
+ start = 0
+ for member in members:
+ end = start + len(member.seeds)
+ parts.append(
+ {
+ **result,
+ "images": images[start:end],
+ "seeds": seeds[start:end] if seeds else list(member.seeds),
+ "seed": member.seeds[0],
+ }
+ )
+ start = end
+ return parts
+
+
+def _share(exc: BaseException, members: list[_Member]) -> None:
+ for member in members:
+ if not member.future.done():
+ member.future.set_exception(type(exc)(*exc.args))
+
+
+def _close(batch: _Batch) -> None:
+ if _OPEN.get(batch.key) is batch:
+ del _OPEN[batch.key]
+
+
+async def generate(
+ key: Hashable,
+ prompt: str,
+ n: int,
+ seed: Optional[int],
+ *,
+ busy: Callable[[], bool],
+ run_batch: Callable[[list[str], list[int]], Awaitable[dict[str, Any]]],
+ shared_outcome: Callable[[BaseException], bool] = lambda exc: False,
+) -> Optional[dict[str, Any]]:
+ cap = batch_max()
+ # An explicit seed renders alone: a batch's pixels differ slightly from a lone render's.
+ if cap <= 1 or n >= cap or seed is not None:
+ return None
+ loop = asyncio.get_running_loop()
+ batch = _OPEN.get(key)
+ if batch is not None:
+ if batch.images + n > cap:
+ return None
+ member = _Member(prompt, request_seeds(seed, n), loop.create_future())
+ batch.members.append(member)
+ batch.images += n
+ try:
+ return await member.future
+ except _RunAlone:
+ return None
+ if not busy():
+ return None
+
+ batch = _OPEN[key] = _Batch(key)
+ batch.members.append(_Member(prompt, request_seeds(seed, n), loop.create_future()))
+ batch.images = n
+ try:
+ try:
+ while busy():
+ await asyncio.sleep(POLL_S)
+ except Exception as exc:
+ if shared_outcome(exc):
+ _share(exc, batch.members[1:])
+ raise
+ _close(batch)
+ members = list(batch.members)
+ if len(members) == 1:
+ return None
+ prompts = [member.prompt for member in members for _ in member.seeds]
+ seeds = [s for member in members for s in member.seeds]
+ logger.info(
+ "openai_images.batch: %d requests render as one call of %d images",
+ len(members),
+ len(seeds),
+ )
+ try:
+ parts = _split(await run_batch(prompts, seeds), members)
+ except Exception as exc: # noqa: BLE001 -- fall back unless the outcome applies to every request
+ if shared_outcome(exc):
+ _share(exc, members[1:])
+ raise
+ logger.info("openai_images.batch_failed: %s; each request runs alone", exc)
+ return None
+ for member, part in zip(members[1:], parts[1:]):
+ if not member.future.done():
+ member.future.set_result(part)
+ return parts[0]
+ finally:
+ _close(batch)
+ for member in batch.members[1:]:
+ if not member.future.done():
+ member.future.set_exception(_RunAlone())
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 5f918ce8d4c..26db62c35d8 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -11409,13 +11409,7 @@ class VideoBackend:
else None
),
"h3_task": getattr(state, "h3_task", None),
- "defaults": {
- "num_frames": fam.default_num_frames,
- "fps": fam.default_fps,
- "frame_step": fam.frame_step,
- "frame_offset": fam.frame_offset,
- "resolution_presets": [list(p) for p in fam.resolution_presets],
- },
+ "defaults": _family_shape_defaults(fam),
}
return status, state
@@ -11460,8 +11454,10 @@ class VideoBackend:
video_id: Optional[str] = None,
expected_state: Optional[object] = None,
live_preview: Optional[bool] = None,
+ before_commit: Optional[Callable[[], None]] = None,
) -> dict[str, int]:
- """Returns at once (proxy caps ~100 s). VIDEO_NOT_LOADED_MSG and VIDEO_GENERATION_BUSY_MSG map to 409."""
+ """Returns at once (proxy caps ~100 s). VIDEO_NOT_LOADED_MSG and VIDEO_GENERATION_BUSY_MSG map to 409.
+ ``before_commit`` runs under the lock just before the reservation; raising aborts it."""
if input_reference is not None and (first_frame is not None or reference_images):
raise ValueError(
"input_reference cannot be combined with explicit conditioning inputs."
@@ -11530,6 +11526,8 @@ class VideoBackend:
validate_video_request_shape(
state.family, width = width, height = height, num_frames = run_num_frames
)
+ if before_commit is not None:
+ before_commit()
self._generate_job_active = True
self._generate_job_token = job_token
self._generate_job_account = current_account_id()
@@ -13722,6 +13720,16 @@ def get_video_backend() -> VideoBackend:
return _backend
+def _family_shape_defaults(fam) -> dict:
+ return {
+ "num_frames": fam.default_num_frames,
+ "fps": fam.default_fps,
+ "frame_step": fam.frame_step,
+ "frame_offset": fam.frame_offset,
+ "resolution_presets": [list(p) for p in fam.resolution_presets],
+ }
+
+
def generation_in_flight() -> bool:
"""Read the background-job marker without constructing or locking the backend."""
backend = _backend
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index e50a476898d..5f26790b5f5 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -50112,6 +50112,105 @@ def _image_generation_defaults(status: dict) -> tuple[int, float]:
)
+async def _batched_openai_images(
+ backend: Any,
+ status: dict,
+ body: ImageGenerationRequest,
+ width: int,
+ height: int,
+ steps: int,
+ guidance: float,
+ expected_load: Any,
+) -> Optional[dict]:
+ from core.inference import image_request_batching
+ from core.inference.diffusion_engine_router import ENGINE_DIFFUSERS, active_engine_name
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+ from core.inference.diffusion_flux3 import FLUX3_FAMILY_NAMES
+ from core.inference.generation_ticket import (
+ GenerationTicket,
+ TicketStopEvent,
+ generation_ticket,
+ )
+
+ holder = getattr(backend, "generation_holder", None)
+ if (
+ holder is None
+ or active_engine_name() != ENGINE_DIFFUSERS
+ or status.get("family") in FLUX3_FAMILY_NAMES
+ ):
+ return None
+
+ ticket = GenerationTicket()
+ stopped = TicketStopEvent(ticket)
+
+ # Wait only for the generation found on arrival, so a later one taking the slot cannot starve the batch.
+ arrived_behind: list = []
+
+ def busy() -> bool:
+ if stopped.is_set():
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+ try:
+ current = holder()
+ except Exception: # noqa: BLE001 -- no busy signal; render individually
+ return False
+ if current is None:
+ return False
+ if not arrived_behind:
+ arrived_behind.append(current)
+ return current is arrived_behind[0]
+
+ async def run_batch(prompts: list[str], seeds: list[int]) -> dict:
+ with generation_ticket(ticket):
+ try:
+ return await asyncio.to_thread(
+ backend.generate,
+ prompt = prompts[0],
+ negative_prompt = body.negative_prompt,
+ width = width,
+ height = height,
+ steps = steps,
+ guidance = guidance,
+ prompts = prompts,
+ seeds = seeds,
+ batch_size = len(seeds),
+ expected_load = expected_load,
+ )
+ except Exception as exc:
+ # An engine Stop sets the bound event, not ``stopped``: no member may retry alone.
+ if ticket.cancelled() and not cancelled(exc):
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG) from exc
+ raise
+
+ key = (
+ current_account_id(),
+ expected_load,
+ width,
+ height,
+ steps,
+ guidance,
+ body.negative_prompt or None,
+ )
+
+ def cancelled(exc: BaseException) -> bool:
+ return isinstance(exc, RuntimeError) and str(exc) == DIFFUSION_CANCELLED_MSG
+
+ waiter = getattr(backend, "generation_waiter", None)
+ with waiter(stopped) if waiter is not None else nullcontext():
+ result = await image_request_batching.generate(
+ key,
+ body.prompt,
+ body.n,
+ body.seed,
+ busy = busy,
+ run_batch = run_batch,
+ shared_outcome = cancelled,
+ )
+ # A Stop recorded while the batch ran is final: never fall back to rendering alone or retry.
+ if result is None and stopped.is_set():
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+ return result
+
+
# Split out so the monitor row can wrap the whole handler without reindenting it.
async def _generate_openai_images(
body: ImageGenerationRequest,
@@ -50171,23 +50270,28 @@ async def _generate_openai_images(
steps, guidance = await asyncio.to_thread(_image_generation_defaults, status)
steps = steps if body.steps is None else body.steps
guidance = guidance if body.guidance is None else body.guidance
+ expected_load = load_identity(
+ status.get("repo_id"), status.get("base_repo"), status.get("family")
+ )
reset_media_generation_progress("image")
try:
with account_access.media_generation("diffusion"):
- result = await asyncio.to_thread(
- backend.generate,
- prompt = body.prompt,
- negative_prompt = body.negative_prompt,
- width = width,
- height = height,
- steps = steps,
- guidance = guidance,
- seed = body.seed,
- batch_size = body.n,
- expected_load = load_identity(
- status.get("repo_id"), status.get("base_repo"), status.get("family")
- ),
- )
+ result = await _batched_openai_images(
+ backend, status, body, width, height, steps, guidance, expected_load
+ )
+ if result is None:
+ result = await asyncio.to_thread(
+ backend.generate,
+ prompt = body.prompt,
+ negative_prompt = body.negative_prompt,
+ width = width,
+ height = height,
+ steps = steps,
+ guidance = guidance,
+ seed = body.seed,
+ batch_size = body.n,
+ expected_load = expected_load,
+ )
break
except DiffusionModelReplacedError:
if attempt > 0:
@@ -50245,7 +50349,6 @@ async def _generate_openai_images(
"workflow": result.get("workflow"),
"created_at": float(created),
}
- # The diffusers batch shares one seed; the native batch uses a distinct seed per image, so record each image's own seed.
per_image_seeds = result.get("seeds")
def _persist() -> list[ImageGenerationData]:
diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.py
index b9f79b72827..b025223b26b 100644
--- a/studio/backend/routes/video.py
+++ b/studio/backend/routes/video.py
@@ -25,6 +25,7 @@ import secrets as _secrets
import threading
import time as _time
import uuid
+import weakref
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from typing import Annotated, Any, Literal, Optional
@@ -43,7 +44,7 @@ from core.inference.model_ids import public_model_id
from hub.dependencies import get_hf_token
from hub.services.models import account_access
from hub.services.models.account_access import media_link_account, media_link_target
-from utils.account_context import run_as
+from utils.account_context import AccountContext, arun_as, current_account, run_as
from loggers import get_logger
from loggers.media_progress import (
byte_fraction,
@@ -884,6 +885,14 @@ async def cancel_video_generation(
from core.inference.video import get_video_backend
video_id = await _cancel_video_id(request)
+ queued = _own_queued_video(video_id) if video_id is not None else None
+ if queued is not None and queued.starting:
+ if not await _await_queued_start(queued):
+ raise HTTPException(status_code = 409, detail = "The video is starting; retry shortly.")
+ queued = _own_queued_video(video_id)
+ if queued is not None:
+ _drop_queued_video(queued)
+ return {"cancelled": True}
backend = get_video_backend()
# One read: a second could name a successor and hand it back as expected_account.
reserved = _reserved_generation_account(backend)
@@ -1778,6 +1787,9 @@ def _await_generate_settled(video_id: str, timeout: float = _DELETE_SETTLE_TIMEO
def _lookup_video(video_id: str) -> Optional[VideoJob]:
from core.inference import video_gallery
+ queued = _own_queued_video(video_id)
+ if queued is not None:
+ return _queued_video_job(queued)
_hydrate_job(video_id)
_sync_jobs()
with _jobs_lock:
@@ -1809,6 +1821,8 @@ def _all_videos() -> list[VideoJob]:
for job in jobs.values()
if job.id not in seen and job.status != "completed"
)
+ seen.update(video.id for video in listed)
+ listed.extend(video for video in _own_queued_videos() if video.id not in seen)
return listed
@@ -2050,32 +2064,24 @@ async def openai_create_video(
prompt = body.prompt,
subject = current_subject,
) as monitor_id:
- job = await _create_openai_video(body, reference, current_subject, hf_token)
+ inputs = await _validated_video_inputs(body, reference)
+ job = await _create_or_queue_openai_video(
+ body, reference, inputs, current_subject, hf_token
+ )
api_monitor.relabel(monitor_id, job.model)
return job
-# Split out so the API monitor can wrap the handler without reindenting, like the OpenAI image route.
-async def _create_openai_video(
- body: VideoJobCreateRequest,
- reference: Optional[str],
- current_subject: str,
- hf_token: Optional[str],
-):
- from core.inference.gpu_arbiter import VIDEO
- from core.inference.media_auto_switch import (
- maybe_auto_switch_media_model,
- resident_answers_media_request,
- )
- from core.inference.video import get_video_backend
- from core.inference.video_families import (
- VIDEO_GENERATION_BUSY_MSG,
- VIDEO_MODEL_CHANGED_MSG,
- VIDEO_NOT_LOADED_MSG,
- VideoShapeError,
- )
- from utils.openai_auto_switch_settings import get_media_auto_switch_enabled
+@dataclass
+class _VideoCreateInputs:
+ size: Optional[tuple[int, int]]
+ seconds: Optional[float]
+ reference_audios: Optional[list[str]]
+
+async def _validated_video_inputs(
+ body: VideoJobCreateRequest, reference: Optional[str]
+) -> _VideoCreateInputs:
try:
size = _parse_openai_video_size(body.size)
except ValueError as exc:
@@ -2084,7 +2090,6 @@ async def _create_openai_video(
seconds = _parse_openai_video_seconds(body.seconds)
except ValueError as exc:
raise _openai_video_error(400, str(exc), param = "seconds")
- width, height = size if size is not None else (None, None)
if reference is not None and body.reference_images:
raise _openai_video_error(
400,
@@ -2134,57 +2139,140 @@ async def _create_openai_video(
)
except Exception: # noqa: BLE001 -- any decode failure is client input feedback
raise _openai_video_error(400, f"{param} is not a readable image.", param = param)
+ return _VideoCreateInputs(size = size, seconds = seconds, reference_audios = reference_audios)
+
+
+def _new_video_id() -> str:
+ return _VIDEO_JOB_ID_PREFIX + uuid.uuid4().hex
+
+
+def _video_seed(body: VideoJobCreateRequest) -> int:
+ return body.seed if body.seed is not None else _secrets.randbelow(2**53)
+
+
+async def _create_openai_video(
+ body: VideoJobCreateRequest,
+ reference: Optional[str],
+ current_subject: str,
+ hf_token: Optional[str],
+):
+ inputs = await _validated_video_inputs(body, reference)
+ return await _start_openai_video(
+ body,
+ reference,
+ inputs,
+ current_subject,
+ hf_token,
+ video_id = _new_video_id(),
+ seed = _video_seed(body),
+ )
+
+
+def _refuse_family_inputs(
+ fam,
+ h3_task,
+ body: VideoJobCreateRequest,
+ reference: Optional[str],
+ inputs: "_VideoCreateInputs",
+) -> None:
+ from core.inference.video import assert_first_frame_pipeline_available
+ from core.inference.video_families import (
+ validate_video_keyframe_conditioning,
+ validate_video_reference_conditioning,
+ validate_video_request_shape,
+ )
+ from core.inference.video_minimax_h3 import H3_TASK_REFERENCES
+
+ width, height = inputs.size if inputs.size is not None else (None, None)
+ seconds = inputs.seconds
+ _refuse_fixed_fps(fam.name, fam.default_fps, body.fps)
+ # Judge seconds against the target family's lattice now: None deferred the refusal until after the target loaded.
+ want_frames = (
+ body.num_frames
+ if body.num_frames is not None
+ else _frames_for_seconds(
+ seconds,
+ {
+ "fps": body.fps or fam.default_fps,
+ "frame_step": fam.frame_step,
+ "frame_offset": fam.frame_offset,
+ },
+ )
+ if seconds is not None
+ else None
+ )
+ validate_video_request_shape(fam, width, height, want_frames)
+ ref2va = h3_task == H3_TASK_REFERENCES
+ validate_video_keyframe_conditioning(
+ fam,
+ h3_task,
+ has_keyframes = (reference is not None and not ref2va) or bool(body.last_frame),
+ has_last_frame = bool(body.last_frame),
+ )
+ if reference is not None and not ref2va:
+ assert_first_frame_pipeline_available(fam)
+ validate_video_reference_conditioning(
+ fam,
+ h3_task,
+ has_references = (reference is not None and ref2va)
+ or bool(body.reference_images or inputs.reference_audios),
+ )
+
+
+class _QueuedAccountDisabled(Exception):
+ pass
+
+
+def _begin_video_generate(backend, queued_account: Optional[AccountContext], kwargs: dict):
+ if queued_account is None:
+ return backend.begin_generate(**kwargs)
+
+ def _still_active() -> None:
+ if not _queued_account_active(queued_account):
+ raise _QueuedAccountDisabled()
+
+ _still_active()
+ return backend.begin_generate(**kwargs, before_commit = _still_active)
+
+async def _start_openai_video(
+ body: VideoJobCreateRequest,
+ reference: Optional[str],
+ inputs: _VideoCreateInputs,
+ current_subject: str,
+ hf_token: Optional[str],
+ *,
+ video_id: str,
+ seed: int,
+ created_at: Optional[int] = None,
+ queued_account: Optional[AccountContext] = None,
+):
+ from core.inference.gpu_arbiter import VIDEO
+ from core.inference.media_auto_switch import (
+ maybe_auto_switch_media_model,
+ resident_answers_media_request,
+ )
+ from core.inference.video import get_video_backend
+ from core.inference.video_families import (
+ VIDEO_GENERATION_BUSY_MSG,
+ VIDEO_MODEL_CHANGED_MSG,
+ VIDEO_NOT_LOADED_MSG,
+ VideoShapeError,
+ )
+ from utils.openai_auto_switch_settings import get_media_auto_switch_enabled
+
+ size, seconds, reference_audios = inputs.size, inputs.seconds, inputs.reference_audios
+ width, height = size if size is not None else (None, None)
audio_checked = [False]
def _refuse_unservable_request(pick) -> None:
from core.inference.media_model_index import expected_partition
- from core.inference.video import _detect_load_family, assert_first_frame_pipeline_available
- from core.inference.video_families import (
- validate_video_keyframe_conditioning,
- validate_video_reference_conditioning,
- validate_video_request_shape,
- )
- from core.inference.video_minimax_h3 import H3_TASK_REFERENCES
+ from core.inference.video import _detect_load_family
fam = _detect_load_family(pick.model_path, pick.gguf_filename, None)
if fam is None:
return
- _refuse_fixed_fps(fam.name, fam.default_fps, body.fps)
- # Judge the duration against the family being switched TO, using its own lattice. Passing None here accepted any
- # seconds and only refused it in begin_generate, after the resident pipeline had been evicted and the target
- # fully loaded.
- want_frames = (
- body.num_frames
- if body.num_frames is not None
- else _frames_for_seconds(
- seconds,
- {
- "fps": body.fps or fam.default_fps,
- "frame_step": fam.frame_step,
- "frame_offset": fam.frame_offset,
- },
- )
- if seconds is not None
- else None
- )
- validate_video_request_shape(fam, width, height, want_frames)
- h3_task = expected_partition(pick)
- ref2va = h3_task == H3_TASK_REFERENCES
- validate_video_keyframe_conditioning(
- fam,
- h3_task,
- has_keyframes = (reference is not None and not ref2va) or bool(body.last_frame),
- has_last_frame = bool(body.last_frame),
- )
- if reference is not None and not ref2va:
- assert_first_frame_pipeline_available(fam)
- validate_video_reference_conditioning(
- fam,
- h3_task,
- has_references = (reference is not None and ref2va)
- or bool(body.reference_images or reference_audios),
- )
+ _refuse_family_inputs(fam, expected_partition(pick), body, reference, inputs)
# Decoded here, like the images above, so unreadable audio never costs a switch.
_refuse_unreadable_audio(reference_audios)
audio_checked[0] = True
@@ -2212,9 +2300,6 @@ async def _create_openai_video(
await asyncio.to_thread(_refuse_unreadable_audio, reference_audios)
backend = get_video_backend()
- video_id = _VIDEO_JOB_ID_PREFIX + uuid.uuid4().hex
- # Drawn once, outside the retry, so a model_changed retry renders the same take.
- seed = body.seed if body.seed is not None else _secrets.randbelow(2**53)
# Managed callers are authorized against the exact resident state, pinned to the reservation;
# on a mismatch, re-authorize once and retry, as /video/generate does.
# An explicit fps is judged against the family, so it is pinned to the state it was judged on.
@@ -2270,7 +2355,9 @@ async def _create_openai_video(
generate_kwargs.update({k: v for k, v in controls.items() if v is not None})
if expected_state is not None:
generate_kwargs["expected_state"] = expected_state
- resolved = await asyncio.to_thread(backend.begin_generate, **generate_kwargs)
+ resolved = await asyncio.to_thread(
+ _begin_video_generate, backend, queued_account, generate_kwargs
+ )
except VideoShapeError as exc:
raise _openai_video_error(400, str(exc), param = _shape_error_param(exc, body))
except ValueError as exc:
@@ -2311,7 +2398,7 @@ async def _create_openai_video(
seconds_text = body.seconds or "auto"
job = _VideoJob(
id = video_id,
- created_at = int(_time.time()),
+ created_at = created_at if created_at is not None else int(_time.time()),
prompt = body.prompt,
model = public_model_id(
str(reserved.get("model") or status.get("repo_id") or body.model or "video")
@@ -2324,6 +2411,422 @@ async def _create_openai_video(
return _job_to_openai(job)
+VIDEO_QUEUE_POLL_S = 2.0
+VIDEO_QUEUE_MAX = 64
+VIDEO_QUEUE_MAX_PER_ACCOUNT = 32
+VIDEO_QUEUE_IDLE_CONFLICTS = 12
+_VIDEO_WAIT_CODES = ("model_busy", "model_loading", "gpu_busy")
+
+
+@dataclass
+class _QueuedVideo:
+ id: str
+ account: AccountContext
+ subject: str
+ hf_token: Optional[str]
+ body: VideoJobCreateRequest
+ reference: Optional[str]
+ inputs: _VideoCreateInputs
+ seed: int
+ created_at: int
+ model: str
+ size: str
+ seconds: str
+ starting: bool = False
+ idle_conflicts: int = 0
+
+
+_VIDEO_QUEUE: list[_QueuedVideo] = []
+_video_drainer: Optional[asyncio.Task] = None
+_video_start_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = (
+ weakref.WeakKeyDictionary()
+)
+
+
+def _video_start_lock() -> asyncio.Lock:
+ loop = asyncio.get_running_loop()
+ lock = _video_start_locks.get(loop)
+ if lock is None:
+ lock = _video_start_locks[loop] = asyncio.Lock()
+ return lock
+
+
+def _video_engine_busy() -> bool:
+ from core.inference.video import get_video_backend
+
+ backend = get_video_backend()
+ progress = backend.generate_progress() or {}
+ return bool(progress.get("active")) or bool(backend.loading_repo_ids())
+
+
+def _refusal_error(exc: BaseException) -> dict:
+ from core.inference.gpu_arbiter import GpuBusyForAnotherAccountError
+
+ if isinstance(exc, GpuBusyForAnotherAccountError):
+ return {"code": "gpu_busy", "message": str(exc)}
+ if isinstance(exc, HTTPException):
+ detail = exc.detail
+ if isinstance(detail, dict) and isinstance(detail.get("error"), dict):
+ return {
+ "code": detail["error"].get("code"),
+ "message": str(detail["error"].get("message") or "Video generation failed."),
+ }
+ if isinstance(detail, dict):
+ return {"code": detail.get("error"), "message": str(detail.get("message") or "")}
+ return {"code": None, "message": str(detail)}
+ return {"code": None, "message": "Video generation failed."}
+
+
+def _video_wait(exc: BaseException) -> Optional[str]:
+ from core.inference.gpu_arbiter import GpuBusyForAnotherAccountError
+ from core.inference.video_families import VIDEO_GENERATION_BUSY_MSG
+
+ if isinstance(exc, GpuBusyForAnotherAccountError):
+ return "busy"
+ if not isinstance(exc, HTTPException) or exc.status_code not in (409, 503):
+ return None
+ error = _refusal_error(exc)
+ if error["message"] == VIDEO_GENERATION_BUSY_MSG or error["code"] in _VIDEO_WAIT_CODES:
+ return "busy"
+ return "conflict" if error["code"] == "model_changed" else None
+
+
+def _queued_video_job(entry: _QueuedVideo) -> VideoJob:
+ return VideoJob(
+ id = entry.id,
+ model = entry.model,
+ status = "queued",
+ progress = 0,
+ created_at = entry.created_at,
+ prompt = entry.body.prompt,
+ size = entry.size,
+ seconds = entry.seconds,
+ seed = entry.seed,
+ )
+
+
+def _own_queued_video(video_id: str) -> Optional[_QueuedVideo]:
+ account_id = current_account_id()
+ return next(
+ (
+ entry
+ for entry in list(_VIDEO_QUEUE)
+ if entry.id == video_id and entry.account.account_id == account_id
+ ),
+ None,
+ )
+
+
+def _own_queued_videos() -> list[VideoJob]:
+ account_id = current_account_id()
+ return [
+ _queued_video_job(entry)
+ for entry in list(_VIDEO_QUEUE)
+ if entry.account.account_id == account_id
+ ]
+
+
+def _keyframe_canvas(
+ fam,
+ h3_task,
+ reference: Optional[str],
+ last_frame: Optional[str],
+ size = None,
+):
+ from core.inference.video import VideoBackend
+ from core.inference.video_minimax_h3 import H3_TASK_REFERENCES
+
+ first = None if h3_task == H3_TASK_REFERENCES else reference
+ width, height = size if size is not None else (None, None)
+ return VideoBackend._resolve_keyframes(fam, h3_task, first, last_frame, width, height)
+
+
+def _queued_job_display(
+ body: VideoJobCreateRequest, reference: Optional[str], inputs: _VideoCreateInputs
+) -> tuple[str, str, str]:
+ from core.inference.gpu_arbiter import VIDEO
+ from core.inference.media_auto_switch import resident_answers_media_request
+ from core.inference.media_model_index import (
+ VIDEO_TASK,
+ expected_partition,
+ resolve_local_media_model,
+ )
+ from core.inference.video import (
+ _detect_load_family,
+ _family_shape_defaults,
+ get_video_backend,
+ )
+ from core.inference.video_families import VideoShapeError
+ from utils.openai_auto_switch_settings import get_media_auto_switch_enabled
+
+ backend = get_video_backend()
+ status = backend.status()
+ named = isinstance(body.model, str) and bool(body.model.strip())
+ resident_renders = (
+ not named
+ or not get_media_auto_switch_enabled()
+ or resident_answers_media_request(status, body.model, owner = VIDEO)
+ )
+ if resident_renders and account_access.managed_account():
+ account_access.require_media_generation_access(status, "video")
+ if resident_renders:
+ state = backend.generation_snapshot()[1]
+ fam, h3_task = getattr(state, "family", None), getattr(state, "h3_task", None)
+ else:
+ pick = resolve_local_media_model(body.model.strip(), task = VIDEO_TASK)
+ fam = pick and _detect_load_family(pick.model_path, pick.gguf_filename, None)
+ h3_task = expected_partition(pick) if fam else None
+ canvas = None
+ if fam is not None:
+ try:
+ _refuse_family_inputs(fam, h3_task, body, reference, inputs)
+ if reference is not None or body.last_frame:
+ canvas = _keyframe_canvas(fam, h3_task, reference, body.last_frame, inputs.size)
+ except VideoShapeError as exc:
+ raise _openai_video_error(400, str(exc), param = _shape_error_param(exc, body))
+ except ValueError as exc:
+ raise _openai_video_error(
+ 400, str(exc), param = _conditioning_param(body, reference, "input_reference")
+ )
+ if resident_renders:
+ defaults = status.get("defaults") or {}
+ else:
+ defaults = _family_shape_defaults(fam) if fam is not None else {}
+ resident = status.get("repo_id") if resident_renders else None
+ if resident:
+ model = public_model_id(str(resident))
+ elif named:
+ model = public_model_id(body.model) or body.model
+ else:
+ model = "video"
+ presets = defaults.get("resolution_presets") or []
+ if inputs.size is not None:
+ size = f"{inputs.size[0]}x{inputs.size[1]}"
+ elif canvas is not None:
+ size = f"{canvas[2]}x{canvas[3]}"
+ elif presets:
+ size = f"{int(presets[0][0])}x{int(presets[0][1])}"
+ else:
+ size = "auto"
+ fps = body.fps or defaults.get("fps")
+ if inputs.seconds is not None:
+ frames = _frames_for_seconds(inputs.seconds, {**defaults, "fps": fps})
+ else:
+ frames = body.num_frames or defaults.get("num_frames")
+ seconds = (
+ _format_seconds(int(frames) / float(fps)) if frames and fps else body.seconds or "auto"
+ )
+ return model, size, seconds
+
+
+async def _create_or_queue_openai_video(
+ body: VideoJobCreateRequest,
+ reference: Optional[str],
+ inputs: _VideoCreateInputs,
+ current_subject: str,
+ hf_token: Optional[str],
+) -> VideoJob:
+ video_id = _new_video_id()
+ seed = _video_seed(body)
+ engine_busy = await asyncio.to_thread(_video_engine_busy)
+ if not _VIDEO_QUEUE and not engine_busy:
+ try:
+ return await _start_openai_video(
+ body,
+ reference,
+ inputs,
+ current_subject,
+ hf_token,
+ video_id = video_id,
+ seed = seed,
+ )
+ except Exception as exc:
+ if _video_wait(exc) != "busy":
+ raise
+ await asyncio.to_thread(_refuse_unreadable_audio, inputs.reference_audios)
+ model, size, seconds = await asyncio.to_thread(_queued_job_display, body, reference, inputs)
+ account = current_account()
+ waiting = sum(1 for entry in _VIDEO_QUEUE if entry.account.account_id == account.account_id)
+ if waiting >= VIDEO_QUEUE_MAX_PER_ACCOUNT or len(_VIDEO_QUEUE) >= VIDEO_QUEUE_MAX:
+ full = (
+ f"{VIDEO_QUEUE_MAX_PER_ACCOUNT} videos are already waiting for this account"
+ if waiting >= VIDEO_QUEUE_MAX_PER_ACCOUNT
+ else f"{VIDEO_QUEUE_MAX} videos are already waiting"
+ )
+ raise _openai_video_error(
+ 429,
+ f"{full} in Unsloth Studio; create this one once some have started.",
+ code = "rate_limit_exceeded",
+ )
+ entry = _QueuedVideo(
+ id = video_id,
+ account = account,
+ subject = current_subject,
+ hf_token = hf_token,
+ body = body,
+ reference = reference,
+ inputs = inputs,
+ seed = seed,
+ created_at = int(_time.time()),
+ model = model,
+ size = size,
+ seconds = seconds,
+ )
+ _VIDEO_QUEUE.append(entry)
+ _ensure_video_drainer()
+ return _queued_video_job(entry)
+
+
+def _fail_queued_video(entry: _QueuedVideo, exc: BaseException) -> None:
+ error = _refusal_error(exc)
+ job = _VideoJob(
+ id = entry.id,
+ created_at = entry.created_at,
+ prompt = entry.body.prompt,
+ model = entry.model,
+ size = entry.size,
+ seconds = entry.seconds,
+ seed = entry.seed,
+ status = "failed",
+ completed_at = int(_time.time()),
+ error = {
+ "code": error["code"] or _VIDEO_FAILED_CODE,
+ "message": error["message"] or "Video generation failed.",
+ },
+ )
+ run_as(entry.account, _remember_job, job)
+
+
+def _drop_queued_video(entry: _QueuedVideo) -> None:
+ if entry in _VIDEO_QUEUE:
+ _VIDEO_QUEUE.remove(entry)
+
+
+async def _settle_failed_queued_video(entry: _QueuedVideo, exc: BaseException) -> None:
+ entry.starting = True
+ try:
+ await asyncio.to_thread(_fail_queued_video, entry, exc)
+ except Exception: # noqa: BLE001 -- persistence failure must not block the queue
+ logger.exception("Could not record queued video %s as failed", entry.id)
+ finally:
+ _drop_queued_video(entry)
+ entry.starting = False
+
+
+async def _await_queued_start(entry: _QueuedVideo) -> bool:
+ deadline = _time.monotonic() + _DELETE_SETTLE_TIMEOUT_S
+ while entry.starting and _time.monotonic() < deadline:
+ await asyncio.sleep(0.05)
+ return not entry.starting
+
+
+def _queued_account_active(account: AccountContext) -> bool:
+ if account.is_owner:
+ return True
+ from auth.storage import get_account_by_id
+ return get_account_by_id(account.account_id) is not None
+
+
+async def _start_queued_video(entry: _QueuedVideo) -> bool:
+ from core.inference import media_keepwarm
+ from core.inference.gpu_arbiter import VIDEO
+ from core.inference.llama_keepwarm import InferenceActivityReservation
+
+ async with _video_start_lock():
+ if entry not in _VIDEO_QUEUE:
+ return True
+ if await asyncio.to_thread(_video_engine_busy):
+ return False
+ active = await asyncio.to_thread(_queued_account_active, entry.account)
+ if entry not in _VIDEO_QUEUE:
+ return True
+ if not active:
+ logger.info("openai_videos.queued_account_disabled: %s", entry.id)
+ _drop_queued_video(entry)
+ return True
+ entry.starting = True
+ entered = started = False
+ # Counted like an HTTP create so an idle unload or auto-switch cannot cancel the accepted clip.
+ activity = InferenceActivityReservation()
+ activity.reserve()
+ try:
+ await activity.start()
+ await media_keepwarm.begin_request(VIDEO)
+ entered = True
+ if await asyncio.to_thread(_video_engine_busy):
+ entry.starting = False
+ return False
+ await arun_as(
+ entry.account,
+ _start_openai_video(
+ entry.body,
+ entry.reference,
+ entry.inputs,
+ entry.subject,
+ entry.hf_token,
+ video_id = entry.id,
+ seed = entry.seed,
+ created_at = entry.created_at,
+ queued_account = entry.account,
+ ),
+ )
+ started = True
+ except _QueuedAccountDisabled:
+ logger.info("openai_videos.queued_account_disabled: %s", entry.id)
+ _drop_queued_video(entry)
+ entry.starting = False
+ return True
+ except Exception as exc: # noqa: BLE001 -- settle failures and continue draining
+ wait = _video_wait(exc)
+ if wait is None and await asyncio.to_thread(_video_engine_busy):
+ wait = "busy"
+ if wait == "conflict":
+ entry.idle_conflicts += 1
+ if entry.idle_conflicts >= VIDEO_QUEUE_IDLE_CONFLICTS:
+ wait = None
+ if wait is not None:
+ entry.starting = False
+ return False
+ if not isinstance(exc, HTTPException):
+ logger.exception("Could not start queued video %s", entry.id)
+ await _settle_failed_queued_video(entry, exc)
+ return True
+ finally:
+ if entered:
+ media_keepwarm.end_request(VIDEO, counted = started)
+ activity.finish()
+ _drop_queued_video(entry)
+ entry.starting = False
+ return True
+
+
+async def _drain_video_queue_once() -> None:
+ if not _VIDEO_QUEUE:
+ return
+ entry = _VIDEO_QUEUE[0]
+ try:
+ await _start_queued_video(entry)
+ except Exception as exc: # noqa: BLE001 -- the next video must still start
+ logger.exception("Could not settle queued video %s", entry.id)
+ if entry in _VIDEO_QUEUE:
+ await _settle_failed_queued_video(entry, exc)
+ entry.starting = False
+
+
+async def _drain_video_queue() -> None:
+ while _VIDEO_QUEUE:
+ head = _VIDEO_QUEUE[0]
+ await _drain_video_queue_once()
+ if _VIDEO_QUEUE and _VIDEO_QUEUE[0] is head:
+ await asyncio.sleep(VIDEO_QUEUE_POLL_S)
+
+
+def _ensure_video_drainer() -> None:
+ global _video_drainer
+ if _video_drainer is None or _video_drainer.done():
+ _video_drainer = asyncio.get_running_loop().create_task(_drain_video_queue())
+
+
@openai_router.get("/videos", response_model = VideoJobListResponse)
async def openai_list_videos(
limit: int = Query(20, ge = 1, le = 100),
@@ -2422,6 +2925,18 @@ async def openai_delete_video(video_id: str, current_subject: str = Depends(get_
from core.inference import video_gallery
from core.inference.video import get_video_backend
+ queued = _own_queued_video(video_id)
+ if queued is not None and queued.starting:
+ if not await _await_queued_start(queued):
+ raise _openai_video_error(
+ 409,
+ "The video is starting; retry the delete shortly.",
+ code = "video_not_ready",
+ )
+ queued = _own_queued_video(video_id)
+ if queued is not None:
+ _drop_queued_video(queued)
+ return VideoJobDeleteResponse(id = video_id)
video = await asyncio.to_thread(_lookup_video, video_id)
if video is None:
raise _not_found(video_id)
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 22fddb398fd..a6cb256a360 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -181,6 +181,21 @@ def _forget_mcp_discovery_state():
_forget()
[email protected](autouse = True)
+def _forget_queued_media_requests():
+ def _forget():
+ video_routes = sys.modules.get("routes.video")
+ if video_routes is not None:
+ video_routes._VIDEO_QUEUE.clear()
+ batching = sys.modules.get("core.inference.image_request_batching")
+ if batching is not None:
+ batching._OPEN.clear()
+
+ _forget()
+ yield
+ _forget()
+
+
@pytest.fixture(autouse = True)
def _forget_load_budget_snapshots():
def _forget():
diff --git a/studio/backend/tests/test_account_openai_video_pin.py b/studio/backend/tests/test_account_openai_video_pin.py
index 7c0ebea4945..3219172a95d 100644
--- a/studio/backend/tests/test_account_openai_video_pin.py
+++ b/studio/backend/tests/test_account_openai_video_pin.py
@@ -114,6 +114,7 @@ def test_openai_video_resident_replaced_after_authorization(monkeypatch):
generation_snapshot = generation_snapshot,
begin_generate = begin_generate,
generate_progress = lambda: {"active": False},
+ loading_repo_ids = lambda: (),
)
monkeypatch.setattr(video_module, "get_video_backend", lambda: backend)
body, content_type = _multipart({"prompt": "a sloth", "size": "320x320"})
diff --git a/studio/backend/tests/test_account_video_generation_owner.py b/studio/backend/tests/test_account_video_generation_owner.py
index c966286b2fa..ebbd0be8897 100644
--- a/studio/backend/tests/test_account_video_generation_owner.py
+++ b/studio/backend/tests/test_account_video_generation_owner.py
@@ -58,6 +58,9 @@ class FakeVideoBackend:
def generate_progress(self):
return {"active": False, "phase": "completed", "step": 5, "total": 5, "video": CLIP}
+ def loading_repo_ids(self):
+ return ()
+
def cancel_generate(self, *args, **kwargs):
from utils.account_context import current_account
self.cancelled.append(current_account().username)
diff --git a/studio/backend/tests/test_openai_images_batching.py b/studio/backend/tests/test_openai_images_batching.py
new file mode 100644
index 00000000000..f6603d55a15
--- /dev/null
+++ b/studio/backend/tests/test_openai_images_batching.py
@@ -0,0 +1,491 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+
+import httpx
+import pytest
+from fastapi import FastAPI
+
+import core.inference.diffusion as diffusion_module
+import core.inference.diffusion_engine_router as engine_router
+import core.inference.image_gallery as gallery_module
+from auth.authentication import get_current_subject
+from core.inference import image_request_batching as batching
+from core.inference.diffusion_batched import SEED_MASK
+from routes.inference import router
+from utils.api_errors import install_api_error_handlers
+
+
+class _Backend:
+ def __init__(self) -> None:
+ self.rendering = False
+ self.fail_lists = False
+ self.cancel_lists = False
+ self.calls: list[dict] = []
+ self.waiters: list = []
+ self.before_generate = None
+ self.is_loaded = True
+ self.holder = object()
+ self.active_cancel = None
+
+ def status(self) -> dict:
+ return {
+ "loaded": True,
+ "repo_id": "unsloth/Z-Image-Turbo-GGUF",
+ "family": "z-image",
+ "base_repo": None,
+ }
+
+ def generation_holder(self):
+ return self.holder if self.rendering else None
+
+ @contextlib.contextmanager
+ def generation_waiter(self, cancel):
+ self.waiters.append(cancel)
+ try:
+ yield
+ finally:
+ self.waiters.remove(cancel)
+
+ def stop_during_a_model_change(self) -> None:
+ for cancel in list(self.waiters):
+ cancel.set()
+
+ def generate(
+ self,
+ *,
+ prompt,
+ width,
+ height,
+ steps,
+ guidance,
+ batch_size = 1,
+ expected_load = None,
+ seed = None,
+ negative_prompt = None,
+ prompts = None,
+ seeds = None,
+ ) -> dict:
+ if self.before_generate is not None:
+ self.before_generate()
+ import threading
+
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+ from core.inference.generation_ticket import bind_generation_cancel
+
+ cancel = self.active_cancel = threading.Event()
+ bind_generation_cancel(cancel)
+ if cancel.is_set():
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+ self.calls.append(
+ dict(prompt = prompt, prompts = prompts, seeds = seeds, seed = seed, batch_size = batch_size)
+ )
+ if prompts is not None and self.fail_lists:
+ raise RuntimeError("CUDA out of memory")
+ if prompts is not None and self.cancel_lists:
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+ if seeds is not None:
+ base, image_seeds = seeds[0], list(seeds)
+ else:
+ base = 4242 if seed is None else seed
+ image_seeds = [base + i for i in range(batch_size)]
+ labels = prompts if prompts is not None else [prompt] * len(image_seeds)
+ return {
+ "images": [f"{label}#{s}" for label, s in zip(labels, image_seeds)],
+ "seed": base,
+ "seeds": image_seeds,
+ "repo_id": "unsloth/Z-Image-Turbo-GGUF",
+ "negative_prompt": negative_prompt,
+ }
+
+
+def studio(monkeypatch):
+ backend = _Backend()
+ saved: list[dict] = []
+
+ def _save(image, meta):
+ record = {**meta, "id": f"img{len(saved)}", "image": image}
+ saved.append(record)
+ return record
+
+ monkeypatch.setattr(diffusion_module, "get_diffusion_backend", lambda: backend)
+ monkeypatch.setattr(gallery_module, "save", _save)
+ monkeypatch.setattr(gallery_module, "image_b64", lambda image_id: "QUJD")
+ app = FastAPI()
+ install_api_error_handlers(app)
+ app.include_router(router, prefix = "/v1")
+ app.dependency_overrides[get_current_subject] = lambda: "test-user"
+ return app, backend, saved
+
+
+async def _until(predicate, timeout = 5.0) -> None:
+ deadline = asyncio.get_running_loop().time() + timeout
+ while not predicate():
+ if asyncio.get_running_loop().time() > deadline:
+ raise AssertionError("condition never held")
+ await asyncio.sleep(0.01)
+
+
+def _waiting() -> int:
+ return sum(len(batch.members) for batch in batching._OPEN.values())
+
+
+def _post_while_rendering(app, backend, bodies: list[dict], waiting: int) -> list[httpx.Response]:
+ async def run():
+ transport = httpx.ASGITransport(app = app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://studio") as client:
+ backend.rendering = True
+ sent = [
+ asyncio.ensure_future(client.post("/v1/images/generations", json = body))
+ for body in bodies
+ ]
+ await _until(lambda: _waiting() >= waiting)
+ backend.rendering = False
+ return await asyncio.gather(*sent)
+
+ return asyncio.run(run())
+
+
+def _b64(body: dict) -> dict:
+ return {"size": "256x256", "response_format": "b64_json", **body}
+
+
+def test_waiting_compatible_requests_render_as_one_call_and_each_gets_its_own_images(studio):
+ app, backend, saved = studio
+ bodies = [
+ _b64({"prompt": "a fox", "n": 2}),
+ _b64({"prompt": "a cat"}),
+ _b64({"prompt": "an owl"}),
+ ]
+ responses = _post_while_rendering(app, backend, bodies, waiting = 3)
+ assert [len(r.json()["data"]) for r in responses] == [2, 1, 1]
+ [call] = backend.calls
+ assert call["batch_size"] == 4 and call["seed"] is None
+ assert call["prompt"] == call["prompts"][0]
+ assert sorted(call["prompts"]) == ["a cat", "a fox", "a fox", "an owl"]
+ seeds = {}
+ for prompt, seed in zip(call["prompts"], call["seeds"]):
+ seeds.setdefault(prompt, []).append(seed)
+ assert seeds["a fox"][1] == (seeds["a fox"][0] + 1) & SEED_MASK
+ by_prompt = {}
+ for record in saved:
+ by_prompt.setdefault(record["prompt"], []).append(
+ (
+ record["image"],
+ record["seed"],
+ record["batch_seed"],
+ record["batch_size"],
+ record["batch_index"],
+ )
+ )
+ fox, cat, owl = seeds["a fox"], seeds["a cat"][0], seeds["an owl"][0]
+ assert by_prompt == {
+ "a fox": [
+ (f"a fox#{fox[0]}", fox[0], fox[0], 2, 0),
+ (f"a fox#{fox[1]}", fox[1], fox[0], 2, 1),
+ ],
+ "a cat": [(f"a cat#{cat}", cat, cat, 1, 0)],
+ "an owl": [(f"an owl#{owl}", owl, owl, 1, 0)],
+ }
+
+
+def test_a_request_with_a_seed_never_joins_a_batch(studio):
+ app, backend, _saved = studio
+
+ async def run():
+ transport = httpx.ASGITransport(app = app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://studio") as client:
+ backend.rendering = True
+ sent = [
+ asyncio.ensure_future(client.post("/v1/images/generations", json = _b64(body)))
+ for body in ({"prompt": "a fox", "seed": 5}, {"prompt": "a cat", "seed": 9})
+ ]
+ await asyncio.sleep(0.2)
+ assert _waiting() == 0
+ backend.rendering = False
+ return await asyncio.gather(*sent)
+
+ assert all(r.status_code == 200 for r in asyncio.run(run()))
+ assert [call["prompts"] for call in backend.calls] == [None, None]
+ assert sorted(call["seed"] for call in backend.calls) == [5, 9]
+
+
+def test_requests_that_differ_in_more_than_prompt_seed_and_n_render_apart(studio):
+ app, backend, _saved = studio
+ bodies = [_b64({"prompt": "a fox"}), _b64({"prompt": "a cat", "size": "512x512"})]
+ responses = _post_while_rendering(app, backend, bodies, waiting = 2)
+ assert all(r.status_code == 200 for r in responses)
+ assert len(backend.calls) == 2
+ assert all(call["prompts"] is None for call in backend.calls)
+
+
+def test_a_failed_shared_call_sends_every_request_back_to_render_alone(studio):
+ app, backend, _saved = studio
+ backend.fail_lists = True
+ bodies = [_b64({"prompt": "a fox"}), _b64({"prompt": "a cat"})]
+ responses = _post_while_rendering(app, backend, bodies, waiting = 2)
+ assert [r.status_code for r in responses] == [200, 200]
+ assert sorted(backend.calls[0]["prompts"]) == ["a cat", "a fox"]
+ assert sorted(call["prompt"] for call in backend.calls[1:]) == ["a cat", "a fox"]
+ assert [call["prompts"] for call in backend.calls[1:]] == [None, None]
+
+
+def test_a_cancelled_batch_ends_every_request_without_rendering_again(studio):
+ app, backend, saved = studio
+ backend.cancel_lists = True
+ bodies = [_b64({"prompt": "a fox"}), _b64({"prompt": "a cat"})]
+ responses = _post_while_rendering(app, backend, bodies, waiting = 2)
+ assert len(backend.calls) == 1 and backend.calls[0]["prompts"] is not None
+ assert saved == []
+ assert all(r.status_code >= 400 for r in responses)
+ assert responses[0].status_code == responses[1].status_code
+
+
+def test_a_stop_while_a_batch_waits_cancels_every_request_in_it(studio):
+ app, backend, saved = studio
+
+ async def run():
+ transport = httpx.ASGITransport(app = app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://studio") as client:
+ backend.rendering = True
+ sent = [
+ asyncio.ensure_future(
+ client.post("/v1/images/generations", json = _b64({"prompt": p}))
+ )
+ for p in ("a fox", "a cat")
+ ]
+ await _until(lambda: _waiting() >= 2 and backend.waiters)
+ backend.stop_during_a_model_change()
+ return await asyncio.gather(*sent)
+
+ responses = asyncio.run(run())
+ assert backend.calls == [] and saved == []
+ assert all(r.status_code >= 400 for r in responses)
+ assert responses[0].status_code == responses[1].status_code
+ assert backend.waiters == []
+
+
+def test_a_stop_after_the_batch_stops_waiting_still_cancels_its_generation(studio):
+ app, backend, saved = studio
+ backend.before_generate = backend.stop_during_a_model_change
+ bodies = [_b64({"prompt": "a fox"}), _b64({"prompt": "a cat"})]
+ responses = _post_while_rendering(app, backend, bodies, waiting = 2)
+ assert backend.calls == [] and saved == []
+ assert all(r.status_code >= 400 for r in responses)
+ assert responses[0].status_code == responses[1].status_code
+
+
+def test_a_stop_during_a_real_model_change_reaches_a_generation_that_starts_after_it():
+ from core.inference.diffusion import DiffusionBackend
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+ from core.inference.generation_ticket import (
+ GenerationTicket,
+ TicketStopEvent,
+ generation_ticket,
+ )
+
+ engine = DiffusionBackend()
+ ticket = GenerationTicket()
+ stopped = TicketStopEvent(ticket)
+ with engine.generation_waiter(stopped):
+ with engine._model_transition_slot():
+ assert engine.cancel_generate() is True
+ assert stopped.is_set()
+ with generation_ticket(ticket):
+ with pytest.raises(RuntimeError, match = DIFFUSION_CANCELLED_MSG):
+ engine.generate(prompt = "a fox", width = 256, height = 256)
+ with generation_ticket(GenerationTicket()):
+ with pytest.raises(RuntimeError) as exc:
+ engine.generate(prompt = "a fox", width = 256, height = 256)
+ assert str(exc.value) != DIFFUSION_CANCELLED_MSG
+
+
+def test_a_stop_while_a_failed_batch_unwinds_keeps_its_requests_from_rendering_alone(studio):
+ app, backend, saved = studio
+ backend.fail_lists = True
+ real_generate = backend.generate
+
+ def _fail_then_stop(**kwargs):
+ try:
+ return real_generate(**kwargs)
+ finally:
+ if kwargs.get("prompts") is not None:
+ backend.stop_during_a_model_change()
+
+ backend.generate = _fail_then_stop
+ bodies = [_b64({"prompt": "a fox"}), _b64({"prompt": "a cat"})]
+ responses = _post_while_rendering(app, backend, bodies, waiting = 2)
+ assert [call["prompts"] is not None for call in backend.calls] == [True]
+ assert saved == []
+ assert all(r.status_code >= 400 for r in responses)
+ assert responses[0].status_code == responses[1].status_code
+
+
+def test_an_engine_stop_acknowledged_as_a_shared_call_fails_keeps_every_request_from_retrying(
+ studio,
+):
+ app, backend, saved = studio
+ backend.fail_lists = True
+ real_generate = backend.generate
+
+ def _fail_as_the_engine_cancels(**kwargs):
+ try:
+ return real_generate(**kwargs)
+ finally:
+ if kwargs.get("prompts") is not None:
+ backend.active_cancel.set()
+
+ backend.generate = _fail_as_the_engine_cancels
+ bodies = [_b64({"prompt": "a fox"}), _b64({"prompt": "a cat"})]
+ responses = _post_while_rendering(app, backend, bodies, waiting = 2)
+ assert [call["prompts"] is not None for call in backend.calls] == [True]
+ assert saved == []
+ assert all(r.status_code >= 400 for r in responses)
+ assert responses[0].json() == responses[1].json()
+
+
+def test_a_batch_stops_waiting_when_the_generation_it_queued_behind_ends(studio):
+ app, backend, saved = studio
+
+ async def run():
+ transport = httpx.ASGITransport(app = app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://studio") as client:
+ backend.rendering = True
+ sent = [
+ asyncio.ensure_future(
+ client.post("/v1/images/generations", json = _b64({"prompt": p}))
+ )
+ for p in ("a fox", "a cat")
+ ]
+ await _until(lambda: _waiting() >= 2)
+ backend.holder = object()
+ await _until(lambda: backend.calls, timeout = 2.0)
+ backend.rendering = False
+ return await asyncio.gather(*sent)
+
+ responses = asyncio.run(run())
+ assert all(r.status_code == 200 for r in responses)
+ [call] = backend.calls
+ assert sorted(call["prompts"]) == ["a cat", "a fox"]
+ assert len(saved) == 2
+
+
+def test_a_request_that_finds_the_engine_idle_renders_at_once_alone(studio):
+ app, backend, _saved = studio
+
+ async def run():
+ transport = httpx.ASGITransport(app = app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://studio") as client:
+ return await client.post(
+ "/v1/images/generations", json = _b64({"prompt": "a fox", "n": 2})
+ )
+
+ assert asyncio.run(run()).status_code == 200
+ assert backend.calls == [
+ dict(prompt = "a fox", prompts = None, seeds = None, seed = None, batch_size = 2)
+ ]
+
+
[email protected]("engine", ["sd_cpp", "mlx"])
+def test_engines_without_prompt_lists_never_batch(studio, monkeypatch, engine):
+ app, backend, _saved = studio
+ monkeypatch.setattr(engine_router, "active_engine_name", lambda: engine)
+
+ async def run():
+ transport = httpx.ASGITransport(app = app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://studio") as client:
+ backend.rendering = True
+ sent = [
+ asyncio.ensure_future(
+ client.post("/v1/images/generations", json = _b64({"prompt": p}))
+ )
+ for p in ("a fox", "a cat")
+ ]
+ await asyncio.sleep(0.2)
+ assert _waiting() == 0
+ backend.rendering = False
+ return await asyncio.gather(*sent)
+
+ assert all(r.status_code == 200 for r in asyncio.run(run()))
+ assert [call["prompts"] for call in backend.calls] == [None, None]
+
+
+def test_request_seeds_derive_like_a_lone_request():
+ assert batching.request_seeds(5, 3) == [5, 6, 7]
+ assert batching.request_seeds(SEED_MASK, 2) == [SEED_MASK, 0]
+ drawn = batching.request_seeds(None, 2)
+ assert drawn[1] == (drawn[0] + 1) & SEED_MASK
+
+
+def test_a_batch_stops_collecting_at_the_cap_and_the_rest_render_alone(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_IMAGE_BATCH_MAX", "4")
+ rendering = {"on": True}
+ batches: list[list[str]] = []
+
+ async def run_batch(prompts, seeds):
+ batches.append(prompts)
+ return {"images": list(prompts), "seeds": list(seeds), "seed": seeds[0]}
+
+ async def run():
+ def request(prompt, n):
+ return asyncio.ensure_future(
+ batching.generate(
+ "key", prompt, n, None, busy = lambda: rendering["on"], run_batch = run_batch
+ )
+ )
+
+ lead, joined, overflow, too_big = (
+ request("a", 2),
+ request("b", 2),
+ request("c", 1),
+ request("d", 4),
+ )
+ await _until(lambda: overflow.done() and too_big.done())
+ assert overflow.result() is None and too_big.result() is None
+ rendering["on"] = False
+ return await lead, await joined
+
+ lead, joined = asyncio.run(run())
+ assert batches == [["a", "a", "b", "b"]]
+ assert lead["images"] == ["a", "a"] and joined["images"] == ["b", "b"]
+
+
+def test_a_cap_of_one_turns_batching_off(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_IMAGE_BATCH_MAX", "1")
+
+ async def run():
+ return await batching.generate(
+ "key",
+ "a",
+ 1,
+ None,
+ busy = lambda: True,
+ run_batch = None, # type: ignore[arg-type]
+ )
+
+ assert asyncio.run(run()) is None
+ assert batching._OPEN == {}
+
+
+def test_the_diffusers_engine_names_each_slot_holder_apart():
+ import threading
+
+ from core.inference.diffusion import DiffusionBackend
+
+ engine = DiffusionBackend()
+ assert engine.generation_holder() is None
+ first, second = threading.Event(), threading.Event()
+ with engine._generation_slot(first):
+ assert engine.generation_holder() is first
+ with engine._model_transition_slot():
+ assert engine.generation_holder() not in (None, first)
+ with engine._generation_slot(second):
+ assert engine.generation_holder() is second
+ assert engine.generation_holder() is None
diff --git a/studio/backend/tests/test_openai_videos_queue.py b/studio/backend/tests/test_openai_videos_queue.py
new file mode 100644
index 00000000000..7758c10735b
--- /dev/null
+++ b/studio/backend/tests/test_openai_videos_queue.py
@@ -0,0 +1,724 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+
+import core.inference.video_gallery as gallery_module
+import routes.video as video_routes
+from utils.account_context import AccountContext
+
+_TESTS_DIR = Path(__file__).resolve().parent
+if str(_TESTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_TESTS_DIR))
+
+from test_openai_videos_route import ( # noqa: E402,F401 -- fixtures are found by name
+ _create,
+ _healthy_diffusers,
+ _wait_terminal,
+ backend,
+ client,
+)
+
+_OTHER = AccountContext("acct-other", "other", "user")
+
+
+def _drain() -> None:
+ asyncio.run(video_routes._drain_video_queue_once())
+
+
+def _delete_in_background(client, video_id: str) -> tuple:
+ import threading
+ import time
+
+ result = {}
+ thread = threading.Thread(
+ target = lambda: result.update(response = client.delete(f"/v1/videos/{video_id}"))
+ )
+ thread.start()
+ time.sleep(0.2)
+ return thread, result
+
+
+def _render_one(
+ client,
+ backend,
+ prompt = "first",
+) -> dict:
+ backend.gate.clear()
+ job = _create(client, {"prompt": prompt}).json()
+ assert job["status"] in ("queued", "in_progress")
+ return job
+
+
+def test_a_create_while_a_video_renders_is_queued_then_starts_with_its_id_seed_and_time(
+ client, backend
+):
+ first = _render_one(client, backend)
+ resp = client.post("/v1/videos", json = {"prompt": "second", "seed": 7, "size": "768x512"})
+ assert resp.status_code == 200, resp.json()
+ queued = resp.json()
+ assert queued["status"] == "queued" and queued["progress"] == 0
+ assert queued["id"].startswith("video_") and queued["seed"] == 7
+ assert queued["size"] == "768x512" and queued["prompt"] == "second"
+ listed = {v["id"]: v["status"] for v in client.get("/v1/videos").json()["data"]}
+ assert listed[queued["id"]] == "queued" and first["id"] in listed and len(listed) == 2
+
+ _drain()
+ assert client.get(f"/v1/videos/{queued['id']}").json()["status"] == "queued"
+ assert len(video_routes._VIDEO_QUEUE) == 1
+
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ _drain()
+ assert video_routes._VIDEO_QUEUE == []
+ done = _wait_terminal(client, queued["id"])
+ assert done["status"] == "completed"
+ assert done["created_at"] == queued["created_at"] and done["seed"] == 7
+ assert backend.last_generate_kwargs["prompt"] == "second"
+ assert backend.last_generate_kwargs["seed"] == 7
+
+
+def test_queued_videos_start_first_in_first_out_and_a_new_create_waits_behind_them(client, backend):
+ first = _render_one(client, backend)
+ a = client.post("/v1/videos", json = {"prompt": "a"}).json()
+ b = client.post("/v1/videos", json = {"prompt": "b"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+
+ c = client.post("/v1/videos", json = {"prompt": "c"}).json()
+ assert c["status"] == "queued"
+ assert [entry.id for entry in video_routes._VIDEO_QUEUE] == [a["id"], b["id"], c["id"]]
+
+ for expected in (a, b, c):
+ _drain()
+ assert _wait_terminal(client, expected["id"])["status"] == "completed"
+ assert backend.last_generate_kwargs["prompt"] == expected["prompt"]
+ assert video_routes._VIDEO_QUEUE == []
+
+
+def test_deleting_a_queued_video_removes_it_before_it_ever_starts(client, backend):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "never"}).json()
+ deleted = client.delete(f"/v1/videos/{queued['id']}")
+ assert deleted.status_code == 200
+ assert deleted.json() == {"id": queued["id"], "object": "video.deleted", "deleted": True}
+ assert video_routes._VIDEO_QUEUE == []
+ assert client.get(f"/v1/videos/{queued['id']}").status_code == 404
+
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ _drain()
+ assert backend.last_generate_kwargs["prompt"] == "first"
+
+
+def test_cancelling_a_queued_video_by_id_keeps_it_from_starting(client, backend):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "never"}).json()
+ cancelled = client.post("/api/inference/video/generate/cancel", json = {"video_id": queued["id"]})
+ assert cancelled.status_code == 200 and cancelled.json() == {"cancelled": True}
+ assert video_routes._VIDEO_QUEUE == []
+
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ _drain()
+ assert backend.last_generate_kwargs["prompt"] == "first"
+
+
+def test_a_stop_by_id_while_a_queued_start_waits_on_admission_stops_its_render(
+ client, backend, monkeypatch
+):
+ import threading
+
+ from core.inference import media_keepwarm
+
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "stopped"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ backend.gate.clear()
+ real_begin = media_keepwarm.begin_request
+ result = {}
+ threads = []
+
+ async def _stop_during_admission(*args, **kwargs):
+ thread = threading.Thread(
+ target = lambda: result.update(
+ response = client.post(
+ "/api/inference/video/generate/cancel", json = {"video_id": queued["id"]}
+ )
+ )
+ )
+ thread.start()
+ threads.append(thread)
+ await asyncio.sleep(0.2)
+ return await real_begin(*args, **kwargs)
+
+ monkeypatch.setattr(media_keepwarm, "begin_request", _stop_during_admission)
+ try:
+ _drain()
+ threads[0].join(10)
+ assert result["response"].status_code == 200, result["response"].json()
+ assert result["response"].json() == {"cancelled": True}
+ assert _wait_terminal(client, queued["id"])["status"] == "failed"
+ finally:
+ backend.gate.set()
+
+
+def test_a_video_deleted_while_its_start_is_in_flight_is_deleted_once_started(
+ client, backend, monkeypatch
+):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "late"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ real_start = video_routes._start_openai_video
+ pending = []
+
+ async def _start_then_delete(*args, **kwargs):
+ job = await real_start(*args, **kwargs)
+ pending.append(await asyncio.to_thread(_delete_in_background, client, queued["id"]))
+ return job
+
+ monkeypatch.setattr(video_routes, "_start_openai_video", _start_then_delete)
+ _drain()
+ [(thread, result)] = pending
+ thread.join(10)
+ assert result["response"].status_code == 200, result["response"].json()
+ assert video_routes._VIDEO_QUEUE == []
+ assert client.get(f"/v1/videos/{queued['id']}").status_code == 404
+ assert gallery_module.owned_video_path(queued["id"]) is None
+
+
+def test_a_queued_video_whose_start_is_refused_fails_with_the_reason(client, backend):
+ first = _render_one(client, backend)
+ doomed = client.post("/v1/videos", json = {"prompt": "doomed"}).json()
+ after = client.post("/v1/videos", json = {"prompt": "after"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+
+ backend.loaded = False
+ _drain()
+ failed = client.get(f"/v1/videos/{doomed['id']}").json()
+ assert failed["status"] == "failed"
+ assert failed["error"]["message"] == video_routes._NO_VIDEO_MODEL_MSG
+ assert failed["created_at"] == doomed["created_at"]
+ assert [entry.id for entry in video_routes._VIDEO_QUEUE] == [after["id"]]
+
+ backend.loaded = True
+ _drain()
+ assert _wait_terminal(client, after["id"])["status"] == "completed"
+
+
+def test_another_accounts_queued_video_is_neither_listed_nor_deletable(client, backend):
+ _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "theirs"}).json()
+ video_routes._VIDEO_QUEUE[0].account = _OTHER
+ assert client.get(f"/v1/videos/{queued['id']}").status_code == 404
+ assert queued["id"] not in [v["id"] for v in client.get("/v1/videos").json()["data"]]
+ assert client.delete(f"/v1/videos/{queued['id']}").status_code == 404
+ assert [entry.id for entry in video_routes._VIDEO_QUEUE] == [queued["id"]]
+ backend.gate.set()
+
+
+def test_the_queue_refuses_past_its_per_account_cap(client, backend, monkeypatch):
+ monkeypatch.setattr(video_routes, "VIDEO_QUEUE_MAX_PER_ACCOUNT", 1)
+ _render_one(client, backend)
+ assert client.post("/v1/videos", json = {"prompt": "one"}).json()["status"] == "queued"
+ full = client.post("/v1/videos", json = {"prompt": "two"})
+ assert full.status_code == 429
+ assert full.json()["error"]["code"] == "rate_limit_exceeded"
+ assert len(video_routes._VIDEO_QUEUE) == 1
+ backend.gate.set()
+
+
+def test_invalid_input_is_refused_at_once_rather_than_queued(client, backend):
+ _render_one(client, backend)
+ bad = client.post("/v1/videos", json = {"prompt": "x", "size": "abc"})
+ assert bad.status_code == 400
+ assert bad.json()["error"]["param"] == "size"
+ assert video_routes._VIDEO_QUEUE == []
+ backend.gate.set()
+
+
+def test_the_chat_tool_path_still_refuses_while_a_video_renders(client, backend):
+ from fastapi import HTTPException
+
+ from core.inference.video_families import VIDEO_GENERATION_BUSY_MSG
+ from models.inference import VideoJobCreateRequest
+
+ _render_one(client, backend)
+
+ async def _create_now():
+ return await video_routes._create_openai_video(
+ VideoJobCreateRequest(prompt = "tool"), None, "test-user", None
+ )
+
+ try:
+ asyncio.run(_create_now())
+ except HTTPException as exc:
+ assert exc.status_code == 409
+ assert exc.detail["error"]["message"] == VIDEO_GENERATION_BUSY_MSG
+ else:
+ raise AssertionError("the chat tool's create must not queue")
+ finally:
+ backend.gate.set()
+ assert video_routes._VIDEO_QUEUE == []
+
+
+def test_a_delete_during_the_drains_busy_check_keeps_the_video_from_starting(
+ client, backend, monkeypatch
+):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "gone"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ real_busy = video_routes._video_engine_busy
+
+ def _busy_check_with_a_delete():
+ assert client.delete(f"/v1/videos/{queued['id']}").status_code == 200
+ return real_busy()
+
+ monkeypatch.setattr(video_routes, "_video_engine_busy", _busy_check_with_a_delete)
+ _drain()
+ assert backend.last_generate_kwargs["prompt"] == "first"
+ assert video_routes._VIDEO_QUEUE == []
+ assert client.get(f"/v1/videos/{queued['id']}").status_code == 404
+
+
+def test_a_delete_while_a_refused_start_is_recorded_leaves_no_failed_job(
+ client, backend, monkeypatch
+):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "refused"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ backend.loaded = False
+ real_fail = video_routes._fail_queued_video
+ pending = []
+
+ def _fail_with_a_delete(entry, exc):
+ pending.append(_delete_in_background(client, queued["id"]))
+ real_fail(entry, exc)
+
+ monkeypatch.setattr(video_routes, "_fail_queued_video", _fail_with_a_delete)
+ _drain()
+ [(thread, result)] = pending
+ thread.join(10)
+ assert result["response"].status_code == 200, result["response"].json()
+ assert video_routes._VIDEO_QUEUE == []
+ assert client.get(f"/v1/videos/{queued['id']}").status_code == 404
+ assert queued["id"] not in [v["id"] for v in client.get("/v1/videos").json()["data"]]
+
+
+def test_a_delete_during_a_start_that_does_not_settle_is_refused_rather_than_reported_done(
+ client, backend, monkeypatch
+):
+ monkeypatch.setattr(video_routes, "_DELETE_SETTLE_TIMEOUT_S", 0.2)
+ _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "slow start"}).json()
+ [entry] = video_routes._VIDEO_QUEUE
+ entry.starting = True
+ still = client.delete(f"/v1/videos/{queued['id']}")
+ assert still.status_code == 409
+ assert still.json()["error"]["code"] == "video_not_ready"
+ assert video_routes._VIDEO_QUEUE == [entry]
+ entry.starting = False
+ assert client.delete(f"/v1/videos/{queued['id']}").status_code == 200
+ assert video_routes._VIDEO_QUEUE == []
+ backend.gate.set()
+
+
+def test_a_queued_video_of_a_disabled_account_never_starts(client, backend):
+ first = _render_one(client, backend)
+ client.post("/v1/videos", json = {"prompt": "theirs"})
+ video_routes._VIDEO_QUEUE[0].account = _OTHER
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ _drain()
+ assert video_routes._VIDEO_QUEUE == []
+ assert backend.last_generate_kwargs["prompt"] == "first"
+
+
+def _disable_while(monkeypatch, module, name) -> dict:
+ live = {"active": True}
+ real = getattr(module, name)
+
+ async def _wait_then_disable(*args, **kwargs):
+ live["active"] = False
+ return await real(*args, **kwargs)
+
+ monkeypatch.setattr(video_routes, "_queued_account_active", lambda account: live["active"])
+ monkeypatch.setattr(module, name, _wait_then_disable)
+ return live
+
+
+def _assert_disabled_mid_start_never_renders(client, backend, queued) -> None:
+ _drain()
+ assert video_routes._VIDEO_QUEUE == []
+ assert backend.last_generate_kwargs["prompt"] == "first"
+ assert client.get(f"/v1/videos/{queued['id']}").status_code == 404
+
+
+def test_an_account_disabled_during_the_admission_wait_never_starts(client, backend, monkeypatch):
+ from core.inference import media_keepwarm
+
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "theirs"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ _disable_while(monkeypatch, media_keepwarm, "begin_request")
+ _assert_disabled_mid_start_never_renders(client, backend, queued)
+
+
+def test_an_account_disabled_during_an_auto_switch_never_starts(client, backend, monkeypatch):
+ from core.inference import media_auto_switch
+
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "theirs"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ _disable_while(monkeypatch, media_auto_switch, "maybe_auto_switch_media_model")
+ _assert_disabled_mid_start_never_renders(client, backend, queued)
+
+
+def test_the_account_check_reads_the_accounts_current_state(monkeypatch):
+ import auth.storage as storage
+
+ accounts = {}
+ monkeypatch.setattr(storage, "get_account_by_id", accounts.get)
+ assert video_routes._queued_account_active(_OTHER) is False
+ accounts[_OTHER.account_id] = _OTHER
+ assert video_routes._queued_account_active(_OTHER) is True
+ owner = AccountContext("owner", "owner", "owner")
+ assert video_routes._queued_account_active(owner) is True
+
+
+def test_a_managed_account_denied_the_loaded_model_gets_a_403_not_its_name(
+ client, backend, monkeypatch
+):
+ from fastapi import HTTPException
+
+ from hub.services.models import account_access
+
+ _render_one(client, backend)
+
+ def _deny(status, modality = None):
+ raise HTTPException(status_code = 403, detail = "Model not found")
+
+ monkeypatch.setattr(account_access, "managed_account", lambda: True)
+ monkeypatch.setattr(account_access, "require_media_generation_access", _deny)
+ denied = client.post("/v1/videos", json = {"prompt": "x"})
+ assert denied.status_code == 403
+ assert "LTX" not in denied.text
+ assert video_routes._VIDEO_QUEUE == []
+ backend.gate.set()
+
+
+def test_a_queued_job_for_another_model_shows_none_of_the_loaded_models_details(
+ client, backend, monkeypatch
+):
+ import utils.openai_auto_switch_settings as auto_switch_settings
+ from core.inference import media_auto_switch
+
+ _render_one(client, backend)
+ monkeypatch.setattr(auto_switch_settings, "get_media_auto_switch_enabled", lambda: True)
+ monkeypatch.setattr(media_auto_switch, "resident_answers_media_request", lambda *a, **k: False)
+ queued = client.post("/v1/videos", json = {"prompt": "x", "model": "org/other-video"}).json()
+ assert queued["status"] == "queued"
+ assert (queued["model"], queued["size"], queued["seconds"]) == (
+ "org/other-video",
+ "auto",
+ "auto",
+ )
+ backend.gate.set()
+
+
+def test_a_queued_job_reports_the_size_and_seconds_an_immediate_start_would(client, backend):
+ first = _render_one(client, backend)
+ immediate_shape = {k: first[k] for k in ("size", "seconds", "model")}
+ queued = client.post("/v1/videos", json = {"prompt": "second"}).json()
+ assert queued["status"] == "queued"
+ assert {k: queued[k] for k in ("size", "seconds", "model")} == immediate_shape
+ timed = client.post("/v1/videos", json = {"prompt": "third", "seconds": "4"}).json()
+ assert timed["seconds"] == "4.04"
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ _drain()
+ started = _wait_terminal(client, queued["id"])
+ assert (started["size"], started["model"]) == (
+ immediate_shape["size"],
+ immediate_shape["model"],
+ )
+
+
+def test_creates_arriving_together_cannot_pass_the_cap_together(client, backend, monkeypatch):
+ import time
+
+ import httpx
+
+ monkeypatch.setattr(video_routes, "VIDEO_QUEUE_MAX_PER_ACCOUNT", 1)
+ real_display = video_routes._queued_job_display
+
+ def _slow_display(*args):
+ time.sleep(0.2)
+ return real_display(*args)
+
+ monkeypatch.setattr(video_routes, "_queued_job_display", _slow_display)
+ _render_one(client, backend)
+
+ async def run():
+ transport = httpx.ASGITransport(app = client.app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://studio") as studio:
+ return await asyncio.gather(
+ *(studio.post("/v1/videos", json = {"prompt": p}) for p in ("one", "two"))
+ )
+
+ try:
+ statuses = sorted(response.status_code for response in asyncio.run(run()))
+ finally:
+ backend.gate.set()
+ assert statuses == [200, 429]
+ assert len(video_routes._VIDEO_QUEUE) == 1
+
+
+def test_a_queued_start_is_counted_by_the_media_admission_gate(client, backend, monkeypatch):
+ from core.inference import media_keepwarm
+ from core.inference.gpu_arbiter import VIDEO
+
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "counted"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ tracker = media_keepwarm._TRACKERS[VIDEO]
+ real_start = video_routes._start_openai_video
+ seen = []
+
+ async def _start_and_count(*args, **kwargs):
+ seen.append(tracker.outstanding())
+ return await real_start(*args, **kwargs)
+
+ monkeypatch.setattr(video_routes, "_start_openai_video", _start_and_count)
+ before = tracker.outstanding()
+ _drain()
+ assert seen == [before + 1]
+ assert tracker.outstanding() == before
+ assert _wait_terminal(client, queued["id"])["status"] == "completed"
+
+
+def test_a_queued_video_waits_while_a_model_load_is_in_flight(client, backend):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "after the load"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ backend.loaded = False
+ backend.loading = ("unsloth/LTX-2.3-GGUF",)
+ _drain()
+ assert client.get(f"/v1/videos/{queued['id']}").json()["status"] == "queued"
+ assert [entry.id for entry in video_routes._VIDEO_QUEUE] == [queued["id"]]
+
+ backend.loading = ()
+ backend.loaded = True
+ _drain()
+ assert _wait_terminal(client, queued["id"])["status"] == "completed"
+
+
+def test_a_load_that_starts_during_the_admission_wait_keeps_the_job_queued(
+ client, backend, monkeypatch
+):
+ from core.inference import media_keepwarm
+
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "second"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ real_begin = media_keepwarm.begin_request
+
+ async def _switch_starts(*args, **kwargs):
+ backend.loaded = False
+ backend.loading = ("unsloth/LTX-2.3-GGUF",)
+ return await real_begin(*args, **kwargs)
+
+ monkeypatch.setattr(media_keepwarm, "begin_request", _switch_starts)
+ _drain()
+ assert client.get(f"/v1/videos/{queued['id']}").json()["status"] == "queued"
+ assert [entry.id for entry in video_routes._VIDEO_QUEUE] == [queued["id"]]
+
+ monkeypatch.setattr(media_keepwarm, "begin_request", real_begin)
+ backend.loading = ()
+ backend.loaded = True
+ _drain()
+ assert _wait_terminal(client, queued["id"])["status"] == "completed"
+
+
+def test_a_not_loaded_refusal_during_a_load_keeps_the_job_queued(client, backend, monkeypatch):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "second"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ real_start = video_routes._start_openai_video
+
+ async def _load_begins_mid_start(*args, **kwargs):
+ backend.loaded = False
+ backend.loading = ("unsloth/LTX-2.3-GGUF",)
+ return await real_start(*args, **kwargs)
+
+ monkeypatch.setattr(video_routes, "_start_openai_video", _load_begins_mid_start)
+ _drain()
+ assert client.get(f"/v1/videos/{queued['id']}").json()["status"] == "queued"
+
+ monkeypatch.setattr(video_routes, "_start_openai_video", real_start)
+ backend.loading = ()
+ backend.loaded = True
+ _drain()
+ assert _wait_terminal(client, queued["id"])["status"] == "completed"
+
+
+def test_model_dependent_input_is_refused_at_once_rather_than_queued(client, backend):
+ bad = ({"num_frames": 2}, {"size": "769x512"})
+ idle = [client.post("/v1/videos", json = {"prompt": "bad", **payload}) for payload in bad]
+ assert [r.status_code for r in idle] == [400, 400]
+ _render_one(client, backend)
+ try:
+ busy = [client.post("/v1/videos", json = {"prompt": "bad", **payload}) for payload in bad]
+ assert [r.status_code for r in busy] == [400, 400]
+ assert [r.json()["error"] for r in busy] == [r.json()["error"] for r in idle]
+ assert video_routes._VIDEO_QUEUE == []
+ finally:
+ backend.gate.set()
+
+
+def test_a_queued_job_names_the_resident_model_when_auto_switch_is_off(
+ client, backend, monkeypatch
+):
+ import utils.openai_auto_switch_settings as auto_switch_settings
+
+ monkeypatch.setattr(auto_switch_settings, "get_media_auto_switch_enabled", lambda: False)
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "x", "model": "sora-2"}).json()
+ assert queued["status"] == "queued"
+ assert queued["model"] == first["model"] != "sora-2"
+ backend.gate.set()
+
+
+def test_a_queued_start_holds_an_inference_slot_so_a_switch_cannot_evict_a_running_chat(
+ client, backend, monkeypatch
+):
+ from core.inference import llama_keepwarm, media_switch_backends
+
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "behind a chat"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ real_start = video_routes._start_openai_video
+ seen = []
+
+ async def _start_and_check(*args, **kwargs):
+ seen.append(media_switch_backends.chat_busy())
+ return await real_start(*args, **kwargs)
+
+ monkeypatch.setattr(video_routes, "_start_openai_video", _start_and_check)
+ before = llama_keepwarm.other_inference_request_count(current_request_counted = False)
+ llama_keepwarm._note_pending()
+ llama_keepwarm._note_start()
+ try:
+ _drain()
+ finally:
+ llama_keepwarm._note_end()
+ assert seen == [True]
+ assert llama_keepwarm.other_inference_request_count(current_request_counted = False) == before
+ assert _wait_terminal(client, queued["id"])["status"] == "completed"
+
+
+def test_an_oversized_reference_with_an_explicit_size_is_refused_rather_than_queued(
+ client, backend
+):
+ from test_openai_videos_route import _reference_image_bytes
+
+ png = _reference_image_bytes("PNG", (5000, 100))
+ wide = {"input_reference": ("frame.png", png, "image/png")}
+ _render_one(client, backend)
+ try:
+ r = _create(client, {"prompt": "wide", "size": "768x512"}, wide)
+ assert r.status_code == 400, r.text
+ assert r.json()["error"]["param"] == "input_reference"
+ assert "4096" in r.json()["error"]["message"]
+ finally:
+ backend.gate.set()
+
+
+def test_a_queued_image_to_video_job_reports_the_canvas_of_its_reference(client, backend):
+ from test_openai_videos_route import _reference_image_bytes
+
+ png = _reference_image_bytes("PNG", (300, 600))
+ portrait = {"input_reference": ("frame.png", png, "image/png")}
+ immediate = _create(client, {"prompt": "tall"}, portrait).json()
+ assert immediate["size"] == "704x1216"
+ _wait_terminal(client, immediate["id"])
+ _render_one(client, backend)
+ try:
+ queued = _create(client, {"prompt": "tall"}, portrait).json()
+ assert queued["status"] == "queued"
+ assert queued["size"] == immediate["size"]
+ assert client.get(f"/v1/videos/{queued['id']}").json()["size"] == immediate["size"]
+ finally:
+ backend.gate.set()
+
+
+def test_an_account_disabled_while_its_conditioning_is_prepared_never_starts(
+ client, backend, monkeypatch
+):
+ first = _render_one(client, backend)
+ queued = client.post("/v1/videos", json = {"prompt": "theirs"}).json()
+ backend.gate.set()
+ _wait_terminal(client, first["id"])
+ live = {"active": True}
+ real_resolve = backend._resolve_keyframes
+
+ def _disable_then_resolve(*args, **kwargs):
+ live["active"] = False
+ return real_resolve(*args, **kwargs)
+
+ monkeypatch.setattr(video_routes, "_queued_account_active", lambda account: live["active"])
+ monkeypatch.setattr(backend, "_resolve_keyframes", _disable_then_resolve)
+ _assert_disabled_mid_start_never_renders(client, backend, queued)
+ assert backend._generate_job_active is False
+ assert backend._active_generate_cancel is None
+
+
+def test_a_queued_job_for_another_downloaded_model_reports_that_models_size_and_seconds(
+ client, backend, monkeypatch
+):
+ from types import SimpleNamespace
+
+ import core.inference.media_model_index as media_model_index
+ import core.inference.video as video_module
+ import utils.openai_auto_switch_settings as auto_switch_settings
+ from core.inference import media_auto_switch
+ from core.inference.video_families import detect_video_family
+
+ wan = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
+ _render_one(client, backend)
+ monkeypatch.setattr(auto_switch_settings, "get_media_auto_switch_enabled", lambda: True)
+ monkeypatch.setattr(media_auto_switch, "resident_answers_media_request", lambda *a, **k: False)
+ monkeypatch.setattr(
+ media_model_index,
+ "resolve_local_media_model",
+ lambda *a, **k: SimpleNamespace(model_path = wan, gguf_filename = None),
+ )
+ monkeypatch.setattr(media_model_index, "expected_partition", lambda pick: None)
+ monkeypatch.setattr(
+ video_module, "_detect_load_family", lambda *a, **k: detect_video_family(wan)
+ )
+ try:
+ queued = client.post("/v1/videos", json = {"prompt": "x", "model": wan}).json()
+ assert queued["status"] == "queued"
+ assert (queued["size"], queued["seconds"]) == ("832x480", "2.06")
+ timed = client.post("/v1/videos", json = {"prompt": "y", "model": wan, "seconds": "4"})
+ assert timed.json()["seconds"] == "4.06"
+ finally:
+ backend.gate.set()
diff --git a/studio/backend/tests/test_openai_videos_route.py b/studio/backend/tests/test_openai_videos_route.py
index 74491ef7d2b..f078f231d7f 100644
--- a/studio/backend/tests/test_openai_videos_route.py
+++ b/studio/backend/tests/test_openai_videos_route.py
@@ -118,6 +118,8 @@ def backend(monkeypatch, tmp_path):
monkeypatch.setitem(gpu_arbiter._EVICTORS, gpu_arbiter.VIDEO, lambda: None)
monkeypatch.setattr(gallery_module, "gallery_dir", lambda: tmp_path)
monkeypatch.setattr(video_routes, "_jobs", {})
+ monkeypatch.setattr(video_routes, "_VIDEO_QUEUE", [])
+ monkeypatch.setattr(video_routes, "_ensure_video_drainer", lambda: None)
return backend
@@ -285,7 +287,9 @@ def test_json_body_and_the_studio_mount_work_too(client, backend):
@pytest.mark.parametrize("path", ["/v1/videos", "/api/inference/videos"])
-def test_new_jobs_rearm_progress_but_busy_requests_preserve_it(client, backend, monkeypatch, path):
+def test_new_jobs_rearm_progress_but_queued_requests_preserve_it(
+ client, backend, monkeypatch, path
+):
from loggers import media_progress
events = []
@@ -303,12 +307,14 @@ def test_new_jobs_rearm_progress_but_busy_requests_preserve_it(client, backend,
assert response.status_code == 200, response.json()
media_progress.log_media_generation_progress("video", progress)
assert len(events) == 1
- busy = client.post(path, json = {"prompt": "another"})
- assert busy.status_code == 409, busy.json()
+ queued = client.post(path, json = {"prompt": "another"})
+ assert queued.status_code == 200, queued.json()
+ assert queued.json()["status"] == "queued"
media_progress.log_media_generation_progress("video", progress)
assert len(events) == 1
backend.gate.set()
_wait_terminal(client, response.json()["id"])
+ video_routes._VIDEO_QUEUE.clear()
finally:
backend.gate.set()
media_progress.reset_media_generation_progress("video")
@@ -332,9 +338,12 @@ def test_in_progress_job_reports_progress_and_the_poll_hint(client, backend):
content = client.get(f"/v1/videos/{job['id']}/content")
assert content.status_code == 400
assert content.json()["error"]["code"] == "video_not_ready"
- busy = _create(client, {"prompt": "another"})
- assert busy.status_code == 409
- assert "error" in busy.json()
+ queued = _create(client, {"prompt": "another"})
+ assert queued.status_code == 200
+ assert queued.json()["status"] == "queued"
+ waiting = client.get(f"/v1/videos/{queued.json()['id']}")
+ assert waiting.json()["status"] == "queued"
+ assert waiting.headers["openai-poll-after-ms"] == "2000"
backend.gate.set()
done = _wait_terminal(client, job["id"])
assert done["status"] == "completed"