Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: name image and video loads so an unload ends only its own (#14827)

Daniel Hanc7f088c342e3
diff --git a/studio/backend/core/inference/chat_swap_kv.py b/studio/backend/core/inference/chat_swap_kv.py
index 4eceec34c63..e3c1bfced3a 100644
--- a/studio/backend/core/inference/chat_swap_kv.py
+++ b/studio/backend/core/inference/chat_swap_kv.py
@@ -68,10 +68,17 @@ def _leave_saving(key: tuple[str, Optional[str]]) -> None:
class PreparingLoad:
- """One load POST still before its engine registers the load; ``/<media>/unload`` cancels it."""
+ """One load POST still before its engine registers the load; ``/<media>/unload`` cancels it.
- def __init__(self, media: str):
+ ``load_token`` is the request's own token, so an unload naming it cancels this POST and no other."""
+
+ def __init__(
+ self,
+ media: str,
+ load_token: Optional[str] = None,
+ ):
self.media = media
+ self.load_token = load_token or None
self._cancelled = threading.Event()
# Held from the check through begin_load's registration, so a cancel lands before or after it.
self.lock = threading.RLock()
@@ -102,15 +109,19 @@ def cancel_preparing_loads(
account: Optional[str],
*,
any_account: bool = False,
+ load_token: Optional[str] = None,
) -> int:
- """Cancel ``media`` load POSTs in preflight: ``account``'s own, or all when the unload is not account scoped."""
+ """Cancel ``media`` load POSTs in preflight: ``account``'s own, or all when the unload is not account scoped.
+
+ With ``load_token``, only ``account``'s POSTs that carry that token (an unload naming its load).
+ Returns once each one either stopped or registered, as the token's lock is held until then."""
with _lock:
tokens = [
t
for (m, a), held in _preparing.items()
if m == media and (any_account or a == account)
for t in held
- if not t.cancelled
+ if not t.cancelled and (load_token is None or t.load_token == load_token)
]
# Outside _lock: this waits for a registration in flight under the token's own lock.
for token in tokens:
@@ -130,11 +141,24 @@ def raise_if_load_cancelled(message: str) -> None:
raise RuntimeError(message)
+def run_unless_load_cancelled(message: str, fn, *args, **kwargs):
+ """``fn(*args, **kwargs)`` under this POST's preflight lock once no unload has cancelled it, so a
+ destructive step (an engine switch, another resident's unload) never runs for a cancelled load:
+ a cancel lands before the check, or waits for ``fn`` to finish."""
+ token = _current_preparing.get()
+ if token is None:
+ return fn(*args, **kwargs)
+ with token.lock:
+ if token.cancelled:
+ raise RuntimeError(message)
+ return fn(*args, **kwargs)
+
+
@contextlib.contextmanager
-def preparing_load(media: str):
+def preparing_load(media: str, load_token: Optional[str] = None):
"""Held around a load POST until begin_load registers it, so load-progress reads it as starting; yields the token an unload cancels."""
key = (media, _account_id())
- token = PreparingLoad(media)
+ token = PreparingLoad(media, load_token)
with _lock:
_preparing.setdefault(key, []).append(token)
reset = _current_preparing.set(token)
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index a44a36afb9e..fbb9f926775 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -14369,13 +14369,21 @@ class DiffusionBackend:
return True
@_invalidates_gpu_memory("diffusion unload")
- def unload(self, *, expected_account: Optional[str] = None) -> dict[str, Any]:
+ def unload(
+ self,
+ *,
+ expected_account: Optional[str] = None,
+ only_if: Optional[Callable[[], bool]] = None,
+ ) -> Optional[dict[str, Any]]:
# fenced, and the try that owns it, start BEFORE the counter moves: a leaked _unload_waiters
# would make _wait_for_pending_unloads block every later load for the life of the process,
# and the flag keeps the finally honest when authorization refuses before the fence is raised.
fenced = False
try:
with self._generation_cancel_lock, self._load_cancel_lock:
+ # Rechecked under the load worker's lock, so a load that ended since costs nothing.
+ if only_if is not None and not only_if():
+ return None
if expected_account is not None:
from .gpu_arbiter import DIFFUSION, GpuBusyForAnotherAccountError
from hub.services.models.account_access import require_resident_control
diff --git a/studio/backend/core/inference/media_load_ids.py b/studio/backend/core/inference/media_load_ids.py
new file mode 100644
index 00000000000..1f32868802d
--- /dev/null
+++ b/studio/backend/core/inference/media_load_ids.py
@@ -0,0 +1,333 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Client load tokens for the image and video slots, so an unload can name the load it means.
+
+A load request may carry ``load_token`` (an opaque id the client makes up). The load route
+records it when the engine accepts the load, under ``account_access.media_load_lock``, together
+with the engine, its load epoch (``_load_token``, bumped by every begin_load and unload) and the
+state that was resident before. ``POST .../unload`` with ``expected_load_id`` then unloads only
+while that record still describes the engine: no begin_load or unload ran since (same epoch),
+and either a load is in flight (it can only be that one) or a state that load published is
+resident. Anything else answers without touching the slot:
+
+- ``superseded``: the token was admitted, but its load ended or another load replaced it;
+- ``cancelled``: the token was never admitted (its POST is still on its way, or was lost), so it
+ is remembered and that load is refused when it arrives, for the account that sent the unload only.
+
+A load sent without a token gets one from Studio, and the status routes report the token of the
+load in the slot as ``load_id``, so a client can unload exactly the model it saw.
+
+The check and the unload run under the same lock as the admission, so no load can be admitted
+between them. Loads without a token, and unloads without ``expected_load_id``, behave as before.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+import uuid
+import weakref
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+# A cancellation is kept until its load arrives and is refused (which consumes it) or this long
+# has passed. No hard bound on a load POST's lifetime exists to derive it from (the MCP forwards
+# in-process with no client timeout, studio_mcp/forward.py, and the route's waits before
+# admission, the chat context save and the arbiter's eviction, have none either), so it is set far
+# beyond any real one: a load still on its way after a day is not one an unload is racing.
+_CANCELLED_TTL_S = 24 * 3600.0
+# Memory guard per (slot, account): only an account's own newer tokens can push one out, never
+# another account's.
+_PER_ACCOUNT = 4096
+_now = time.monotonic
+# Bumped under _lock on every admission and token unload, so a status read can tell whether its
+# state changed while it ran.
+_generation = 0
+# Status reads snapshot() tries before answering with no load id (an admission landed in each).
+_SNAPSHOT_TRIES = 3
+MAX_TOKEN_LEN = 128
+
+
+class LoadCancelledError(RuntimeError):
+ """A load whose token an unload already cancelled; the load routes answer it as a 409."""
+
+ def __init__(self) -> None:
+ super().__init__("This load was cancelled by an unload before it started.")
+
+
+@dataclass
+class _Record:
+ token: Optional[str]
+ engine: Any
+ epoch: Any
+ # A weak reference to the state resident before this load (see _weak), never the model itself.
+ prior: Callable[[], Any]
+
+
+def _weak(state: Any) -> Callable[[], Any]:
+ """``state`` without keeping it alive: a torn-down model (pipeline, weights) must be freed as soon
+ as its engine drops it, not when this registry is next cleared. Call it to get the state back,
+ or None once it is gone. The engines' states are plain dataclasses, so weakly referable; anything
+ that is not is held as is."""
+ if state is None:
+ return lambda: None
+ try:
+ return weakref.ref(state)
+ except TypeError:
+ return lambda: state
+
+
+_lock = threading.Lock()
+_current: dict[str, _Record] = {}
+# The load that published the resident state before the last admission, as (token, engine, state):
+# a newer load that failed leaves it resident.
+_published: dict[str, tuple[str, Any, Any]] = {}
+# Per (slot, account) whose load was admitted with the token. Losing an old one only turns a
+# superseded answer into cancelled, which touches nothing either.
+_seen: dict[tuple[str, str], "OrderedDict[str, None]"] = {}
+# Per (slot, account) that sent the unload: token -> expiry (time.monotonic()). An unload cancels
+# a token for its own account only.
+_cancelled: dict[tuple[str, str], "OrderedDict[str, float]"] = {}
+
+
+def _account() -> str:
+ from utils.account_context import current_account_id
+ return current_account_id()
+
+
+def _remember(
+ table: dict,
+ modality: str,
+ token: str,
+ value: Any = None,
+) -> None:
+ tokens = table.setdefault((modality, _account()), OrderedDict())
+ tokens[token] = value
+ tokens.move_to_end(token)
+ while len(tokens) > _PER_ACCOUNT:
+ tokens.popitem(last = False)
+
+
+def _cancel(modality: str, token: str) -> None:
+ now = _now()
+ tokens = _cancelled.get((modality, _account()))
+ while tokens and next(iter(tokens.values())) <= now:
+ tokens.popitem(last = False)
+ _remember(_cancelled, modality, token, now + _CANCELLED_TTL_S)
+
+
+def _media_lock():
+ from hub.services.models.account_access import media_load_lock
+ return media_load_lock
+
+
+def refuse_cancelled(modality: str, load_token: Optional[str]) -> None:
+ """Raise LoadCancelledError when an unload already cancelled ``load_token``."""
+ if load_token:
+ with _lock:
+ tokens = _cancelled.get((modality, _account()), {})
+ expiry = tokens.pop(load_token, None)
+ # Consumed: its one load has now arrived, and is refused.
+ if expiry is not None and expiry > _now():
+ raise LoadCancelledError()
+
+
+def admitting(modality: str, engine: Any, load_token: Optional[str]) -> Any:
+ """Before the engine's begin_load: refuse a cancelled token, and snapshot what is resident.
+
+ Called inside ``account_access.admit_media_load``, so under ``media_load_lock``, as is
+ ``admitted`` once begin_load returned. Every image and video load goes through both, with or
+ without a token, so a stale record never outlives a newer load.
+ """
+ refuse_cancelled(modality, load_token)
+ with _lock:
+ record = _current.get(modality)
+ state = getattr(engine, "_state", None)
+ loading = getattr(engine, "loading_repo_ids", None)
+ in_flight = callable(loading) and bool(loading())
+ if (
+ record is not None
+ and state is not None
+ and not in_flight
+ and _describes(record, engine)
+ ):
+ # The resident model's load: still named while a newer load fails to replace it.
+ _published[modality] = (record.token, engine, _weak(state))
+ # Before the worker starts: a cached load can publish its state before begin_load returns.
+ return state
+
+
+def admitted(modality: str, engine: Any, load_token: Optional[str], prior: Any) -> None:
+ """The engine accepted the load: record it as the one in the slot (a load without a token gets one)."""
+ global _generation
+ with _lock:
+ _generation += 1
+ token = load_token or f"studio-{uuid.uuid4().hex}"
+ _current[modality] = _Record(
+ token, engine, getattr(engine, "_load_token", None), _weak(prior)
+ )
+ # Per account like _cancelled: another account's earlier use of a token never answers
+ # superseded for this one, which then records its own cancellation.
+ _remember(_seen, modality, token)
+
+
+def _describes(record: _Record, engine: Any) -> bool:
+ # The engine still holds the load this record names: no begin_load or unload since, and
+ # either a load in flight or the state that load published.
+ if record.engine is not engine or getattr(engine, "_load_token", None) != record.epoch:
+ return False
+ loading = getattr(engine, "loading_repo_ids", None)
+ if callable(loading) and loading():
+ return True
+ state = getattr(engine, "_state", None)
+ return state is not None and state is not record.prior()
+
+
+def _slot_token(modality: str, engine: Any) -> Optional[str]:
+ # The token of the load in flight or resident in ``engine``, if Studio can name it (under _lock).
+ record = _current.get(modality)
+ if record is not None and _describes(record, engine):
+ return record.token
+ loading = getattr(engine, "loading_repo_ids", None)
+ if callable(loading) and loading():
+ return None
+ published = _published.get(modality)
+ state = getattr(engine, "_state", None)
+ if published is not None and published[1] is engine and state is not None:
+ if published[2]() is state:
+ return published[0]
+ return None
+
+
+def _resident_token(modality: str, engine: Any) -> Optional[str]:
+ # The token of the state resident in ``engine`` (under _lock). Unlike _slot_token, a load in
+ # flight over it does not hide it: a replacement that fails leaves this model loaded.
+ state = getattr(engine, "_state", None)
+ if state is None:
+ return None
+ record = _current.get(modality)
+ loading = getattr(engine, "loading_repo_ids", None)
+ in_flight = callable(loading) and bool(loading())
+ if record is not None and not in_flight and _describes(record, engine):
+ return record.token
+ published = _published.get(modality)
+ if published is not None and published[1] is engine and published[2]() is state:
+ return published[0]
+ return None
+
+
+def snapshot(
+ modality: str, engine: Any, read_status: Callable[[], Any]
+) -> tuple[Any, Optional[str]]:
+ """``read_status()`` and the ``load_id`` of the same state, so the id never names a load the status
+ does not show.
+
+ The read runs outside every lock (it can block, and must not hold up admissions or unloads); the
+ id is taken only if no admission or token unload landed while it ran, else the read is retried.
+ After ``_SNAPSHOT_TRIES`` the answer carries no id. A load still starting when the read begins
+ records its token only at its admission, which moves the generation, and an engine that began a
+ load since its record was taken is not named by it, so neither pairing can mix two states.
+ """
+ status, load_id, _ = snapshot_ids(modality, engine, read_status)
+ return status, load_id
+
+
+def snapshot_ids(
+ modality: str, engine: Any, read_status: Callable[[], Any]
+) -> tuple[Any, Optional[str], Optional[str]]:
+ """``snapshot`` plus ``resident_load_id``: the token of the model resident under a load still in
+ flight (the same as ``load_id`` when none is), taken under the same generation check."""
+ status = None
+ for _ in range(_SNAPSHOT_TRIES):
+ with _lock:
+ generation = _generation
+ status = read_status()
+ with _lock:
+ if _generation == generation:
+ return status, _slot_token(modality, engine), _resident_token(modality, engine)
+ return status, None, None
+
+
+def current_id(modality: str, engine: Any) -> Optional[str]:
+ """The ``load_id`` the status routes report: the token of the load in the slot, or None."""
+ with _lock:
+ return _slot_token(modality, engine)
+
+
+def _in_flight_unlocked(engine: Any) -> bool:
+ # Read inside the engine's own lock (from its unload), so never through loading_repo_ids(),
+ # which takes that lock: its in-flight marker, as loading_repo_ids() reads it.
+ if not hasattr(engine, "_loading"):
+ loading = getattr(engine, "loading_repo_ids", None)
+ return bool(callable(loading) and loading())
+ loading = engine._loading
+ return loading is not None and getattr(loading, "error", None) is None
+
+
+def _still_named(record: Optional[_Record], published: Any, engine: Any) -> Callable[[], bool]:
+ """What the engine's unload rechecks under the lock its load worker settles under: the load the
+ token named is still the one in flight or resident (no load ended or began since the check)."""
+ if record is not None:
+
+ def guard() -> bool:
+ if getattr(engine, "_load_token", None) != record.epoch:
+ return False
+ if _in_flight_unlocked(engine):
+ return True
+ state = getattr(engine, "_state", None)
+ return state is not None and state is not record.prior()
+
+ return guard
+ # The weak reference, so this recheck does not keep alive the model the unload tears down.
+ resident = published[2]
+
+ def guard_resident() -> bool:
+ state = getattr(engine, "_state", None)
+ return state is not None and state is resident() and not _in_flight_unlocked(engine)
+
+ return guard_resident
+
+
+def unload_if_current(
+ modality: str, engine: Any, load_token: str, unload: Callable[[Callable[[], bool]], Any]
+) -> tuple[str, Any]:
+ """Unload only the load ``load_token`` names: ("unloaded", status), ("superseded", None) or ("cancelled", None).
+
+ ``unload(only_if)`` runs the engine's unload with that recheck: it answers None, and tears
+ nothing down, if the named load ended in between (a replacement whose worker failed leaves the
+ previous model resident, which is not this token's to unload).
+ """
+ global _generation
+ with _media_lock():
+ with _lock:
+ record = _current.get(modality)
+ if _slot_token(modality, engine) != load_token:
+ if load_token in _seen.get((modality, _account()), ()):
+ return "superseded", None
+ # Not admitted yet: refuse it when it comes, which a lost POST response cannot stop.
+ # Only for this account, so no account can cancel another's load in advance.
+ _cancel(modality, load_token)
+ return "cancelled", None
+ named = record if record is not None and record.token == load_token else None
+ only_if = _still_named(named, _published.get(modality), engine)
+ # Still under media_load_lock, so no load is admitted before this unload has run
+ # (as retire_media_load does).
+ status = unload(only_if)
+ if status is None:
+ return "superseded", None
+ with _lock:
+ _generation += 1
+ if _current.get(modality) is record:
+ del _current[modality]
+ _published.pop(modality, None)
+ return "unloaded", status
+
+
+def _reset_for_tests() -> None:
+ with _lock:
+ _current.clear()
+ _published.clear()
+ _seen.clear()
+ _cancelled.clear()
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.py
index 97a25465a78..925962551b9 100644
--- a/studio/backend/core/inference/mlx_diffusion_backend.py
+++ b/studio/backend/core/inference/mlx_diffusion_backend.py
@@ -1460,8 +1460,16 @@ class MlxDiffusionBackend:
# ── Unload / status ──────────────────────────────────────────────────────
@_invalidates_gpu_memory("mlx diffusion unload")
- def unload(self, *, expected_account: Optional[str] = None) -> dict[str, Any]:
+ def unload(
+ self,
+ *,
+ expected_account: Optional[str] = None,
+ only_if: Optional[Callable[[], bool]] = None,
+ ) -> Optional[dict[str, Any]]:
with self._lock:
+ # Rechecked under the load worker's lock, so a load that ended since costs nothing.
+ if only_if is not None and not only_if():
+ return None
if expected_account is not None:
from .gpu_arbiter import DIFFUSION, GpuBusyForAnotherAccountError
from hub.services.models.account_access import require_resident_control
diff --git a/studio/backend/core/inference/mlx_diffusion_routing.py b/studio/backend/core/inference/mlx_diffusion_routing.py
index cb49220c24c..b4c22dde6bc 100644
--- a/studio/backend/core/inference/mlx_diffusion_routing.py
+++ b/studio/backend/core/inference/mlx_diffusion_routing.py
@@ -410,6 +410,9 @@ class RoutedMlxEngine:
self._load_kwargs: Optional[dict[str, Any]] = None
self._fallback_gate: Optional[Callable[..., Any]] = None
self._epoch = 0
+ # The logical load's epoch (media_load_ids reads it as _load_token): moved by begin_load and
+ # unload only, never by a fallback, whose engines keep counters of their own.
+ self._logical_load = 0
# Replacement loads between their first epoch bump and registering their options: no fallback registers.
self._replacing = 0
self._diffusers_ours = False
@@ -442,6 +445,10 @@ class RoutedMlxEngine:
def _state(self) -> Any:
return getattr(self._primary(), "_state", None)
+ @property
+ def _load_token(self) -> int:
+ return self._logical_load
+
@property
def is_loaded(self) -> bool:
return self._state is not None
@@ -529,6 +536,8 @@ class RoutedMlxEngine:
self._last = None
self._mlx().begin_load(repo_id, **kwargs)
with self._lock:
+ # Only once MLX accepted it: a refused load leaves the in-flight load's epoch alone.
+ self._logical_load += 1
self._load_kwargs = {"repo_id": repo_id, **kwargs}
self._fallback_gate = fallback_gate
finally:
@@ -777,7 +786,17 @@ class RoutedMlxEngine:
return self._diffusers().cancel_generate(expected_account = expected_account)
return self._mlx().cancel_generate(expected_account = expected_account)
- def unload(self, *, expected_account: Optional[str] = None) -> dict[str, Any]:
+ def unload(
+ self,
+ *,
+ expected_account: Optional[str] = None,
+ only_if: Optional[Callable[[], bool]] = None,
+ ) -> Optional[dict[str, Any]]:
+ # A conditional unload (a load token naming the load): refused before the drain, which would
+ # end the fallback that load runs on; the MLX engine rechecks it under the lock its worker
+ # settles under. None, and nothing torn down, when the named load already ended.
+ if only_if is not None and not only_if():
+ return None
# Drain the fallback first and stay fenced until teardown ends, so none reloads MLX or registers meanwhile.
if expected_account is not None:
with self._lock:
@@ -795,16 +814,18 @@ class RoutedMlxEngine:
raise GpuBusyForAnotherAccountError(DIFFUSION, 1)
ours = self._drain_fallback(replacing = True)
try:
- if expected_account is None:
- self._mlx().unload()
- else:
- self._mlx().unload(expected_account = expected_account)
+ kwargs = {} if expected_account is None else {"expected_account": expected_account}
+ if only_if is not None:
+ kwargs["only_if"] = only_if
+ if self._mlx().unload(**kwargs) is None and only_if is not None:
+ return None
if ours:
if expected_account is None:
self._diffusers().unload()
else:
self._diffusers().unload(expected_account = expected_account)
with self._lock:
+ self._logical_load += 1
self._diffusers_ours = False
self._mlx_dropped = False
self._origin_identity = None
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index 494048bff36..bb9e4e661f2 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -32,7 +32,7 @@ import time
import weakref
from dataclasses import dataclass, field, replace
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Optional
from core.inference.inference_gate import holds_inference, pauses_background_embedding
from core.inference.diffusion_auto_policy import build_resolved_record, format_generation_for_log
@@ -5288,8 +5288,16 @@ class SdCppDiffusionBackend:
# ── Unload / status ──────────────────────────────────────────────────────
@_invalidates_gpu_memory("sd.cpp unload")
- def unload(self, *, expected_account: Optional[str] = None) -> dict[str, Any]:
+ def unload(
+ self,
+ *,
+ expected_account: Optional[str] = None,
+ only_if: Optional[Callable[[], bool]] = None,
+ ) -> Optional[dict[str, Any]]:
with self._lock:
+ # Rechecked under the load worker's lock, so a load that ended since costs nothing.
+ if only_if is not None and not only_if():
+ return None
if expected_account is not None:
from .gpu_arbiter import DIFFUSION, GpuBusyForAnotherAccountError
from hub.services.models.account_access import require_resident_control
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 7bdd98e9930..e7a59075bd7 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -13649,8 +13649,16 @@ class VideoBackend:
reclaim_host_memory(logger = logger)
@_invalidates_gpu_memory("video unload")
- def unload(self, *, expected_account: Optional[str] = None) -> dict[str, Any]:
+ def unload(
+ self,
+ *,
+ expected_account: Optional[str] = None,
+ only_if: Optional[Callable[[], bool]] = None,
+ ) -> Optional[dict[str, Any]]:
with self._lock:
+ # Rechecked under the load worker's lock, so a load that ended since costs nothing.
+ if only_if is not None and not only_if():
+ return None
if expected_account is not None:
from .gpu_arbiter import VIDEO, GpuBusyForAnotherAccountError
from hub.services.models.account_access import require_resident_control
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index efd4fac7356..7e725aef751 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -4612,9 +4612,42 @@ class AnthropicMessagesResponse(BaseModel):
usage: AnthropicUsage = Field(default_factory = AnthropicUsage)
+class MediaUnloadRequest(BaseModel):
+ """Optional body of POST /images/unload and /video/unload."""
+
+ expected_load_id: Optional[str] = Field(
+ None,
+ max_length = 128,
+ description = "Unload only the load started with this load_token, and only while it is in "
+ "flight or resident. A load that ended or was replaced is left alone (outcome superseded); "
+ "one that has not started yet is refused when it arrives (outcome cancelled). Omit to "
+ "unload whatever is loaded.",
+ )
+
+
+class MediaUnloadOutcome(BaseModel):
+ """POST /images/unload or /video/unload with expected_load_id that touched nothing.
+
+ Nothing of the slot is described: the load in it may be another account's.
+ """
+
+ unloaded: Literal[False] = Field(..., description = "Always false: nothing was unloaded")
+ outcome: Literal["superseded", "cancelled"] = Field(
+ ...,
+ description = "superseded: that load ended or a newer one replaced it. cancelled: it has "
+ "not started yet, and is refused when it arrives.",
+ )
+
+
class DiffusionLoadRequest(BaseModel):
"""Request to load a local diffusion (text-to-image) checkpoint."""
+ load_token: Optional[str] = Field(
+ None,
+ max_length = 128,
+ description = "Opaque id the client picks for this load; POST /images/unload with "
+ "expected_load_id set to it unloads this load only, and cancels it if it has not started yet.",
+ )
model_path: str = Field(..., description = "Diffusion repo id or local path")
display_repo_id: Optional[str] = Field(None, description = "Logical Hub id of a local snapshot")
_blank_display_id = field_validator("display_repo_id")(lambda v: (v or "").strip() or None)
@@ -5876,6 +5909,16 @@ class DiffusionDownloadPlanResponse(BaseModel):
class DiffusionStatusResponse(BaseModel):
"""Current diffusion backend state."""
+ load_id: Optional[str] = Field(
+ None,
+ description = "The load_token of the load in flight or resident (one Studio made up for a "
+ "load sent without one); POST unload with expected_load_id set to it unloads that load only.",
+ )
+ resident_load_id: Optional[str] = Field(
+ None,
+ description = "The load_token of the model resident in the slot, also while another load is "
+ "in flight over it (equal to load_id when none is); a failed replacement leaves this model loaded.",
+ )
resident_mib: Optional[int] = Field(
None,
description = "Best-effort memory the resident model holds, in MiB; null when unknown.",
@@ -6135,6 +6178,20 @@ class DiffusionStatusResponse(BaseModel):
)
+class DiffusionLoadUnloadedResponse(DiffusionStatusResponse):
+ """POST /images/unload with expected_load_id that unloaded that load: the state it left behind."""
+
+ unloaded: Literal[True] = Field(..., description = "Always true: that load was unloaded")
+ outcome: Literal["unloaded"] = Field(..., description = "Always unloaded")
+
+
+# What POST /images/unload answers: the state it left behind without expected_load_id, and with it
+# either that, marked unloaded, or the outcome alone.
+DiffusionUnloadResponse = Union[
+ DiffusionLoadUnloadedResponse, MediaUnloadOutcome, DiffusionStatusResponse
+]
+
+
class DiffusionInferenceInfo(BaseModel):
"""bf16 component sizes and 0.1 GB resident estimates by transformer quantization scheme."""
@@ -7017,6 +7074,12 @@ class VideoJobDeleteResponse(BaseModel):
class VideoLoadRequest(BaseModel):
"""Request to load a local text-to-video checkpoint."""
+ load_token: Optional[str] = Field(
+ None,
+ max_length = 128,
+ description = "Opaque id the client picks for this load; POST /video/unload with "
+ "expected_load_id set to it unloads this load only, and cancels it if it has not started yet.",
+ )
model_path: str = Field(..., description = "Video repo id or local path")
display_repo_id: Optional[str] = Field(None, description = "Logical Hub id of a local snapshot")
_blank_display_id = field_validator("display_repo_id")(lambda v: (v or "").strip() or None)
@@ -7563,6 +7626,16 @@ class VideoGenerationDefaults(BaseModel):
class VideoStatusResponse(BaseModel):
"""Current video backend state."""
+ load_id: Optional[str] = Field(
+ None,
+ description = "The load_token of the load in flight or resident (one Studio made up for a "
+ "load sent without one); POST unload with expected_load_id set to it unloads that load only.",
+ )
+ resident_load_id: Optional[str] = Field(
+ None,
+ description = "The load_token of the model resident in the slot, also while another load is "
+ "in flight over it (equal to load_id when none is); a failed replacement leaves this model loaded.",
+ )
resident_mib: Optional[int] = Field(
None,
description = "Best-effort memory the resident model holds, in MiB; null when unknown.",
@@ -7707,3 +7780,15 @@ class VideoStatusResponse(BaseModel):
description = "Per-control resolved value + provenance (source auto|explicit + reason), "
"keyed by Advanced control name; null when unloaded or unavailable.",
)
+
+
+class VideoLoadUnloadedResponse(VideoStatusResponse):
+ """POST /video/unload with expected_load_id that unloaded that load: the state it left behind."""
+
+ unloaded: Literal[True] = Field(..., description = "Always true: that load was unloaded")
+ outcome: Literal["unloaded"] = Field(..., description = "Always unloaded")
+
+
+# What POST /video/unload answers: the state it left behind without expected_load_id, and with it
+# either that, marked unloaded, or the outcome alone.
+VideoUnloadResponse = Union[VideoLoadUnloadedResponse, MediaUnloadOutcome, VideoStatusResponse]
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3aaff1d0fc0..c34a2128b48 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3797,6 +3797,10 @@ from models.inference import (
SttLoadRequest,
GenerateRequest,
DiffusionLoadRequest,
+ MediaUnloadOutcome,
+ MediaUnloadRequest,
+ DiffusionLoadUnloadedResponse,
+ DiffusionUnloadResponse,
DiffusionGenerateRequest,
DiffusionGenerateResponse,
MaskSegmentRequest,
@@ -51397,6 +51401,31 @@ def _activate_checked(check, activate):
before_engine_switch.reset(token)
+def _activate_unless_cancelled(preflight, check, activate):
+ """``_activate_checked``, refused before an engine switch unloads the active engine once an unload
+ cancelled this load's preflight. The preflight lock is taken at that check and held until the
+ switch is done (not through an sd-cli install before it), so a cancel lands before or after it."""
+ if preflight is None:
+ return _activate_checked(check, activate)
+ from core.inference.chat_swap_kv import raise_if_load_cancelled
+
+ held = []
+
+ def _check():
+ if not held:
+ preflight.lock.acquire()
+ held.append(True)
+ raise_if_load_cancelled(_DIFFUSION_LOAD_CANCELLED)
+ if check is not None:
+ check()
+
+ try:
+ return _activate_checked(_check, activate)
+ finally:
+ if held:
+ preflight.lock.release()
+
+
def _guard_diffusion_load_against_training() -> None:
"""Refuse loading an image model while a training run is active. Unlike chat,
a diffusion pipeline's VRAM can't be cheaply estimated before the load, so the
@@ -51880,7 +51909,7 @@ async def load_diffusion_model(
from core.inference.chat_swap_kv import preparing_load
try:
- with preparing_load("image"):
+ with preparing_load("image", request.load_token):
loaded = await load_diffusion_model_gated(request, current_subject, user_initiated = True)
except HTTPException as exc:
# A load that RAISES skips both wrappers, and the inner loader redacted native paths
@@ -51982,8 +52011,12 @@ async def load_diffusion_model_gated(
from core.inference.diffusion_hub_evidence import prefetch_for_pick
from utils.native_path_leases import redact_native_paths
+ from core.inference import media_load_ids
+
backend = get_diffusion_backend()
try:
+ # A load an unload already cancelled by its token starts nothing (checked again at admission).
+ media_load_ids.refuse_cancelled("diffusion", request.load_token)
# Resolve the load kind once (gguf / single_file / pipeline) so validation, engine selection and the load agree. A bad kind raises here, so a 400.
kind = resolve_model_kind(request.gguf_filename, request.model_kind)
# A local On-Device pick can be a bare single-file .safetensors directory; if it holds exactly one checkpoint, reinterpret it as a single_file load so all three paths agree.
@@ -52186,17 +52219,29 @@ async def load_diffusion_model_gated(
note_evictions(replaced_media, evicted_slots)
# begin_load signals whatever generation is running, so guard on every device, not just GPU.
require_no_foreign_generations()
- from core.inference.chat_swap_kv import preparing_load_token, raise_if_load_cancelled
+ from core.inference.chat_swap_kv import (
+ preparing_load_token,
+ raise_if_load_cancelled,
+ run_unless_load_cancelled,
+ )
raise_if_load_cancelled(_DIFFUSION_LOAD_CANCELLED)
+ preflight = preparing_load_token()
if off_torch is not None and pending_name == ENGINE_SD_CPP:
+ # Checked under the preflight lock, so a cancelled load never unloads the video model.
await asyncio.to_thread(
- _unload_native_video_sharing_the_sd_cpp_tree, multi_model_mode, evicted_slots
+ run_unless_load_cancelled,
+ _DIFFUSION_LOAD_CANCELLED,
+ _unload_native_video_sharing_the_sd_cpp_tree,
+ multi_model_mode,
+ evicted_slots,
)
+ raise_if_load_cancelled(_DIFFUSION_LOAD_CANCELLED)
# Pick the engine for this host (diffusers on GPU, native sd.cpp otherwise), installing sd-cli if needed, BEFORE evicting chat.
# Under Always the switch re-checks under the router's transition lock: another load may have landed meanwhile.
engine = await asyncio.to_thread(
- _activate_checked,
+ _activate_unless_cancelled,
+ preflight,
functools.partial(refuse_replacing_media, DIFFUSION, multi_model_mode, media_names)
if multi_model_mode == "always"
else None,
@@ -52297,6 +52342,7 @@ async def load_diffusion_model_gated(
# Snapshotted BEFORE the worker starts: a cached load can publish before begin_load returns.
prior_state = loaded_state_of(engine)
# Kicks the slow load onto a background thread and returns at once (the client polls images/load-progress).
+ prior_load = media_load_ids.admitting("diffusion", engine, request.load_token)
started = engine.begin_load(
request.model_path,
display_repo_id = request.display_repo_id,
@@ -52331,6 +52377,7 @@ async def load_diffusion_model_gated(
_mlx_fallback_load_gate(request, gpu_ordinal, multi_model_mode),
),
)
+ media_load_ids.admitted("diffusion", engine, request.load_token, prior_load)
# A warm start that took the generation slot between the stop above and begin_load publishing the load would
# bring the old model's upsampler up beside this one: cancel it now that the load is visible.
_cancel_upsampler_warm_start()
@@ -52339,8 +52386,6 @@ async def load_diffusion_model_gated(
note_engine_pick(request.engine, engine, prior_state)
return started
- preflight = preparing_load_token()
-
def _begin_load():
# Under the router transition lock: begin_load on a deactivated engine leaves a resident model nothing can reach.
def _register():
@@ -53972,34 +54017,70 @@ async def clear_gallery_transcripts(current_subject: str = Depends(get_current_s
return {"removed": removed}
-@studio_router.post("/images/unload", response_model = DiffusionStatusResponse)
+@studio_router.post("/images/unload", response_model = DiffusionUnloadResponse)
@account_access.gpu_busy_route
async def unload_diffusion_model(
+ request: Optional[MediaUnloadRequest] = None,
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
):
- account_access.require_resident_control("diffusion")
+ expected_load_id = request.expected_load_id if request is not None else None
+ if not expected_load_id:
+ account_access.require_resident_control("diffusion")
from core.inference.diffusion_engine_router import annotate_status, get_active_diffusion_engine
from core.inference.gpu_arbiter import release_if, DIFFUSION
- if account_access.managed_account():
+ if not expected_load_id and account_access.managed_account():
account_access.require_resident_control(
"diffusion", get_active_diffusion_engine().status().get("repo_id")
)
- from core.inference.chat_swap_kv import cancel_preparing_loads
+ if not expected_load_id:
+ # Load POSTs still in preflight for this account (all, when the unload is not account scoped).
+ # A token unload ends only the load it names, and refuses that one if it has not started.
+ from core.inference.chat_swap_kv import cancel_preparing_loads
+ await asyncio.to_thread(
+ cancel_preparing_loads,
+ "image",
+ current_account_id(),
+ any_account = account_access.account_scope() is None,
+ )
+ else:
+ from core.inference.chat_swap_kv import cancel_preparing_loads
- await asyncio.to_thread(
- cancel_preparing_loads,
- "image",
- current_account_id(),
- any_account = account_access.account_scope() is None,
- )
- unload = get_active_diffusion_engine().unload
+ # The POST carrying this token, if it is still in preflight: stopped, or registered (which the
+ # check below then names), before any engine switch or eviction it would run. The engine is
+ # resolved after that settles: a POST that finished switching registered on the new one.
+ await asyncio.to_thread(
+ cancel_preparing_loads,
+ "image",
+ current_account_id(),
+ load_token = expected_load_id,
+ )
+ active = get_active_diffusion_engine()
+ unload = active.unload
if account_access.account_scope() is not None:
unload = functools.partial(unload, expected_account = current_account_id())
- # Before the teardown waits on the generation slot a launching warm start holds.
- _cancel_upsampler_warm_start()
- status_dict = await asyncio.to_thread(unload)
+ if expected_load_id:
+ from core.inference import media_load_ids
+
+ def guarded_unload(only_if):
+ account_access.require_resident_control("diffusion")
+ if account_access.managed_account():
+ account_access.require_resident_control("diffusion", active.status().get("repo_id"))
+ _cancel_upsampler_warm_start()
+ return unload(only_if = only_if)
+
+ # Only the load that token started, checked and unloaded under the admission lock; any other
+ # load (a newer one, the UI's, another account's) is left alone and not described.
+ outcome, status_dict = await asyncio.to_thread(
+ media_load_ids.unload_if_current, "diffusion", active, expected_load_id, guarded_unload
+ )
+ if outcome != "unloaded":
+ return MediaUnloadOutcome(unloaded = False, outcome = outcome)
+ else:
+ # Before the teardown waits on the generation slot a launching warm start holds.
+ _cancel_upsampler_warm_start()
+ status_dict = await asyncio.to_thread(unload)
# The FLUX.3 prompt upsampler serves the image model: it leaves with it instead of holding VRAM to its keep-alive.
from core.inference import flux3_prompt_upsampler as pup
@@ -54017,12 +54098,17 @@ async def unload_diffusion_model(
# An unload answers with the state it left behind, which still names the model it dropped.
from hub.utils.host_paths import redact_host_paths, restore_inventory_handles
- return redact_host_paths(
+ answer = redact_host_paths(
restore_inventory_handles(
DiffusionStatusResponse(**(await asyncio.to_thread(annotate_status, status_dict)))
),
via_api_key = via_api_key,
)
+ if expected_load_id:
+ return DiffusionLoadUnloadedResponse(
+ **jsonable_encoder(answer), unloaded = True, outcome = "unloaded"
+ )
+ return answer
@studio_router.get("/images/status", response_model = DiffusionStatusResponse)
@@ -54032,10 +54118,17 @@ async def diffusion_status(
):
if account_access.resident_hidden("diffusion"):
return account_access.hidden_resident_response()
- from core.inference.diffusion_engine_router import active_status
+ from core.inference import media_load_ids
+ from core.inference.diffusion_engine_router import annotate_status, get_active_diffusion_engine
from hub.utils.host_paths import redact_host_paths
- status_dict = await asyncio.to_thread(active_status)
+ engine = get_active_diffusion_engine()
+ # What active_status() answers, with the load id of the same state, so an unload can name
+ # exactly what this answer shows.
+ raw_status, load_id, resident_load_id = await asyncio.to_thread(
+ media_load_ids.snapshot_ids, "diffusion", engine, engine.status
+ )
+ status_dict = await asyncio.to_thread(annotate_status, raw_status)
if account_access.resident_hidden("diffusion", status_dict.get("repo_id")):
return account_access.hidden_resident_response()
# Step-skip counters trace a render as it runs, which generate-progress hides from other accounts:
@@ -54053,6 +54146,7 @@ async def diffusion_status(
if status_dict.get("loaded") and status_dict.get("resident_mib") is None:
from core.inference.gpu_residency import media_owner_held_mib
status_dict = {**status_dict, "resident_mib": media_owner_held_mib("diffusion")}
+ status_dict = {**status_dict, "load_id": load_id, "resident_load_id": resident_load_id}
# Answered long after the resolving request ended, so no handle is in context.
return redact_host_paths(DiffusionStatusResponse(**status_dict), via_api_key = via_api_key)
diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.py
index f64478fbca9..ac085202b31 100644
--- a/studio/backend/routes/video.py
+++ b/studio/backend/routes/video.py
@@ -30,6 +30,7 @@ from datetime import datetime, timezone
from typing import Annotated, Any, Literal, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
+from fastapi.encoders import jsonable_encoder
from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
from starlette.datastructures import UploadFile
@@ -59,6 +60,8 @@ from models.inference import (
GalleryProjectRequest,
GalleryProjectResponse,
GalleryVideo,
+ MediaUnloadOutcome,
+ MediaUnloadRequest,
VideoGalleryListResponse,
VideoGenerateProgressResponse,
VideoGenerateRequest,
@@ -70,7 +73,9 @@ from models.inference import (
VideoJobListResponse,
VideoLoadProgressResponse,
VideoLoadRequest,
+ VideoLoadUnloadedResponse,
VideoStatusResponse,
+ VideoUnloadResponse,
)
from utils.account_context import current_account_id, is_owner_context
from utils.api_errors import openai_error_body
@@ -262,6 +267,9 @@ async def video_download_plan(
raise HTTPException(status_code = 409, detail = redact_native_paths(str(exc)))
+_VIDEO_LOAD_CANCELLED = "Video load was cancelled."
+
+
@router.post("/video/load", response_model = VideoStatusResponse)
@account_access.gpu_busy_route
async def load_video_model(
@@ -277,8 +285,11 @@ async def load_video_model(
redact_host_paths,
restore_inventory_handles,
)
+ from core.inference.chat_swap_kv import preparing_load
+
try:
- loaded = await load_video_model_gated(request, current_subject, user_initiated = True)
+ with preparing_load("video", request.load_token):
+ loaded = await load_video_model_gated(request, current_subject, user_initiated = True)
except HTTPException as exc:
# A load that RAISES skips both wrappers below, and the inner loader redacted only
# native paths, which do not know inventory handles.
@@ -360,8 +371,12 @@ async def load_video_model_gated(
)
from utils.native_path_leases import redact_native_paths
+ from core.inference import media_load_ids
+
backend = get_video_backend()
try:
+ # A load an unload already cancelled by its token starts nothing (checked again at admission).
+ media_load_ids.refuse_cancelled("video", request.load_token)
# Resolve the load kind once (gguf / single_file / pipeline) so validation and the load agree; a bad kind raises
# here, so a 400.
kind = resolve_video_model_kind(request.gguf_filename, request.model_kind)
@@ -438,8 +453,9 @@ async def load_video_model_gated(
for _ref in _media_repos_to_record:
_note_load_fetched_with_a_request_token(_ref, request.hf_token)
# Kicks the (slow) load onto a background thread and returns at once; begin_load itself validates
- # network-free.
- return backend.begin_load(
+ # network-free. Recorded with its token, so an unload can name this load.
+ prior = media_load_ids.admitting("video", backend, request.load_token)
+ started = backend.begin_load(
request.model_path,
display_repo_id = request.display_repo_id,
# a load nobody asked for may not reach the hub: the switch verified locality
@@ -464,9 +480,26 @@ async def load_video_model_gated(
gpu_ordinal = gpu_ordinal,
allow_oversized_load = request.allow_oversized_load,
)
+ media_load_ids.admitted("video", backend, request.load_token, prior)
+ return started
+
+ from core.inference.chat_swap_kv import (
+ preparing_load_token,
+ raise_if_load_cancelled,
+ run_unless_load_cancelled,
+ )
+
+ preflight = preparing_load_token()
def _begin_load():
- return account_access.admit_media_load("video", _start_load, request.model_path)
+ # Checked under the preflight lock through admission, so a cancel lands before or after it.
+ return run_unless_load_cancelled(
+ _VIDEO_LOAD_CANCELLED,
+ account_access.admit_media_load,
+ "video",
+ _start_load,
+ request.model_path,
+ )
# begin_load signals whatever generation is running, so guard on every device.
require_no_foreign_generations()
@@ -495,7 +528,10 @@ async def load_video_model_gated(
if multi_model_mode == "always":
raise refusal_for(image_slots, "image")
note_evictions(image_slots, evicted)
- await asyncio.to_thread(images.unload)
+ # A load an unload cancelled never unloads the image model.
+ await asyncio.to_thread(
+ run_unless_load_cancelled, _VIDEO_LOAD_CANCELLED, images.unload
+ )
# sd.cpp runs a paravirtual Mac GPU on the CPU backend: take no VIDEO claim, but keep any existing one until the
# worker tears the old model down and releases it.
forced_cpu = (
@@ -517,20 +553,40 @@ async def load_video_model_gated(
)
def _acquire_and_begin():
+ raise_if_load_cancelled(_VIDEO_LOAD_CANCELLED)
+ admitted = []
+
+ def _admit():
+ # Under the arbiter lock, before anything is evicted; the preflight lock is held
+ # until the load registers, so a cancel never lands between the two.
+ if preflight is None:
+ return
+ preflight.lock.acquire()
+ admitted.append(True)
+ raise_if_load_cancelled(_VIDEO_LOAD_CANCELLED)
+
token = eviction_sink.set(evicted)
try:
with _diffusion_training_admission():
return acquire_for_request(
VIDEO,
guarded_media_begin(VIDEO, multi_model_mode, media_names, _begin_load),
+ admit = _admit,
**fit_kwargs,
)
finally:
+ if admitted:
+ preflight.lock.release()
eviction_sink.reset(token)
from core.inference.chat_swap_kv import acquire_then_close, save_chat_kv_before_swap
swap = await save_chat_kv_before_swap(VIDEO, "video", **fit_kwargs)
+ try:
+ raise_if_load_cancelled(_VIDEO_LOAD_CANCELLED)
+ except BaseException:
+ swap.close()
+ raise
status_dict = await acquire_then_close(swap, _acquire_and_begin)
else:
await asyncio.to_thread(release, VIDEO)
@@ -903,8 +959,13 @@ async def video_status(
from core.inference.video import get_video_backend
from hub.utils.host_paths import redact_host_paths
+ from core.inference import media_load_ids
+
backend = get_video_backend()
- status_dict = await asyncio.to_thread(backend.status)
+ # With the load id of the same state, so an unload can name exactly what this answer shows.
+ status_dict, load_id, resident_load_id = await asyncio.to_thread(
+ media_load_ids.snapshot_ids, "video", backend, backend.status
+ )
if account_access.resident_hidden("video", status_dict.get("repo_id")):
return account_access.hidden_resident_response()
# Step-skip counters trace a clip as it runs, which generate-progress hides from other accounts:
@@ -920,28 +981,59 @@ async def video_status(
if status_dict.get("loaded") and status_dict.get("resident_mib") is None:
from core.inference.gpu_residency import media_owner_held_mib
status_dict = {**status_dict, "resident_mib": media_owner_held_mib("video")}
+ status_dict = {**status_dict, "load_id": load_id, "resident_load_id": resident_load_id}
# This route answers long after the request that resolved the reference ended, so there is
# no handle in context to put back.
return redact_host_paths(VideoStatusResponse(**status_dict), via_api_key = via_api_key)
[email protected]("/video/unload", response_model = VideoStatusResponse)
[email protected]("/video/unload", response_model = VideoUnloadResponse)
@account_access.gpu_busy_route
async def unload_video_model(
+ request: Optional[MediaUnloadRequest] = None,
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
):
- account_access.require_resident_control("video")
from core.inference.gpu_arbiter import VIDEO, release_if
from core.inference.video import get_video_backend
+ expected_load_id = request.expected_load_id if request is not None else None
+ if not expected_load_id:
+ account_access.require_resident_control("video")
backend = get_video_backend()
- if account_access.managed_account():
+ if not expected_load_id and account_access.managed_account():
account_access.require_resident_control("video", backend.status().get("repo_id"))
unload_kwargs = {}
if account_access.account_scope() is not None:
unload_kwargs["expected_account"] = current_account_id()
- status_dict = await asyncio.to_thread(backend.unload, **unload_kwargs)
+ if expected_load_id:
+ from core.inference import media_load_ids
+ from core.inference.chat_swap_kv import cancel_preparing_loads
+
+ # The POST carrying this token, if it is still in preflight: stopped (or registered, which
+ # the check below then names) before any eviction it would run.
+ await asyncio.to_thread(
+ cancel_preparing_loads,
+ "video",
+ current_account_id(),
+ load_token = expected_load_id,
+ )
+
+ def unload(only_if):
+ account_access.require_resident_control("video")
+ if account_access.managed_account():
+ account_access.require_resident_control("video", backend.status().get("repo_id"))
+ return backend.unload(**unload_kwargs, only_if = only_if)
+
+ # Only the load that token started, checked and unloaded under the admission lock; any other
+ # load (a newer one, the UI's, another account's) is left alone and not described.
+ outcome, status_dict = await asyncio.to_thread(
+ media_load_ids.unload_if_current, "video", backend, expected_load_id, unload
+ )
+ if outcome != "unloaded":
+ return MediaUnloadOutcome(unloaded = False, outcome = outcome)
+ else:
+ status_dict = await asyncio.to_thread(backend.unload, **unload_kwargs)
# Drop VIDEO ownership only if nothing is resident AND no load is in flight; the check and release must be ATOMIC
# (release_if). Mirrors images.
await asyncio.to_thread(
@@ -952,10 +1044,15 @@ async def unload_video_model(
# An unload answers with the state it left behind, which still names the model it dropped.
from hub.utils.host_paths import redact_host_paths, restore_inventory_handles
- return redact_host_paths(
+ answer = redact_host_paths(
restore_inventory_handles(VideoStatusResponse(**status_dict)),
via_api_key = via_api_key,
)
+ if expected_load_id:
+ return VideoLoadUnloadedResponse(
+ **jsonable_encoder(answer), unloaded = True, outcome = "unloaded"
+ )
+ return answer
@router.get("/video/gallery", response_model = VideoGalleryListResponse)
diff --git a/studio/backend/studio_mcp/loading.py b/studio/backend/studio_mcp/loading.py
index bda68236759..056c137b6a2 100644
--- a/studio/backend/studio_mcp/loading.py
+++ b/studio/backend/studio_mcp/loading.py
@@ -8,8 +8,10 @@ from __future__ import annotations
import asyncio
import dataclasses
import hashlib
+import itertools
import os
import time
+import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Awaitable, Callable, Optional
@@ -20,7 +22,8 @@ from fastmcp.exceptions import ToolError
from studio_mcp import job_store
from studio_mcp import placement as place
from studio_mcp.caller import Caller
-from studio_mcp.errors import tool_error
+from studio_mcp.errors import raise_for_route, tool_error
+from studio_mcp.forward import forward
from studio_mcp.outputs import EvictedSlot, LoadResult, SlotModel, UnloadAllResult, UnloadResult
from studio_mcp.tools import as_dict, number, opt_text, route_json, strings, try_json
@@ -79,6 +82,8 @@ class LoadJob(job_store.Job):
posted: bool = False
# A media load POST, shielded: cancelling the job leaves it running, since the route registers its load from a thread.
post: Optional[asyncio.Future] = field(default = None, repr = False)
+ # The media load's load_token: an unload naming it ends this load only, even if its POST is still out.
+ load_token: Optional[str] = None
def note(self, fraction: Optional[float], phase: str) -> None:
# A step without a fraction keeps the last one, so progress never drops back.
@@ -653,15 +658,24 @@ async def load_media(
async def run(job: LoadJob, before: dict[str, list[str]]) -> LoadResult:
await prepare()
+ # An unload this process sent for the slot could cancel this load if it ran after it: post after it.
+ while unloading := {u for u in _MEDIA_UNLOADING.get(kind, ()) if not u.done()}:
+ await asyncio.wait(unloading)
+ # Named before it goes out, so an unload can end this load even if the POST's answer never comes.
+ job.load_token = uuid.uuid4().hex
+ _remember_media_token(caller, kind, job.load_token)
# The answer names what the arbiter unloaded to make room; the load itself goes on in the background.
job.post = asyncio.ensure_future(
- route_json("POST", routes["load"], caller = caller, json_body = {**body, **knobs})
+ _post_media_load(
+ caller, kind, job.load_token, {**body, **knobs, "load_token": job.load_token}
+ )
)
job.post.add_done_callback(lambda post: post.cancelled() or post.exception())
answer = await with_progress(
None, asyncio.shield(job.post), _media_poll(caller, routes), job
)
status = await _media_ready(caller, kind, model, routes, job, body)
+ _media_token_ready(caller, kind, job.load_token)
name = _media_name(status) or model
return await finish_load(
caller, kind, name, before, answer, placement = place.media_placement(status)
@@ -730,26 +744,180 @@ async def _media_ready(
async def unload_media(caller: Caller, kind: str) -> UnloadResult:
+ # Raised before anything is read and held until the unload has answered: a load of the slot this
+ # process starts meanwhile posts after it, so the status read below never names that load.
+ barrier = asyncio.get_running_loop().create_future()
+ unloading = _MEDIA_UNLOADING.setdefault(kind, set())
+ unloading.add(barrier)
+ try:
+ return await _unload_media(caller, kind)
+ finally:
+ unloading.discard(barrier)
+ barrier.set_result(None)
+
+
+async def _unload_media(caller: Caller, kind: str) -> UnloadResult:
routes = MEDIA_ROUTES[kind]
- # A load_model still loading runs on in the job store; the unload route cancels its load too.
+ # A load_model still loading runs on in the job store; its load is ended below by its token.
stopped = _stop_loads(caller, (kind,))
- # A stopped load whose POST is still out registers its load once that returns: unload after it, so the unload cancels it.
- posts = {job.post for job in stopped if job.post is not None and not job.post.done()}
- if posts:
- await asyncio.wait(posts, timeout = MAX_WAIT_S)
- before = as_dict(await route_json("GET", routes["status"], caller = caller))
- if before.get("loaded") is not True and not stopped:
- return UnloadResult(kind = kind, unloaded = False)
- after = await route_json("POST", routes["unload"], caller = caller)
+ # Each media load this caller posted ends by its load_token: the backend unloads it only while it is
+ # the load in flight or resident, and refuses it on arrival if its POST has not reached it yet, so
+ # this never waits on a POST and never ends a newer load (this caller's, the UI's or another account's).
+ tokens = list(_MEDIA_TOKENS.get(_media_key(caller, kind), ()))
+ tokens += [job.load_token for job in stopped if job.load_token and job.load_token not in tokens]
+ # What the slot held when this call came, read before any unload goes out: the unload of the
+ # resident below names that load, so one admitted later is never touched.
+ status_read = asyncio.ensure_future(
+ asyncio.wait_for(route_json("GET", routes["status"], caller = caller), READ_S)
+ )
+ try:
+ await asyncio.wait({status_read})
+ finally:
+ # Even if this call is cancelled meanwhile, so a load it stopped still ends.
+ ended = [_unload_media_token(caller, kind, token) for token in tokens]
+ outcomes = await asyncio.shield(asyncio.gather(*ended)) if ended else []
+ try:
+ before = as_dict(status_read.result())
+ except asyncio.TimeoutError:
+ raise ToolError(
+ f"Unsloth Studio did not answer the {kind} status read within {READ_S:.0f} s. Loads this "
+ f"call stopped were ended, but a resident {kind} model may still be loaded; try unload_model again."
+ ) from None
model = (
opt_text(before.get("display_repo_id"))
or opt_text(before.get("repo_id"))
or (stopped[-1].model if stopped else None)
)
- unloaded = isinstance(after, dict) and after.get("loaded") is not True
+ unloaded = "unloaded" in outcomes
+ if before.get("loaded") is not True:
+ return UnloadResult(kind = kind, model = model if unloaded else None, unloaded = unloaded)
+ resident = opt_text(before.get("load_id"))
+ if resident is None and "load_id" in before:
+ # Studio reports load ids but could not name this state (admissions or unloads kept landing
+ # while it read): an unconditional unload could end a load admitted since, so none is sent.
+ if not unloaded:
+ raise ToolError(
+ f"Unsloth Studio could not identify the loaded {kind} model while loads were changing; "
+ "nothing was unloaded. Try unload_model again."
+ )
+ elif resident is None:
+ # No load id at all: another account's model (Studio hides its id) or a Studio without load
+ # tokens, which refuses or unloads it for this caller just as before.
+ after = await _media_unload_post(caller, kind)
+ unloaded = unloaded or (isinstance(after, dict) and after.get("loaded") is not True)
+ elif resident not in tokens:
+ answer = as_dict(await _media_unload_post(caller, kind, {"expected_load_id": resident}))
+ unloaded = unloaded or answer.get("outcome") == "unloaded"
+ # load_id names a load in flight; the model it was replacing is resident_load_id. A replacement
+ # that failed before its unload above left that model loaded, so end it too, by its own id
+ # (a load admitted since is never it), also when it is this caller's own token: its unload above
+ # answered superseded while the replacement was still in flight. Studio without the field:
+ # nothing more to name.
+ held = opt_text(before.get("resident_load_id"))
+ if held is not None and held != resident:
+ answer = as_dict(await _media_unload_post(caller, kind, {"expected_load_id": held}))
+ unloaded = unloaded or answer.get("outcome") == "unloaded"
return UnloadResult(kind = kind, model = model, unloaded = unloaded)
+@dataclass
+class _MediaToken:
+ # Order numbers (from _MEDIA_ORDER): when its load POST went out, and when it answered 2xx.
+ sent: int
+ answered: Optional[int] = None
+
+
+# Media load tokens this process posted, per (account, slot), whose load may still be in flight or
+# resident. Each leaves only on a definitive outcome: a token unload's answer, a 4xx refusal of its
+# POST (no load registered), or a newer load of the slot shown ready after its POST had answered.
+_MEDIA_TOKENS: dict[tuple[str, str], dict[str, _MediaToken]] = {}
+_MEDIA_ORDER = itertools.count(1)
+# Unload POSTs this process sent per slot and not answered yet: a load posts only after them.
+_MEDIA_UNLOADING: dict[str, set[asyncio.Future]] = {}
+# Tries a token unload makes on route errors before leaving the token for the next unload_model.
+_TOKEN_UNLOAD_ATTEMPTS = 5
+
+
+def _media_key(caller: Caller, kind: str) -> tuple[str, str]:
+ return (caller.account_id, kind)
+
+
+def _remember_media_token(caller: Caller, kind: str, token: str) -> None:
+ _MEDIA_TOKENS.setdefault(_media_key(caller, kind), {})[token] = _MediaToken(next(_MEDIA_ORDER))
+
+
+async def _post_media_load(caller: Caller, kind: str, token: str, body: dict) -> Any:
+ response = await forward(caller, "POST", MEDIA_ROUTES[kind]["load"], json_body = body)
+ if 400 <= response.status_code < 500:
+ # Refused before it was admitted (busy, invalid, a load already running): nothing to end.
+ _forget_media_token(caller, kind, token)
+ elif response.status_code < 400:
+ state = _MEDIA_TOKENS.get(_media_key(caller, kind), {}).get(token)
+ if state is not None:
+ state.answered = next(_MEDIA_ORDER)
+ return raise_for_route(response)
+
+
+def _media_token_ready(caller: Caller, kind: str, token: Optional[str]) -> None:
+ # ``token``'s load is shown ready: one whose POST answered before ``token``'s went out was admitted
+ # before it, so it is over (Studio admits no load while another is in flight).
+ tokens = _MEDIA_TOKENS.get(_media_key(caller, kind), {})
+ newer = tokens.get(token) if token else None
+ if newer is None:
+ return
+ for old, state in list(tokens.items()):
+ if state.answered is not None and state.answered < newer.sent:
+ del tokens[old]
+
+
+def _forget_media_token(caller: Caller, kind: str, token: str) -> None:
+ key = _media_key(caller, kind)
+ tokens = _MEDIA_TOKENS.get(key, {})
+ tokens.pop(token, None)
+ if not tokens:
+ _MEDIA_TOKENS.pop(key, None)
+
+
+async def _media_unload_post(
+ caller: Caller,
+ kind: str,
+ json_body: Optional[dict] = None,
+) -> Any:
+ # Tracked while out, so a load of the slot posts after it rather than racing it.
+ post = asyncio.ensure_future(
+ route_json("POST", MEDIA_ROUTES[kind]["unload"], caller = caller, json_body = json_body)
+ )
+ unloading = _MEDIA_UNLOADING.setdefault(kind, set())
+ unloading.add(post)
+ post.add_done_callback(unloading.discard)
+ return await asyncio.shield(post)
+
+
+def _unload_media_token(caller: Caller, kind: str, token: str) -> asyncio.Task:
+ """End the load ``token`` names, in the background (held in _LATE_UNLOADS); the task answers its outcome."""
+
+ async def unload() -> Optional[str]:
+ for failures in range(1, _TOKEN_UNLOAD_ATTEMPTS + 1):
+ try:
+ answer = as_dict(
+ await _media_unload_post(caller, kind, {"expected_load_id": token})
+ )
+ except Exception:
+ # A transient route error: try again; past the last try the token stays for the next unload.
+ if failures < _TOKEN_UNLOAD_ATTEMPTS:
+ await asyncio.sleep(POLL_INTERVAL_S * failures)
+ continue
+ # unloaded, superseded or cancelled: in each case that load is over, or will never start.
+ _forget_media_token(caller, kind, token)
+ return opt_text(answer.get("outcome"))
+ return None
+
+ task = asyncio.ensure_future(unload())
+ _LATE_UNLOADS.add(task)
+ task.add_done_callback(_LATE_UNLOADS.discard)
+ return task
+
+
STT_STATUS = "/api/inference/audio/stt/status"
# Transformers first: it is Studio's default and serves custom Whisper repos no list names.
STT_ENGINES = ("transformers", "mtmd", "audiocpp", "gguf")
@@ -880,7 +1048,7 @@ async def load_stt(
return LoadResult(kind = "stt", model = resident)
-# Unloads waiting on an STT load that outlasted the unload call, held so they are not collected.
+# Unloads waiting on an STT load that outlasted the unload call, and media token unloads, held so they are not collected.
_LATE_UNLOADS: set[asyncio.Task] = set()
diff --git a/studio/backend/studio_mcp/tools/models.py b/studio/backend/studio_mcp/tools/models.py
index 2a0b646331e..6a4940a33ad 100644
--- a/studio/backend/studio_mcp/tools/models.py
+++ b/studio/backend/studio_mcp/tools/models.py
@@ -469,7 +469,7 @@ async def load_model(
async def unload_model(kind: UnloadKind = "llm", model: Optional[str] = None) -> ToolResult:
"""Unload a model to free memory.
Without ``model`` the active one of that kind is unloaded; image and video have one slot each, so ``model`` is ignored there.
- For "image", "video" and "stt", a load_model of that kind still running is stopped too (a speech-to-text load past its download is unloaded once it finishes, after this call if it takes longer than 240 s).
+ For "image", "video" and "stt", a load_model of that kind still running is stopped too (a speech-to-text load past its download is unloaded once it finishes, after this call if it takes longer than 240 s; an image or video load is ended by its load id, even one whose request has not reached Unsloth Studio yet, and never a newer load of the slot).
kind "all" unloads every chat, speech, image, video and speech-to-text model in one call, stopping image, video and speech-to-text loads as those kinds do, and lists them under ``unloaded``.
The embedding model is managed by Unsloth Studio and is not unloaded here.
``unloaded`` is false when nothing matching was loaded."""
diff --git a/studio/backend/tests/mcp_harness.py b/studio/backend/tests/mcp_harness.py
index 32351d172b9..e82b70b6943 100644
--- a/studio/backend/tests/mcp_harness.py
+++ b/studio/backend/tests/mcp_harness.py
@@ -249,6 +249,15 @@ def bodies(studio: FastAPI, path: str) -> list[Any]:
return [json.loads(c[3]) for c in studio.state.calls if c[1] == path]
+def media_load_bodies(studio: FastAPI, path: str) -> list[Any]:
+ """The bodies of an image or video load, each checked to name its load and shown without that id."""
+ sent = bodies(studio, path)
+ for body in sent:
+ token = body.pop("load_token", None)
+ assert isinstance(token, str) and len(token) == 32, body
+ return sent
+
+
def queries(studio: FastAPI, path: str) -> list[dict[str, str]]:
return [
dict(parse_qsl(q, keep_blank_values = True)) for _m, p, q in studio.state.queries if p == path
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index 67957236b48..2d505c250c6 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -3367,6 +3367,139 @@ def test_a_retired_flux3_full_copy_reaches_the_backend_as_its_base(client):
assert validated and set(validated) == {(None, "pipeline")}, seen
+def token_client(client, monkeypatch):
+ """The client, with begin_load publishing a state and moving the load epoch as the real engines do."""
+ from core.inference import media_load_ids
+
+ media_load_ids._reset_for_tests()
+ backend = diffusion_module.get_diffusion_backend()
+ begin, unload = backend.begin_load, backend.unload
+ backend._load_token, backend._state, backend.unloads = 0, None, 0
+
+ def begin_load(model_path, **kwargs):
+ backend._load_token += 1
+ backend._state = object()
+ return begin(model_path, **kwargs)
+
+ def counted_unload(**kwargs):
+ only_if = kwargs.get("only_if")
+ if only_if is not None and not only_if():
+ return None
+ backend.unloads += 1
+ backend._load_token += 1
+ backend._state = None
+ return unload()
+
+ monkeypatch.setattr(backend, "begin_load", begin_load)
+ monkeypatch.setattr(backend, "unload", counted_unload)
+ yield client, backend
+ media_load_ids._reset_for_tests()
+
+
+def _load_with_token(client, token):
+ body = {
+ "model_path": "unsloth/Z-Image-Turbo-GGUF",
+ "gguf_filename": "z-image-turbo-Q4_K_S.gguf",
+ }
+ return _post_load(client, **body, **({"load_token": token} if token else {}))
+
+
+def test_an_image_unload_naming_the_load_token_unloads_that_load(token_client):
+ client, backend = token_client
+ assert _load_with_token(client, "t1").status_code == 200
+ resp = client.post("/api/inference/images/unload", json = {"expected_load_id": "t1"})
+ assert resp.status_code == 200
+ assert (resp.json()["outcome"], resp.json()["loaded"]) == ("unloaded", False)
+ assert backend.unloads == 1
+
+
[email protected]("newer", [None, "t2"])
+def test_an_image_unload_naming_a_stale_token_leaves_the_newer_load_alone(token_client, newer):
+ client, backend = token_client
+ assert _load_with_token(client, "t1").status_code == 200
+ assert _load_with_token(client, newer).status_code == 200
+ resp = client.post("/api/inference/images/unload", json = {"expected_load_id": "t1"})
+ assert resp.json() == {"unloaded": False, "outcome": "superseded"}
+ assert backend.unloads == 0 and backend.loaded
+
+
+def test_an_image_unload_naming_a_load_not_started_yet_refuses_that_load(token_client):
+ client, backend = token_client
+ resp = client.post("/api/inference/images/unload", json = {"expected_load_id": "t1"})
+ assert resp.json() == {"unloaded": False, "outcome": "cancelled"}
+ refused = _load_with_token(client, "t1")
+ assert refused.status_code == 409 and "cancelled" in refused.json()["detail"]
+ assert backend.unloads == 0 and not backend.loaded
+
+
+def test_image_status_names_the_load_an_unload_can_name(token_client):
+ client, backend = token_client
+ assert _load_with_token(client, "t1").status_code == 200
+ assert client.get("/api/inference/images/status").json()["load_id"] == "t1"
+
+
+def test_image_status_and_load_id_name_the_same_load(token_client):
+ import threading
+
+ from core.inference import media_load_ids
+ from hub.services.models import account_access
+
+ client, backend = token_client
+ assert _load_with_token(client, "t1").status_code == 200
+ read = backend.status
+ admitted = threading.Event()
+
+ def newer_load():
+ with account_access.media_load_lock:
+ prior = media_load_ids.admitting("diffusion", backend, "t2")
+ backend._load_token += 1
+ backend.loading = ("unsloth/Z-Image-Turbo-GGUF",)
+ media_load_ids.admitted("diffusion", backend, "t2", prior)
+ admitted.set()
+
+ reads = []
+
+ def racing_status():
+ # The first read is slow and a load is admitted meanwhile, which it does not hold up.
+ reads.append(True)
+ if len(reads) == 1:
+ worker = threading.Thread(target = newer_load)
+ worker.start()
+ worker.join(5)
+ assert not worker.is_alive()
+ return read()
+
+ backend.status = racing_status
+ try:
+ answer = client.get("/api/inference/images/status").json()
+ finally:
+ backend.status = read
+ assert admitted.wait(5)
+ # The read that straddled the admission was read again, so the id is that of the state it shows.
+ assert answer["load_id"] == "t2" and len(reads) == 2
+ backend.loading = ()
+
+
+def test_the_image_unload_answers_are_declared_in_the_schema(token_client):
+ import json
+
+ client, backend = token_client
+ schema = client.app.openapi()
+ answer = schema["paths"]["/api/inference/images/unload"]["post"]["responses"]["200"]
+ refs = json.dumps(answer)
+ for model in ("DiffusionLoadUnloadedResponse", "MediaUnloadOutcome", "DiffusionStatusResponse"):
+ assert f"#/components/schemas/{model}" in refs
+ assert _load_with_token(client, "t1").status_code == 200
+ plain = client.post("/api/inference/images/unload").json()
+ assert set(plain) == set(inference_routes.DiffusionStatusResponse.model_fields)
+ assert _load_with_token(client, "t2").status_code == 200
+ named = client.post("/api/inference/images/unload", json = {"expected_load_id": "t2"}).json()
+ assert set(named) == set(plain) | {"unloaded", "outcome"}
+ stale = client.post("/api/inference/images/unload", json = {"expected_load_id": "t2"}).json()
+ assert stale == {"unloaded": False, "outcome": "superseded"}
+
+
def test_zero_weight_loras_neither_refuse_mlx_nor_route_away_from_it():
# Weight 0 disables an adapter and every loader drops it, so an all-zero list is no LoRA load.
from fastapi import HTTPException
diff --git a/studio/backend/tests/test_mcp_tools_loading.py b/studio/backend/tests/test_mcp_tools_loading.py
index e72047549cc..e7a16bf95fe 100644
--- a/studio/backend/tests/test_mcp_tools_loading.py
+++ b/studio/backend/tests/test_mcp_tools_loading.py
@@ -7,6 +7,7 @@ import time
import pytest
from fastapi.responses import JSONResponse, Response
+from fastmcp.exceptions import ToolError
from studio_mcp import loading
@@ -14,6 +15,7 @@ from .mcp_harness import (
DEFERRED,
WHISPER,
bodies,
+ media_load_bodies,
call_to,
call_with_progress,
fake_studio,
@@ -61,8 +63,10 @@ PAYLOADS = {
def _no_loads_left():
# A job left running by one test would be followed by the next one loading the same model.
loading.LOADS.reset()
+ getattr(loading, "_MEDIA_TOKENS", {}).clear()
yield
loading.LOADS.reset()
+ getattr(loading, "_MEDIA_TOKENS", {}).clear()
# tts / audio load through the LLM route but report the kind asked for. STATUS serves LLM
@@ -248,7 +252,7 @@ def test_a_gguf_image_repo_sends_its_gguf_filename(monkeypatch, fast_polls, extr
expected = {"model_path": FLUX_GGUF, "gguf_filename": filename, "model_kind": "gguf"}
expected.update({"hf_token": extra["hf_token"]} if "hf_token" in extra else {})
assert bodies(studio, "/api/inference/images/download-plan") == [expected]
- assert bodies(studio, "/api/inference/images/load") == [expected]
+ assert media_load_bodies(studio, "/api/inference/images/load") == [expected]
lookup_token = call_to(studio, "/api/hub/gguf-variants")[2].get("x-unsloth-hf-token")
assert lookup_token == extra.get("hf_token")
@@ -258,7 +262,7 @@ def test_a_plan_entry_supplies_the_checkpoint_file(monkeypatch, fast_polls):
plan = {"entries": [{**entry, "gguf_filename": "model.safetensors"}]}
studio = _media_studio("image", statuses = [{"loaded": True, "repo_id": "a/b"}], plan = plan)
run_tool(monkeypatch, studio, "load_model", {"model": "a/b", "kind": "image"})
- assert bodies(studio, "/api/inference/images/load") == [
+ assert media_load_bodies(studio, "/api/inference/images/load") == [
{"model_path": "a/b", "gguf_filename": "model.safetensors"}
]
assert [c for c in studio.state.calls if c[1] == "/api/hub/gguf-variants"] == []
@@ -308,7 +312,7 @@ def test_a_media_load_failure_is_a_tool_error(
assert result["isError"] is True
assert message in result["content"][0]["text"]
if "variant" in args:
- assert bodies(studio, "/api/inference/images/load") == []
+ assert media_load_bodies(studio, "/api/inference/images/load") == []
def test_a_previous_model_answer_then_the_new_one_loads(monkeypatch, fast_polls):
@@ -402,7 +406,7 @@ def test_video_progress_uses_its_own_field_names(monkeypatch, fast_polls):
assert [p["progress"] for p in progress] == [0.25, 1.0]
assert progress[0]["message"] == "Downloading"
assert result["structuredContent"]["model"] == LTX
- assert bodies(studio, "/api/inference/video/load") == [{"model_path": LTX}]
+ assert media_load_bodies(studio, "/api/inference/video/load") == [{"model_path": LTX}]
@pytest.mark.parametrize(
@@ -828,19 +832,139 @@ def test_unloading_stt_stops_a_load_still_downloading(monkeypatch, fast_polls):
assert bodies(studio, f"{STT}/load") == []
-def test_unloading_media_stops_a_load_still_running(monkeypatch, fast_polls):
- # Nothing is resident yet, so before this the unload answered "nothing loaded" and posted nothing.
- unload = "/api/inference/video/unload"
- studio = fake_studio(
- {
- ("POST", "/api/inference/video/download-plan"): {"entries": []},
- ("POST", "/api/inference/video/load"): slow(LTX_LOADED, 0.5),
- ("GET", "/api/inference/video/load-progress"): READY,
- VIDEO_STATUS: UNLOADED,
- IMAGE_STATUS: UNLOADED,
- ("POST", unload): UNLOADED,
- }
+class _TokenVideo:
+ """The video routes, keeping the load-token contract of core/inference/media_load_ids.py.
+
+ A load registers ``admit_after`` seconds into its POST (the arbiter freeing the GPU) and stays in
+ flight until ``finish()``. An unload naming a token ends that load only while it is the last one
+ admitted, answers ``superseded`` for one admitted earlier, and refuses one not admitted yet. A
+ load sent without a token gets ``studio-N``; status names the last admitted load as ``load_id``.
+ A load posted while another is in flight is refused with a 409, as Studio does.
+ """
+
+ def __init__(
+ self,
+ *,
+ admit_after: float = 0.0,
+ unload_s: float = 0.0,
+ answer_status: int = 200,
+ ):
+ self.admit_after, self.unload_s, self.answer_status = admit_after, unload_s, answer_status
+ self.current = None # the last admitted load: [model, token, phase]
+ self.resident = None
+ self.seen: set = set()
+ self.cancelled: set = set()
+ self.events: list = []
+ # Called while an unload is out, before it answers (another client acting meanwhile).
+ self.during_unload = None
+ self.stall_status = False
+ self.status_s = 0.0
+ self.made = 0
+
+ def admit(self, model, token) -> bool:
+ if token in self.cancelled:
+ return False
+ if token is None:
+ self.made += 1
+ token = f"studio-{self.made}"
+ self.current = [model, token, "loading"]
+ self.seen.add(token)
+ self.events.append(("admitted", token))
+ return True
+
+ def finish(self) -> None:
+ self.current[2] = "ready"
+ self.resident = self.current[0]
+
+ async def load(self, request, body):
+ sent = json.loads(body)
+ await asyncio.sleep(self.admit_after)
+ if self.current is not None and self.current[2] == "loading":
+ self.events.append(("busy", sent.get("load_token")))
+ return JSONResponse({"detail": "A video load is already in progress."}, status_code = 409)
+ if not self.admit(sent["model_path"], sent.get("load_token")):
+ self.events.append(("refused", sent.get("load_token")))
+ return JSONResponse(
+ {"detail": "This load was cancelled by an unload before it started."},
+ status_code = 409,
+ )
+ if self.answer_status != 200:
+ return JSONResponse({"detail": "lost"}, status_code = self.answer_status)
+ return UNLOADED
+
+ def progress(self, request, body):
+ if self.current is None:
+ return {"phase": None}
+ return {"phase": self.current[2]}
+
+ async def status(self, request, body):
+ if self.stall_status:
+ await asyncio.sleep(30)
+ delay, self.status_s = self.status_s, 0.0
+ await asyncio.sleep(delay)
+ if not self.resident:
+ return UNLOADED
+ return {"loaded": True, "repo_id": self.resident, "load_id": self.current[1]}
+
+ async def unload(self, request, body):
+ expected = json.loads(body).get("expected_load_id") if body else None
+ self.events.append(("unload", expected))
+ await asyncio.sleep(self.unload_s)
+ if self.during_unload is not None:
+ self.during_unload()
+ self.events.append(("unload answered", expected))
+ if expected is None or (self.current is not None and self.current[1] == expected):
+ self.current = self.resident = None
+ return {**UNLOADED, **({"unloaded": True, "outcome": "unloaded"} if expected else {})}
+ if expected in self.seen:
+ return {"unloaded": False, "outcome": "superseded"}
+ self.cancelled.add(expected)
+ return {"unloaded": False, "outcome": "cancelled"}
+
+ def app(self):
+ return fake_studio(
+ {
+ ("POST", "/api/inference/video/download-plan"): {"entries": []},
+ ("POST", "/api/inference/video/load"): self.load,
+ ("GET", "/api/inference/video/load-progress"): self.progress,
+ VIDEO_STATUS: self.status,
+ IMAGE_STATUS: UNLOADED,
+ ("POST", "/api/inference/video/unload"): self.unload,
+ }
+ )
+
+
+async def _video_load(caller):
+ # Returns at once (no wait), leaving the load running in the job store, once its POST is out.
+ result = await loading.load_media(
+ caller,
+ None,
+ kind = "video",
+ model = LTX,
+ variant = None,
+ hf_token = None,
+ deadline = time.monotonic(),
)
+ job = [job for job in loading.LOADS.jobs_of(caller.account_id) if job.slot == "video"][-1]
+ while job.post is None and not job.task.done():
+ await asyncio.sleep(0.01)
+ return result
+
+
+async def _late_unloads_done():
+ loop = asyncio.get_running_loop()
+ while held := {t for t in loading._LATE_UNLOADS if t.get_loop() is loop and not t.done()}:
+ await asyncio.wait(held, timeout = 5)
+
+
+def _tokens_sent(backend):
+ return [token for event, token in backend.events if event == "admitted" and token]
+
+
+def test_unloading_media_stops_a_load_still_running(monkeypatch, fast_polls):
+ # Nothing is resident yet: the unload names the load, which ends it in flight.
+ backend = _TokenVideo()
+ studio = backend.app()
with session(monkeypatch, studio) as call:
early = call("load_model", {"model": LTX, "kind": "video", "wait_s": 0})
job_id = early["structuredContent"]["job_id"]
@@ -848,46 +972,286 @@ def test_unloading_media_stops_a_load_still_running(monkeypatch, fast_polls):
job = call("get_job", {"kind": "load", "id": job_id})
assert unloaded == {"kind": "video", "model": LTX, "unloaded": True}
assert job["structuredContent"]["status"] == "cancelled"
- assert len([c for c in studio.state.calls if c[1] == unload]) == 1
+ (token,) = _tokens_sent(backend)
+ assert bodies(studio, "/api/inference/video/unload") == [{"expected_load_id": token}]
+ assert backend.current is None
+
+
+def test_a_media_load_post_that_outlasts_the_unload_never_registers(monkeypatch, fast_polls):
+ # The unload reaches Studio before the load POST registers its load: the load is refused on
+ # arrival, and the unload never waits for the POST.
+ backend = _TokenVideo(admit_after = 0.6)
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ result = await _video_load(caller)
+ began = time.monotonic()
+ unloaded = await loading.unload_media(caller, "video")
+ took = time.monotonic() - began
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ await _late_unloads_done()
+ return result, unloaded, took
+
+ result, unloaded, took = asyncio.run(scenario())
+ assert result.status == "loading"
+ assert took < 0.5
+ assert unloaded.unloaded is False
+ assert [e for e, _t in backend.events] == ["unload", "unload answered", "refused"]
+ assert backend.current is None and backend.resident is None
+
+
+def test_a_stopped_load_never_cancels_a_newer_load_from_the_ui(monkeypatch, fast_polls):
+ backend = _TokenVideo(admit_after = 0.6)
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ await _video_load(caller)
+ await loading.unload_media(caller, "video")
+ backend.admit(LTX, None)
+ backend.finish()
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ await _late_unloads_done()
+ asyncio.run(scenario())
+ assert backend.events[-1][0] == "refused"
+ assert backend.current == [LTX, "studio-1", "ready"] and backend.resident == LTX
-def test_unloading_media_waits_for_a_load_post_still_freeing_the_gpu(monkeypatch, fast_polls):
- # The route registers its load from a thread once the arbiter freed the GPU; cancelling the
- # awaiting job does not stop that thread, so an unload posted earlier finds nothing to cancel.
- unload = "/api/inference/video/unload"
- registered = []
- seen_at_unload = []
- def handoff() -> None:
- time.sleep(0.5)
- registered.append(True)
+def test_a_cancelled_media_unload_still_ends_the_load_it_stopped(monkeypatch, fast_polls):
+ backend = _TokenVideo(unload_s = 0.4)
+ caller = make_caller(studio_app = backend.app())
- async def load(request, body):
- await asyncio.to_thread(handoff)
- return LTX_LOADED
+ async def scenario():
+ await _video_load(caller)
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ unload = asyncio.ensure_future(loading.unload_media(caller, "video"))
+ await asyncio.sleep(0.1)
+ unload.cancel()
+ await asyncio.wait({unload}, timeout = 5)
+ await _late_unloads_done()
- def unload_route(request, body):
- seen_at_unload.append(bool(registered))
- return UNLOADED
+ asyncio.run(scenario())
+ (token,) = _tokens_sent(backend)
+ assert ("unload answered", token) in backend.events
+ assert backend.current is None
- studio = fake_studio(
- {
- ("POST", "/api/inference/video/download-plan"): {"entries": []},
- ("POST", "/api/inference/video/load"): load,
- ("GET", "/api/inference/video/load-progress"): READY,
- VIDEO_STATUS: UNLOADED,
- IMAGE_STATUS: UNLOADED,
- ("POST", unload): unload_route,
- }
- )
+
+def test_a_lost_load_answer_still_lets_the_token_end_the_load(monkeypatch, fast_polls):
+ # The load registered but its answer failed: the job ended, the load runs on, and a later
+ # unload still ends it by its token (nothing is resident yet, so status alone would not).
+ backend = _TokenVideo(answer_status = 500)
+ studio = backend.app()
with session(monkeypatch, studio) as call:
- early = call("load_model", {"model": LTX, "kind": "video", "wait_s": 0})
- job_id = early["structuredContent"]["job_id"]
+ failed = call("load_model", {"model": LTX, "kind": "video", "wait_s": 30})
+ assert failed["isError"] is True
+ assert backend.current is not None
unloaded = call("unload_model", {"kind": "video"})["structuredContent"]
- job = call("get_job", {"kind": "load", "id": job_id})
assert unloaded["unloaded"] is True
- assert job["structuredContent"]["status"] == "cancelled"
- assert seen_at_unload == [True]
+ (token,) = _tokens_sent(backend)
+ assert bodies(studio, "/api/inference/video/unload") == [{"expected_load_id": token}]
+ assert backend.current is None
+ assert loading._MEDIA_TOKENS == {}
+
+
+def test_a_superseded_token_leaves_the_newer_load_alone(monkeypatch, fast_polls):
+ # This caller's load finished, then the UI started another; the old token ends nothing.
+ backend = _TokenVideo()
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ await _video_load(caller)
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ backend.finish()
+ await asyncio.wait({job.task}, timeout = 5)
+ backend.resident = None
+ backend.admit(LTX, None)
+ return await loading.unload_media(caller, "video")
+
+ unloaded = asyncio.run(scenario())
+ assert unloaded.unloaded is False
+ assert backend.events[-1] == ("unload answered", _tokens_sent(backend)[0])
+ assert backend.current == [LTX, "studio-1", "loading"]
+
+
+def test_another_accounts_unload_sends_none_of_this_accounts_tokens(monkeypatch, fast_polls):
+ backend = _TokenVideo()
+ studio = backend.app()
+ mine = make_caller(studio_app = studio)
+ theirs = make_caller(studio_app = studio, account_id = "acct-b")
+
+ async def scenario():
+ await _video_load(mine)
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ return await loading.unload_media(theirs, "video")
+
+ unloaded = asyncio.run(scenario())
+ assert unloaded.unloaded is False
+ assert bodies(studio, "/api/inference/video/unload") == []
+ assert backend.current is not None
+
+
+def test_a_media_load_posts_after_an_unload_already_sent(monkeypatch, fast_polls):
+ # An unload of the slot still out would cancel a load registered before it ran.
+ backend = _TokenVideo(unload_s = 0.4)
+ backend.admit(LTX, None)
+ backend.finish()
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ unload = asyncio.ensure_future(loading.unload_media(caller, "video"))
+ await asyncio.sleep(0.1)
+ await _video_load(caller)
+ await asyncio.wait({unload}, timeout = 5)
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+
+ asyncio.run(scenario())
+ events = [e for e, _t in backend.events]
+ assert events == ["admitted", "unload", "unload answered", "admitted"]
+ assert backend.current is not None and backend.current[2] == "loading"
+
+
+def test_the_resident_unload_names_the_load_status_showed(monkeypatch, fast_polls):
+ # While this caller's token unload is out, the UI starts another load over the resident one.
+ # The unload of the resident names the load status showed, so it leaves that newer load alone.
+ backend = _TokenVideo(unload_s = 0.2)
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ await _video_load(caller)
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ backend.finish()
+ await asyncio.wait({job.task}, timeout = 5)
+ # The UI's load replaces it and is resident; this caller's token is now stale.
+ backend.admit(LTX, None)
+ backend.finish()
+
+ def newer_load():
+ if backend.current[1] == "studio-1":
+ backend.admit(LTX, None)
+
+ backend.during_unload = newer_load
+ return await loading.unload_media(caller, "video")
+
+ unloaded = asyncio.run(scenario())
+ assert unloaded.unloaded is False
+ token = _tokens_sent(backend)[0]
+ assert [e for e in backend.events if e[0] == "unload"] == [
+ ("unload", token),
+ ("unload", "studio-1"),
+ ]
+ assert backend.current == [LTX, "studio-2", "loading"]
+
+
+def test_the_resident_unload_unloads_what_status_showed(monkeypatch, fast_polls):
+ backend = _TokenVideo()
+ backend.admit(LTX, None)
+ backend.finish()
+ studio = backend.app()
+ with session(monkeypatch, studio) as call:
+ unloaded = call("unload_model", {"kind": "video"})["structuredContent"]
+ assert unloaded == {"kind": "video", "model": LTX, "unloaded": True}
+ assert bodies(studio, "/api/inference/video/unload") == [{"expected_load_id": "studio-1"}]
+ assert backend.current is None
+
+
+def test_a_live_token_outlasts_any_number_of_refused_loads(monkeypatch, fast_polls):
+ # The first load registered but its answer failed, so it runs on; the retries are refused while it
+ # does. Its token must still be named by the unload.
+ backend = _TokenVideo(answer_status = 500)
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ for _ in range(10):
+ await _video_load(caller)
+ (job,) = [j for j in loading.LOADS.jobs_of("owner") if not j.task.done()] or [None]
+ if job is not None:
+ await asyncio.wait({job.task}, timeout = 5)
+ return await loading.unload_media(caller, "video")
+
+ unloaded = asyncio.run(scenario())
+ first = _tokens_sent(backend)[0]
+ assert [e for e, _t in backend.events].count("busy") == 9
+ assert unloaded.unloaded is True
+ assert ("unload answered", first) in backend.events
+ assert backend.current is None
+ assert loading._MEDIA_TOKENS == {}
+
+
+def test_a_load_shown_ready_retires_the_tokens_before_it(monkeypatch, fast_polls):
+ backend = _TokenVideo()
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ for _ in range(2):
+ await _video_load(caller)
+ (job,) = [j for j in loading.LOADS.jobs_of("owner") if not j.task.done()]
+ await asyncio.wait({job.post}, timeout = 5)
+ backend.finish()
+ await asyncio.wait({job.task}, timeout = 5)
+
+ asyncio.run(scenario())
+ first, second = _tokens_sent(backend)
+ assert list(loading._MEDIA_TOKENS[("owner", "video")]) == [second]
+
+
+def test_a_stalled_status_read_still_ends_the_stopped_load(monkeypatch, fast_polls):
+ monkeypatch.setattr(loading, "READ_S", 0.2)
+ backend = _TokenVideo()
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ await _video_load(caller)
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ backend.stall_status = True
+ began = time.monotonic()
+ with pytest.raises(ToolError, match = "did not answer the video status read"):
+ await loading.unload_media(caller, "video")
+ return time.monotonic() - began
+
+ took = asyncio.run(scenario())
+ assert took < 5
+ (token,) = _tokens_sent(backend)
+ assert ("unload answered", token) in backend.events
+ assert backend.current is None
+
+
+def test_a_load_started_during_the_unload_status_read_waits_for_the_unload(monkeypatch, fast_polls):
+ # The status read is slow and this process starts a load of the slot meanwhile: that load posts
+ # only once the unload answered, so status never names it and the unload never ends it.
+ backend = _TokenVideo()
+ backend.admit(LTX, None)
+ backend.finish()
+ caller = make_caller(studio_app = backend.app())
+
+ async def scenario():
+ backend.status_s = 0.3
+ unload = asyncio.ensure_future(loading.unload_media(caller, "video"))
+ await asyncio.sleep(0.05)
+ load = asyncio.ensure_future(_video_load(caller))
+ result = await unload
+ await load
+ (job,) = loading.LOADS.jobs_of("owner")
+ await asyncio.wait({job.post}, timeout = 5)
+ return result
+
+ result = asyncio.run(scenario())
+ assert result.unloaded is True
+ (token,) = [t for t in _tokens_sent(backend) if not t.startswith("studio-")]
+ events = [e for e in backend.events if e[0] in ("unload answered", "admitted")]
+ assert events == [
+ ("admitted", "studio-1"),
+ ("unload answered", "studio-1"),
+ ("admitted", token),
+ ]
+ assert backend.current == [LTX, token, "loading"]
def test_unloading_stt_lets_a_load_past_its_download_finish_then_unloads_it(
@@ -1069,3 +1433,102 @@ def test_running_chats_block_a_load_with_a_wait_or_cancel_hint(monkeypatch):
)
assert "thread" not in message
assert "force_cancel_active" not in message
+
+
+def _replacement_failed_studio(*, held):
+ # The UI's model (token held) is resident; this caller's replacement t1 was in flight at the status
+ # read and then failed, so its token unload answers superseded and the old model stays loaded.
+ state = {"resident": LTX}
+
+ def status(request, body):
+ if state["resident"] is None:
+ return UNLOADED
+ return {"loaded": True, "repo_id": LTX, "load_id": "t1", "resident_load_id": held}
+
+ def unload(request, body):
+ expected = json.loads(body).get("expected_load_id") if body else None
+ if expected is not None and expected == held and state["resident"] is not None:
+ state["resident"] = None
+ return {"loaded": False, "unloaded": True, "outcome": "unloaded"}
+ return {"unloaded": False, "outcome": "superseded"}
+
+ return fake_studio(
+ {VIDEO_STATUS: status, ("POST", "/api/inference/video/unload"): unload}
+ ), state
+
+
+def test_an_unload_ends_the_model_a_failed_replacement_left_resident(monkeypatch):
+ studio, state = _replacement_failed_studio(held = "ui-1")
+ caller = make_caller(studio_app = studio)
+ loading._remember_media_token(caller, "video", "t1")
+ unloaded = asyncio.run(loading.unload_media(caller, "video"))
+ assert bodies(studio, "/api/inference/video/unload") == [
+ {"expected_load_id": "t1"},
+ {"expected_load_id": "ui-1"},
+ ]
+ assert unloaded.unloaded is True and state["resident"] is None
+
+
+def test_an_unload_names_no_resident_id_studio_does_not_report(monkeypatch):
+ # An older Studio reports no resident_load_id: only the token unload goes out, as before.
+ studio, state = _replacement_failed_studio(held = None)
+ caller = make_caller(studio_app = studio)
+ loading._remember_media_token(caller, "video", "t1")
+ unloaded = asyncio.run(loading.unload_media(caller, "video"))
+ assert bodies(studio, "/api/inference/video/unload") == [{"expected_load_id": "t1"}]
+ assert unloaded.unloaded is False and state["resident"] == LTX
+
+
+def _status_studio(status, answers = None):
+ # Status answers `status`; each unload naming a token answers the next entry of answers[token].
+ answers = {k: list(v) for k, v in (answers or {}).items()}
+
+ def unload(request, body):
+ expected = json.loads(body).get("expected_load_id") if body else None
+ queue = answers.get(expected) or [{"unloaded": False, "outcome": "superseded"}]
+ return queue.pop(0) if len(queue) > 1 else queue[0]
+
+ return fake_studio({VIDEO_STATUS: status, ("POST", "/api/inference/video/unload"): unload})
+
+
+def test_an_unload_sends_no_unconditional_unload_when_studio_cannot_name_the_load(monkeypatch):
+ # Studio reports load ids, but this read could not name the state: an unconditional unload
+ # could end a load admitted meanwhile, so none goes out.
+ studio = _status_studio(
+ {"loaded": True, "repo_id": LTX, "load_id": None, "resident_load_id": None}
+ )
+ caller = make_caller(studio_app = studio)
+ with pytest.raises(ToolError, match = "could not identify"):
+ asyncio.run(loading.unload_media(caller, "video"))
+ assert bodies(studio, "/api/inference/video/unload") == []
+
+
+def test_an_unload_of_a_model_studio_hides_keeps_the_plain_unload(monkeypatch):
+ # Another account's resident (Studio hides its id and status fields): the plain unload as before,
+ # which Studio refuses or performs for this caller.
+ studio = _status_studio({"loaded": True, "yours": False})
+ caller = make_caller(studio_app = studio)
+ asyncio.run(loading.unload_media(caller, "video"))
+ sent = [c[3] for c in studio.state.calls if c[1] == "/api/inference/video/unload"]
+ assert len(sent) == 1 and "expected_load_id" not in (sent[0] or "")
+
+
+def test_an_unload_retries_this_callers_resident_after_its_replacement_fails(monkeypatch):
+ # This caller loaded A, then replacement B was in flight at the read: A's unload answers superseded
+ # while B is loading, B fails and its own unload answers superseded, so A is named again after.
+ studio = _status_studio(
+ {"loaded": True, "repo_id": LTX, "load_id": "B", "resident_load_id": "A"},
+ {
+ "A": [
+ {"unloaded": False, "outcome": "superseded"},
+ {"loaded": False, "unloaded": True, "outcome": "unloaded"},
+ ]
+ },
+ )
+ caller = make_caller(studio_app = studio)
+ loading._remember_media_token(caller, "video", "A")
+ loading._remember_media_token(caller, "video", "B")
+ unloaded = asyncio.run(loading.unload_media(caller, "video"))
+ sent = bodies(studio, "/api/inference/video/unload")
+ assert sent.count({"expected_load_id": "A"}) == 2 and {"expected_load_id": "B"} in sent
+ assert unloaded.unloaded is True
diff --git a/studio/backend/tests/test_mcp_tools_placement.py b/studio/backend/tests/test_mcp_tools_placement.py
index 743349b499d..9183f93e63b 100644
--- a/studio/backend/tests/test_mcp_tools_placement.py
+++ b/studio/backend/tests/test_mcp_tools_placement.py
@@ -12,6 +12,7 @@ from studio_mcp import forward, model_downloads
from .mcp_harness import ( # noqa: F401
bodies,
+ media_load_bodies,
fake_studio,
fast_polls,
run_tool,
@@ -88,7 +89,7 @@ def test_a_plan_that_refuses_the_knobs_is_asked_without_them(monkeypatch, fast_p
)
plans = bodies(studio, "/api/inference/video/download-plan")
assert plans == [{"model_path": "a/v", "h3_task": "fl2va"}, {"model_path": "a/v"}]
- assert bodies(studio, "/api/inference/video/load") == [
+ assert media_load_bodies(studio, "/api/inference/video/load") == [
{"model_path": "a/v", "h3_task": "fl2va"}
]
diff --git a/studio/backend/tests/test_media_load_ids.py b/studio/backend/tests/test_media_load_ids.py
new file mode 100644
index 00000000000..8f988fc48da
--- /dev/null
+++ b/studio/backend/tests/test_media_load_ids.py
@@ -0,0 +1,425 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The load-token contract of the image and video unload routes, against a fake engine."""
+
+from __future__ import annotations
+
+import gc
+import threading
+import time
+import weakref
+
+import pytest
+
+from core.inference import media_load_ids
+from hub.services.models import account_access
+from utils.account_context import AccountContext, run_as
+
+
+class _State:
+ """Stands in for an engine's resident state (pipeline and weights): weakly referable, as theirs are."""
+
+
+class _Engine:
+ """begin_load / unload bump the epoch as the real engines do; a load stays in flight until finish()."""
+
+ def __init__(self) -> None:
+ self._load_token = 0
+ self._state = None
+ self.loading: tuple = ()
+ self.unloads = 0
+
+ def begin_load(self, repo: str) -> dict:
+ self._load_token += 1
+ self.loading = (repo,)
+ return {"loaded": False}
+
+ def finish(self) -> None:
+ self._state = _State()
+ self.loading = ()
+
+ def fail(self) -> None:
+ self.loading = ()
+
+ def loading_repo_ids(self) -> tuple:
+ return self.loading
+
+ def unload(self, only_if = None) -> dict:
+ if only_if is not None and not only_if():
+ return None
+ self.unloads += 1
+ self._load_token += 1
+ self._state = None
+ self.loading = ()
+ return {"loaded": False}
+
+
[email protected](autouse = True)
+def _fresh_records():
+ media_load_ids._reset_for_tests()
+ yield
+ media_load_ids._reset_for_tests()
+
+
+def _load(
+ engine: _Engine,
+ token,
+ repo = "org/model",
+ modality = "video",
+):
+ # As the routes call it: inside admit_media_load, so under media_load_lock.
+ with account_access.media_load_lock:
+ prior = media_load_ids.admitting(modality, engine, token)
+ engine.begin_load(repo)
+ media_load_ids.admitted(modality, engine, token, prior)
+
+
+def _unload(
+ engine: _Engine,
+ token,
+ modality = "video",
+):
+ return media_load_ids.unload_if_current(modality, engine, token, engine.unload)
+
+
[email protected]("finished", [False, True])
+def test_the_token_unloads_its_own_load_in_flight_or_resident(finished):
+ engine = _Engine()
+ _load(engine, "t1")
+ if finished:
+ engine.finish()
+ assert _unload(engine, "t1") == ("unloaded", {"loaded": False})
+ assert engine.unloads == 1
+ assert _unload(engine, "t1") == ("superseded", None)
+ assert engine.unloads == 1
+
+
[email protected]("newer_token", [None, "t2"])
[email protected]("finished", [False, True])
+def test_a_stale_token_leaves_a_same_model_replacement_alone(newer_token, finished):
+ # A newer load of the very same repo, from the UI (no token) or any other client.
+ engine = _Engine()
+ _load(engine, "t1")
+ engine.finish()
+ _load(engine, newer_token)
+ if finished:
+ engine.finish()
+ assert _unload(engine, "t1") == ("superseded", None)
+ assert engine.unloads == 0
+ assert engine.loading or engine._state is not None
+
+
+def test_a_token_whose_load_failed_leaves_the_previous_model_alone():
+ engine = _Engine()
+ _load(engine, None)
+ engine.finish()
+ previous = engine._state
+ _load(engine, "t1")
+ engine.fail()
+ assert _unload(engine, "t1") == ("superseded", None)
+ assert engine._state is previous and engine.unloads == 0
+
+
+def test_a_token_whose_model_was_evicted_elsewhere_leaves_the_slot_alone():
+ engine = _Engine()
+ _load(engine, "t1")
+ engine.finish()
+ # The arbiter's evictor, Studio's idle unload: an unload that is not this token's.
+ engine.unload()
+ engine.unloads = 0
+ assert _unload(engine, "t1") == ("superseded", None)
+ assert engine.unloads == 0
+
+
+def test_a_token_names_a_load_on_one_engine_only():
+ # The image router switched engines since: the record names the old one.
+ old, new = _Engine(), _Engine()
+ _load(old, "t1")
+ new.begin_load("org/model")
+ assert _unload(new, "t1") == ("superseded", None)
+ assert new.unloads == 0 and old.unloads == 0
+
+
+def test_an_unseen_token_cancels_its_load_before_it_starts():
+ # The POST is still on its way, or its answer was lost: the unload comes first.
+ engine = _Engine()
+ assert _unload(engine, "t1") == ("cancelled", None)
+ assert engine.unloads == 0
+ # The route's early check refuses it (and so does admission, for one already past that check).
+ with pytest.raises(media_load_ids.LoadCancelledError):
+ media_load_ids.refuse_cancelled("video", "t1")
+ assert _unload(engine, "t3") == ("cancelled", None)
+ with pytest.raises(media_load_ids.LoadCancelledError):
+ _load(engine, "t3")
+ assert engine._load_token == 0 and not engine.loading
+ _load(engine, "t2")
+ _load(engine, None)
+ assert engine._load_token == 2
+
+
+def test_slots_keep_their_own_tokens():
+ image, video = _Engine(), _Engine()
+ _load(image, "t1", modality = "diffusion")
+ _load(video, "t2")
+ assert _unload(video, "t1") == ("cancelled", None)
+ assert _unload(image, "t1", modality = "diffusion")[0] == "unloaded"
+ assert video.loading and video.unloads == 0
+
+
+def test_the_unload_runs_under_the_admission_lock():
+ # No load can be admitted between the check and the unload.
+ engine = _Engine()
+ _load(engine, "t1")
+ held = []
+
+ def unload(only_if):
+ held.append(account_access.media_load_lock.locked())
+ return engine.unload(only_if)
+
+ assert media_load_ids.unload_if_current("video", engine, "t1", unload)[0] == "unloaded"
+ assert held == [True]
+
+
+def test_a_load_admitted_while_the_unload_waits_is_not_cancelled():
+ # The unload is blocked on the lock while a newer load is admitted: it then finds that load.
+ engine = _Engine()
+ _load(engine, "t1")
+ outcome = []
+ with account_access.media_load_lock:
+ worker = threading.Thread(target = lambda: outcome.append(_unload(engine, "t1")))
+ worker.start()
+ worker.join(0.2)
+ assert worker.is_alive()
+ prior = media_load_ids.admitting("video", engine, "t2")
+ engine.begin_load("org/model")
+ media_load_ids.admitted("video", engine, "t2", prior)
+ worker.join(5)
+ assert outcome == [("superseded", None)]
+ assert engine.loading and engine.unloads == 0
+
+
+def test_a_cancellation_is_pushed_out_only_by_the_same_accounts_tokens(monkeypatch):
+ monkeypatch.setattr(media_load_ids, "_PER_ACCOUNT", 3)
+ engine = _Engine()
+ assert _as("a", _unload, engine, "mine") == ("cancelled", None)
+ # Another account cancelling many tokens cannot evict this account's cancellation.
+ for n in range(10):
+ _as("b", _unload, engine, f"theirs{n}")
+ assert [t for t in media_load_ids._cancelled[("video", "b")]] == [
+ "theirs7",
+ "theirs8",
+ "theirs9",
+ ]
+ with pytest.raises(media_load_ids.LoadCancelledError):
+ _as("a", _load, engine, "mine")
+
+
+def test_a_cancellation_is_consumed_by_its_load_or_expires(monkeypatch):
+ engine = _Engine()
+ assert _unload(engine, "t1") == ("cancelled", None)
+ with pytest.raises(media_load_ids.LoadCancelledError):
+ _load(engine, "t1")
+ assert not any("t1" in tokens for tokens in media_load_ids._cancelled.values())
+ clock = [1000.0]
+ monkeypatch.setattr(media_load_ids, "_now", lambda: clock[0])
+ assert _unload(engine, "t2") == ("cancelled", None)
+ clock[0] += media_load_ids._CANCELLED_TTL_S + 1
+ _load(engine, "t2")
+ assert engine.loading
+
+
+def _as(account: str, fn, *args):
+ return run_as(AccountContext(account_id = account, username = account), fn, *args)
+
+
+def test_a_cancelled_token_refuses_only_the_cancelling_accounts_load():
+ # Another account naming a token in advance cannot stop this account's load with it.
+ engine = _Engine()
+ assert _as("b", _unload, engine, "t1") == ("cancelled", None)
+ _as("a", _load, engine, "t1")
+ assert engine.loading and engine._load_token == 1
+ assert _as("b", _unload, engine, "t2") == ("cancelled", None)
+ with pytest.raises(media_load_ids.LoadCancelledError):
+ _as("b", _load, engine, "t2")
+
+
+def test_status_names_the_load_in_the_slot():
+ engine = _Engine()
+ assert media_load_ids.current_id("video", engine) is None
+ _load(engine, "t1")
+ assert media_load_ids.current_id("video", engine) == "t1"
+ engine.finish()
+ assert media_load_ids.current_id("video", engine) == "t1"
+ _load(engine, None)
+ made = media_load_ids.current_id("video", engine)
+ assert made.startswith("studio-") and made != "t1"
+ assert _unload(engine, "t1") == ("superseded", None)
+ assert _unload(engine, made)[0] == "unloaded"
+ assert media_load_ids.current_id("video", engine) is None
+
+
+def test_a_resident_stays_named_after_a_newer_load_fails():
+ engine = _Engine()
+ _load(engine, "t1")
+ engine.finish()
+ resident = engine._state
+ _load(engine, "t2")
+ assert media_load_ids.current_id("video", engine) == "t2"
+ engine.fail()
+ assert engine._state is resident
+ assert media_load_ids.current_id("video", engine) == "t1"
+ assert _unload(engine, "t2") == ("superseded", None)
+ assert _unload(engine, "t1")[0] == "unloaded"
+
+
+def test_a_blocking_status_read_holds_up_no_admission_or_unload():
+ engine = _Engine()
+ _load(engine, "t1")
+ engine.finish()
+ release, reads, result = threading.Event(), [], []
+
+ def read():
+ reads.append(account_access.media_load_lock.locked())
+ if len(reads) == 1:
+ release.wait(10)
+ return {"read": len(reads)}
+
+ reader = threading.Thread(
+ target = lambda: result.append(media_load_ids.snapshot("video", engine, read))
+ )
+ reader.start()
+ while not reads:
+ time.sleep(0.01)
+ # While the read blocks: a token unload and a new admission both go through.
+ done = []
+ worker = threading.Thread(
+ target = lambda: done.extend([_unload(engine, "t1")[0], _load(engine, "t2")])
+ )
+ worker.start()
+ worker.join(5)
+ assert not worker.is_alive() and done[0] == "unloaded"
+ release.set()
+ reader.join(5)
+ # The first read straddled them, so it was read again: the id names the state the answer shows.
+ assert result == [({"read": 2}, "t2")]
+ assert reads == [False, False]
+
+
+def test_a_status_read_an_admission_lands_in_every_time_names_no_load(monkeypatch):
+ engine = _Engine()
+ _load(engine, "t1")
+ engine.finish()
+ tokens = iter(f"n{n}" for n in range(10))
+
+ def read():
+ _load(engine, next(tokens))
+ return {"loaded": True}
+
+ assert media_load_ids.snapshot("video", engine, read) == ({"loaded": True}, None)
+
+
+def test_a_token_another_account_used_is_cancelled_for_this_account():
+ # Account a once loaded with t1 (now replaced); account b naming t1 records its own cancellation.
+ engine = _Engine()
+ _as("a", _load, engine, "t1")
+ _as("a", _load, engine, "t2")
+ assert _as("a", _unload, engine, "t1") == ("superseded", None)
+ assert _as("b", _unload, engine, "t1") == ("cancelled", None)
+ with pytest.raises(media_load_ids.LoadCancelledError):
+ _as("b", _load, engine, "t1")
+ assert engine.unloads == 0 and engine._load_token == 2
+
+
[email protected]("named", ["t2", "t1"])
+def test_a_load_that_ends_before_the_teardown_costs_no_other_model(named):
+ # t2 is in flight over t1's resident model; between the check and the engine's own recheck,
+ # t2's worker fails (naming t2) or the resident is replaced (naming t1).
+ engine = _Engine()
+ _load(engine, "t1")
+ engine.finish()
+ resident = engine._state
+ _load(engine, "t2")
+ if named == "t1":
+ engine.fail()
+
+ def unload(only_if):
+ if named == "t2":
+ engine.fail()
+ else:
+ engine._state = _State()
+ return engine.unload(only_if)
+
+ assert media_load_ids.unload_if_current("video", engine, named, unload) == ("superseded", None)
+ assert engine.unloads == 0
+ assert engine._state is (resident if named == "t2" else engine._state)
+
+
+def test_the_real_engines_tear_nothing_down_when_the_recheck_fails():
+ from core.inference.diffusion import DiffusionBackend
+ from core.inference.video import VideoBackend
+ for engine in (VideoBackend(), DiffusionBackend()):
+ epoch = engine._load_token
+ assert engine.unload(only_if = lambda: False) is None
+ assert engine._load_token == epoch
+ # The recheck reads the engine's in-flight marker without its lock.
+ assert media_load_ids._in_flight_unlocked(engine) is False
+ assert engine.unload(only_if = lambda: True)["loaded"] is False
+ assert engine._load_token == epoch + 1
+
+
+def _collected(ref) -> bool:
+ gc.collect()
+ return ref() is None
+
+
[email protected]("how", ["replaced", "unloaded", "evicted", "token_unload"])
+def test_the_registry_keeps_no_torn_down_model_alive(how):
+ engine = _Engine()
+ _load(engine, "t1")
+ engine.finish()
+ old = weakref.ref(engine._state)
+ # A newer load that fails leaves t1's state resident, recorded as the published resident.
+ _load(engine, "t2")
+ engine.fail()
+ assert media_load_ids.current_id("video", engine) == "t1"
+ if how == "replaced":
+ _load(engine, "t3")
+ engine.finish()
+ elif how == "token_unload":
+ assert _unload(engine, "t1")[0] == "unloaded"
+ else:
+ # Studio's own unload route without a token, or the arbiter's evictor.
+ engine.unload()
+ assert _collected(old)
+ assert media_load_ids.current_id("video", engine) == ("t3" if how == "replaced" else None)
+
+
+def test_the_real_engine_states_are_weakly_referable():
+ from dataclasses import fields
+
+ from core.inference.diffusion import _LoadState
+ from core.inference.video import _VideoLoadState
+
+ for cls in (_LoadState, _VideoLoadState):
+ state = cls(**{f.name: None for f in fields(cls) if f.init})
+ assert media_load_ids._weak(state)() is state
+
+
+def test_the_resident_load_id_names_the_model_under_a_load_in_flight():
+ engine = _Engine()
+ read = lambda: {"loaded": engine._state is not None}
+ assert media_load_ids.snapshot_ids("video", engine, read) == ({"loaded": False}, None, None)
+ _load(engine, "t1")
+ engine.finish()
+ assert media_load_ids.snapshot_ids("video", engine, read) == ({"loaded": True}, "t1", "t1")
+ # A replacement in flight: load_id names it, resident_load_id the model still loaded under it.
+ _load(engine, "t2")
+ assert media_load_ids.snapshot_ids("video", engine, read)[1:] == ("t2", "t1")
+ engine.finish()
+ assert media_load_ids.snapshot_ids("video", engine, read)[1:] == ("t2", "t2")
+ # Replaced again, and that replacement fails: the resident stays named by both ids.
+ _load(engine, "t3")
+ engine.fail()
+ assert media_load_ids.snapshot_ids("video", engine, read)[1:] == ("t2", "t2")
+ assert media_load_ids.snapshot("video", engine, read) == ({"loaded": True}, "t2")
diff --git a/studio/backend/tests/test_mlx_diffusion_routing.py b/studio/backend/tests/test_mlx_diffusion_routing.py
index 1faaaa4fd02..9298e51f0df 100644
--- a/studio/backend/tests/test_mlx_diffusion_routing.py
+++ b/studio/backend/tests/test_mlx_diffusion_routing.py
@@ -38,8 +38,11 @@ class _Engine:
self.fail_next_load: Optional[str] = None
self.hang_load = False
self.cancelled = threading.Event()
+ # Each engine's own load epoch, as the real engines keep one (bumped by begin_load / unload).
+ self._load_token = 0
def begin_load(self, repo_id: str, **kwargs: Any) -> dict:
+ self._load_token += 1
self.log.append((self.name, "load"))
self.load_kwargs = {"repo_id": repo_id, **kwargs}
self._load_error = None
@@ -65,8 +68,17 @@ class _Engine:
return {"phase": "downloading"}
return {"phase": "ready" if self._state is not None else None}
- def unload(self, *, expected_account: Optional[str] = None) -> dict:
+ def unload(
+ self,
+ *,
+ expected_account: Optional[str] = None,
+ only_if = None,
+ ) -> Optional[dict]:
+ # As the real engines: a conditional unload rechecks under the engine's lock first.
+ if only_if is not None and not only_if():
+ return None
self.log.append((self.name, "unload"))
+ self._load_token += 1
self._state = None
self._loading = False
return self.status()
@@ -524,6 +536,90 @@ def test_unload_frees_both_engines(rig):
assert ("diffusers", "load") not in rig.log
+def test_a_conditional_unload_the_load_no_longer_holds_tears_nothing_down(rig):
+ # A load token whose load ended: refused before the drain, so the fallback keeps running.
+ rig.fit["coexist"] = True
+ _img2img(rig.engine)
+ rig.log.clear()
+ cancel = threading.Event()
+ rig.engine._fallback_cancel = cancel
+ epoch = rig.engine._epoch
+ assert rig.engine.unload(only_if = lambda: False) is None
+ assert rig.log == [] and not cancel.is_set() and rig.engine._epoch == epoch
+
+
+def test_a_conditional_unload_of_the_named_load_frees_both_engines(rig):
+ rig.fit["coexist"] = True
+ _img2img(rig.engine)
+ rig.log.clear()
+ status = rig.engine.unload(only_if = lambda: True)
+ assert rig.log == [("mlx", "unload"), ("diffusers", "unload")]
+ assert status["loaded"] is False
+
+
+def test_a_conditional_unload_refused_by_the_mlx_recheck_keeps_the_fallback_model(rig):
+ # The named load ended between the router's check and the MLX engine's recheck under its lock.
+ rig.fit["coexist"] = True
+ _img2img(rig.engine)
+ rig.log.clear()
+ answers = iter([True, False])
+ assert rig.engine.unload(only_if = lambda: next(answers)) is None
+ assert ("diffusers", "unload") not in rig.log
+
+
+def test_the_routed_load_epoch_survives_the_fallback(rig):
+ # media_load_ids reads _load_token as the load's epoch: a diffusers fallback serves the same load,
+ # so the epoch must not jump to the other engine's own counter.
+ from core.inference import media_load_ids
+ from hub.services.models import account_access
+
+ media_load_ids._reset_for_tests()
+ try:
+ rig.mlx._load_token = 5 # the two engines' own counters differ
+ with account_access.media_load_lock:
+ prior = media_load_ids.admitting("diffusion", rig.engine, "t1")
+ rig.engine.begin_load(
+ "unsloth/Z-Image-Turbo-GGUF",
+ gguf_filename = "z-image-turbo-Q4_0.gguf",
+ local_files_only = True,
+ hf_token = None,
+ model_kind = "gguf",
+ )
+ media_load_ids.admitted("diffusion", rig.engine, "t1", prior)
+ epoch = rig.engine._load_token
+ assert media_load_ids.current_id("diffusion", rig.engine) == "t1"
+ _img2img(rig.engine) # served by the diffusers fallback
+ assert (
+ rig.engine._primary() is rig.diffusers
+ and rig.diffusers._load_token != rig.mlx._load_token
+ )
+ assert rig.engine._load_token == epoch
+ assert media_load_ids.current_id("diffusion", rig.engine) == "t1"
+ rig.engine.unload()
+ assert rig.engine._load_token == epoch + 1
+ finally:
+ media_load_ids._reset_for_tests()
+
+
+def test_a_refused_load_keeps_the_routed_load_epoch(rig, monkeypatch):
+ # MLX refuses a second load while one is in flight; that load's admitted epoch must still match.
+ epoch = rig.engine._load_token
+
+ def refuse(repo_id, **kwargs):
+ raise RuntimeError("A diffusion load is already in progress.")
+
+ monkeypatch.setattr(rig.mlx, "begin_load", refuse)
+ with pytest.raises(RuntimeError):
+ rig.engine.begin_load(
+ "unsloth/Z-Image-Turbo-GGUF",
+ gguf_filename = "z-image-turbo-Q4_0.gguf",
+ local_files_only = True,
+ hf_token = None,
+ model_kind = "gguf",
+ )
+ assert rig.engine._load_token == epoch
+
+
def test_a_scoped_unload_never_cancels_another_accounts_fallback(rig):
from core.inference.gpu_arbiter import GpuBusyForAnotherAccountError
diff --git a/studio/backend/tests/test_swap_kv_resume.py b/studio/backend/tests/test_swap_kv_resume.py
index 09b5de15672..bc04d25081f 100644
--- a/studio/backend/tests/test_swap_kv_resume.py
+++ b/studio/backend/tests/test_swap_kv_resume.py
@@ -652,3 +652,205 @@ def test_an_unload_while_the_acquire_waits_never_evicts_chat(
and arb._owner == arb.CHAT
)
assert cs._preparing == {}
+
+
+def _post_in_thread(client, url, body):
+ answer: dict = {}
+ poster = threading.Thread(target = lambda: answer.setdefault("resp", client.post(url, json = body)))
+ poster.start()
+ return poster, answer
+
+
[email protected]("media", ["image", "video"])
+def test_a_token_unload_while_the_acquire_waits_stops_that_load_before_the_eviction(
+ client,
+ env,
+ monkeypatch,
+ healthy_diffusers,
+ media, # noqa: F811
+):
+ from core.inference import media_load_ids
+
+ _cuda(monkeypatch)
+ if media == "video":
+ client = _video_client(monkeypatch)
+ base, body = _LOADS[media]
+ owner = arb.DIFFUSION if media == "image" else arb.VIDEO
+ media_load_ids._reset_for_tests()
+ monkeypatch.setattr(cs, "_preparing", {})
+ entered, gate = threading.Event(), threading.Event()
+ real_acquire_for = arb.acquire_for
+
+ def _waiting_acquire_for(*args, **kwargs):
+ if args and args[0] == owner:
+ entered.set()
+ gate.wait(10)
+ return real_acquire_for(*args, **kwargs)
+
+ monkeypatch.setattr(arb, "acquire_for", _waiting_acquire_for)
+ poster, answer = _post_in_thread(client, f"{base}/load", {**body, "load_token": "t1"})
+ try:
+ assert entered.wait(5)
+ stopped = client.post(f"{base}/unload", json = {"expected_load_id": "t1"})
+ finally:
+ gate.set()
+ poster.join(10)
+ assert not poster.is_alive()
+ assert stopped.json() == {"unloaded": False, "outcome": "cancelled"}
+ resp = answer["resp"]
+ assert resp.status_code == 409 and "cancelled" in resp.json()["detail"]
+ # The resident chat model was never evicted for a load its unload had already cancelled.
+ assert not [c for c in env.calls if c[0] == "evict-chat"]
+ assert env.is_loaded and arb._owner == arb.CHAT
+ assert client.get(f"{base}/status").json()["loaded"] is False
+ assert cs._preparing == {}
+ media_load_ids._reset_for_tests()
+
+
[email protected]("media", ["image", "video"])
+def test_a_token_unload_leaves_a_preparing_load_with_another_token_alone(
+ client,
+ env,
+ monkeypatch,
+ healthy_diffusers,
+ media, # noqa: F811
+):
+ from core.inference import media_load_ids
+
+ _cuda(monkeypatch)
+ if media == "video":
+ client = _video_client(monkeypatch)
+ base, body = _LOADS[media]
+ owner = arb.DIFFUSION if media == "image" else arb.VIDEO
+ media_load_ids._reset_for_tests()
+ monkeypatch.setattr(cs, "_preparing", {})
+ entered, gate = threading.Event(), threading.Event()
+ real_acquire_for = arb.acquire_for
+
+ def _waiting_acquire_for(*args, **kwargs):
+ if args and args[0] == owner:
+ entered.set()
+ gate.wait(10)
+ return real_acquire_for(*args, **kwargs)
+
+ monkeypatch.setattr(arb, "acquire_for", _waiting_acquire_for)
+ poster, answer = _post_in_thread(client, f"{base}/load", {**body, "load_token": "t1"})
+ try:
+ assert entered.wait(5)
+ other = client.post(f"{base}/unload", json = {"expected_load_id": "t2"})
+ finally:
+ gate.set()
+ poster.join(10)
+ assert not poster.is_alive()
+ assert other.json() == {"unloaded": False, "outcome": "cancelled"}
+ assert answer["resp"].status_code == 200
+ assert [c[0] for c in env.calls] == ["evict-chat"]
+ assert media_load_ids._current["diffusion" if media == "image" else "video"].token == "t1"
+ media_load_ids._reset_for_tests()
+
+
+def test_a_token_unload_during_the_image_preflight_stops_it_before_the_engine_switch(
+ client,
+ env,
+ monkeypatch,
+ healthy_diffusers, # noqa: F811
+):
+ import core.inference.diffusion_engine_router as router
+ import core.inference.diffusion_hub_evidence as evidence
+ from core.inference import media_load_ids
+
+ _cuda(monkeypatch)
+ base, body = _LOADS["image"]
+ media_load_ids._reset_for_tests()
+ monkeypatch.setattr(cs, "_preparing", {})
+ entered, gate = threading.Event(), threading.Event()
+
+ def _slow_prefetch(*args, **kwargs):
+ entered.set()
+ gate.wait(10)
+
+ switched = []
+ real_select = router.select_and_activate_engine
+
+ def _select(*args, **kwargs):
+ switched.append(True)
+ return real_select(*args, **kwargs)
+
+ monkeypatch.setattr(evidence, "prefetch_for_pick", _slow_prefetch)
+ monkeypatch.setattr(router, "select_and_activate_engine", _select)
+ poster, answer = _post_in_thread(client, f"{base}/load", {**body, "load_token": "t1"})
+ try:
+ assert entered.wait(5)
+ stopped = client.post(f"{base}/unload", json = {"expected_load_id": "t1"})
+ finally:
+ gate.set()
+ poster.join(10)
+ assert not poster.is_alive()
+ assert stopped.json() == {"unloaded": False, "outcome": "cancelled"}
+ assert answer["resp"].status_code == 409
+ assert switched == [] and not [c for c in env.calls if c[0] == "evict-chat"]
+ assert cs._preparing == {}
+ media_load_ids._reset_for_tests()
+
+
+def test_an_engine_switch_waits_for_a_cancel_or_refuses_after_it(monkeypatch):
+ import routes.inference as routes
+
+ calls = []
+ with cs.preparing_load("image", "t1") as preflight:
+ assert (
+ routes._activate_unless_cancelled(preflight, None, lambda: calls.append(1) or "e")
+ == "e"
+ )
+ cs.cancel_preparing_loads("image", current_account_id(), load_token = "t2")
+ assert not preflight.cancelled
+ cs.cancel_preparing_loads("image", current_account_id(), load_token = "t1")
+ assert preflight.cancelled
+
+ def _switching():
+ from core.inference.diffusion_engine_router import before_engine_switch
+ before_engine_switch.get()()
+ calls.append(2)
+
+ with pytest.raises(RuntimeError, match = "cancelled"):
+ routes._activate_unless_cancelled(preflight, None, _switching)
+ assert calls == [1]
+
+
+def test_a_cancelled_video_preflight_never_unloads_the_image_model():
+ with cs.preparing_load("video", "t1"):
+ cs.cancel_preparing_loads("video", current_account_id(), load_token = "t1")
+ unloaded = []
+ with pytest.raises(RuntimeError, match = "Video load was cancelled"):
+ cs.run_unless_load_cancelled("Video load was cancelled.", unloaded.append, 1)
+ assert unloaded == []
+
+
+def test_a_token_unload_names_the_engine_the_preflight_settled_on(client, env, monkeypatch):
+ # The named POST finishes switching engines while its preflight is being cancelled: the unload
+ # must check the load on the engine it registered on, not the one active before the cancel.
+ import types
+
+ import core.inference.diffusion_engine_router as router
+ from core.inference import media_load_ids
+
+ old = types.SimpleNamespace(name = "old", unload = None)
+ new = types.SimpleNamespace(name = "new", unload = None)
+ active = {"engine": old}
+ monkeypatch.setattr(router, "get_active_diffusion_engine", lambda: active["engine"])
+
+ def _cancel(*args, **kwargs):
+ active["engine"] = new # the switch completed and the load registered before the cancel ran
+ return 0
+
+ monkeypatch.setattr(cs, "cancel_preparing_loads", _cancel)
+ seen = []
+
+ def _unload_if_current(modality, engine, token, unload):
+ seen.append(engine.name)
+ return "superseded", None
+
+ monkeypatch.setattr(media_load_ids, "unload_if_current", _unload_if_current)
+ resp = client.post(f"{_LOADS['image'][0]}/unload", json = {"expected_load_id": "t1"})
+ assert resp.status_code == 200, resp.text
+ assert seen == ["new"]
diff --git a/studio/backend/tests/test_video_routes.py b/studio/backend/tests/test_video_routes.py
index e4cff0b5c9e..d647a080af7 100644
--- a/studio/backend/tests/test_video_routes.py
+++ b/studio/backend/tests/test_video_routes.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import json
import threading
import time
@@ -308,8 +309,14 @@ def test_load_happy_path_and_arbiter_acquired(client, monkeypatch):
devmod = _shared_setup_3(monkeypatch)
acquired: list = []
- def _fake_acquire(role, register = None):
+ def _fake_acquire(
+ role,
+ register = None,
+ admit = None,
+ ):
acquired.append(role)
+ if admit is not None:
+ admit()
return register() if register is not None else None
monkeypatch.setattr(gpu_arbiter, "acquire_for", _fake_acquire)
@@ -339,7 +346,9 @@ def test_load_forwards_the_gpu_selection(client, monkeypatch):
devmod, "resolve_diffusion_device_target", lambda: types.SimpleNamespace(device = "cuda")
)
monkeypatch.setattr(devmod, "resolve_selected_cuda_ordinal", lambda ids: max(ids))
- monkeypatch.setattr(gpu_arbiter, "acquire_for", lambda role, register = None: register())
+ monkeypatch.setattr(
+ gpu_arbiter, "acquire_for", lambda role, register = None, admit = None: register()
+ )
resp = client.post(
"/api/inference/video/load",
json = {
@@ -1781,3 +1790,154 @@ def test_video_download_plan_still_refuses_a_bad_gpu_while_training_holds_the_ca
)
assert resp.status_code == 400
assert seen == {"ids": [7], "allow_ranking": False}
+
+
+def token_client(client, monkeypatch):
+ """The client, with begin_load and unload moving the load epoch as the real backend's do."""
+ from core.inference import media_load_ids
+
+ media_load_ids._reset_for_tests()
+ backend = video_module.get_video_backend()
+ begin, unload = backend.begin_load, backend.unload
+ backend.unloads = 0
+
+ def begin_load(model_path, **kwargs):
+ backend._load_token += 1
+ return begin(model_path, **kwargs)
+
+ def counted_unload(**kwargs):
+ only_if = kwargs.get("only_if")
+ if only_if is not None and not only_if():
+ return None
+ backend.unloads += 1
+ backend._load_token += 1
+ return unload()
+
+ monkeypatch.setattr(backend, "begin_load", begin_load)
+ monkeypatch.setattr(backend, "unload", counted_unload)
+ yield client, backend
+ media_load_ids._reset_for_tests()
+
+
+def _load_with(client, token):
+ body = {"model_path": "unsloth/LTX-2.3-GGUF", "gguf_filename": "q.gguf"}
+ return client.post(
+ "/api/inference/video/load", json = {**body, **({"load_token": token} if token else {})}
+ )
+
+
+def test_an_unload_naming_the_load_token_unloads_that_load(token_client):
+ client, backend = token_client
+ assert _load_with(client, "t1").status_code == 200
+ resp = client.post("/api/inference/video/unload", json = {"expected_load_id": "t1"})
+ assert resp.status_code == 200
+ assert (resp.json()["outcome"], resp.json()["unloaded"], resp.json()["loaded"]) == (
+ "unloaded",
+ True,
+ False,
+ )
+ assert backend.unloads == 1
+
+
[email protected]("newer", [None, "t2"])
+def test_an_unload_naming_a_stale_token_leaves_the_newer_load_alone(token_client, newer):
+ client, backend = token_client
+ assert _load_with(client, "t1").status_code == 200
+ assert _load_with(client, newer).status_code == 200
+ resp = client.post("/api/inference/video/unload", json = {"expected_load_id": "t1"})
+ assert resp.status_code == 200
+ assert resp.json() == {"unloaded": False, "outcome": "superseded"}
+ assert backend.unloads == 0 and backend.loaded
+
+
+def test_an_unload_naming_a_load_not_started_yet_refuses_that_load(token_client):
+ client, backend = token_client
+ resp = client.post("/api/inference/video/unload", json = {"expected_load_id": "t1"})
+ assert resp.json() == {"unloaded": False, "outcome": "cancelled"}
+ refused = _load_with(client, "t1")
+ assert refused.status_code == 409
+ assert "cancelled" in refused.json()["detail"]
+ assert backend.unloads == 0 and not backend.loaded
+ assert _load_with(client, None).status_code == 200 and backend.loaded
+
+
+def test_an_unload_without_a_body_unloads_whatever_is_loaded(token_client):
+ client, backend = token_client
+ assert _load_with(client, "t1").status_code == 200
+ resp = client.post("/api/inference/video/unload")
+ assert resp.status_code == 200 and "outcome" not in resp.json()
+ assert backend.unloads == 1 and not backend.loaded
+
+
+def test_status_names_the_load_an_unload_can_name(token_client):
+ client, backend = token_client
+ assert client.get("/api/inference/video/status").json()["load_id"] is None
+ assert _load_with(client, None).status_code == 200
+ made = client.get("/api/inference/video/status").json()["load_id"]
+ assert made.startswith("studio-")
+ resp = client.post("/api/inference/video/unload", json = {"expected_load_id": made})
+ assert resp.json()["outcome"] == "unloaded" and backend.unloads == 1
+
+
+def test_status_and_load_id_name_the_same_load(token_client):
+ # A load admitted while status reads the backend makes it read again, so the id in the answer
+ # never names a load the answer does not show.
+ from core.inference import media_load_ids
+ from hub.services.models import account_access
+
+ client, backend = token_client
+ assert _load_with(client, "t1").status_code == 200
+ read = backend.status
+ admitted = []
+
+ def newer_load():
+ with account_access.media_load_lock:
+ prior = media_load_ids.admitting("video", backend, "t2")
+ backend._load_token += 1
+ backend.loading = ("unsloth/LTX-2.3-GGUF",)
+ media_load_ids.admitted("video", backend, "t2", prior)
+ admitted.append(True)
+
+ reads = []
+
+ def racing_status():
+ # The first read is slow and a load is admitted meanwhile, which it does not hold up.
+ reads.append(True)
+ if len(reads) == 1:
+ worker = threading.Thread(target = newer_load)
+ worker.start()
+ worker.join(5)
+ assert not worker.is_alive()
+ return read()
+
+ backend.status = racing_status
+ try:
+ answer = client.get("/api/inference/video/status").json()
+ finally:
+ backend.status = read
+ deadline = time.monotonic() + 5
+ while not admitted and time.monotonic() < deadline:
+ time.sleep(0.01)
+ # The read that straddled the admission was read again, so the id is that of the state it shows.
+ assert answer["load_id"] == "t2" and len(reads) == 2
+ assert admitted == [True]
+ backend.loading = ()
+
+
+def test_the_unload_answers_are_declared_in_the_schema(token_client):
+ client, backend = token_client
+ schema = client.app.openapi()
+ answer = schema["paths"]["/api/inference/video/unload"]["post"]["responses"]["200"]
+ refs = json.dumps(answer)
+ for model in ("VideoLoadUnloadedResponse", "MediaUnloadOutcome", "VideoStatusResponse"):
+ assert f"#/components/schemas/{model}" in refs
+ outcome = schema["components"]["schemas"]["MediaUnloadOutcome"]
+ assert set(outcome["required"]) == {"unloaded", "outcome"}
+ assert _load_with(client, "t1").status_code == 200
+ plain = client.post("/api/inference/video/unload").json()
+ assert set(plain) == set(video_routes.VideoStatusResponse.model_fields)
+ assert _load_with(client, "t2").status_code == 200
+ named = client.post("/api/inference/video/unload", json = {"expected_load_id": "t2"}).json()
+ assert set(named) == set(plain) | {"unloaded", "outcome"}
+ assert (named["unloaded"], named["outcome"]) == (True, "unloaded")