Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: image and video models stay loaded beside chat when they fit (#14106)

Etherl9bac1786d3f8
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index 500167fc9ea..21ebcaae7dd 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -47,6 +47,8 @@ _prior_account: Optional[str] = None
_residents: dict[str, Optional[frozenset[int]]] = {}
# (account, prior account) for residents other than ``_owner``, whose pair lives in the globals above.
_accounts: dict[str, tuple[Optional[str], Optional[str]]] = {}
+# What each image or video resident was admitted with under When they fit / Always (gpu_residency.MediaNeed).
+_media_needs: dict[str, dict] = {}
# Phrases the refusal copy always contains, for readers that lose the X-Unsloth-Refusal header.
MULTI_MODEL_REFUSAL_MARKERS = ("is set to Always", "is set to keep loaded")
@@ -444,6 +446,7 @@ def _drop_resident(owner: str) -> None:
if owner not in _residents:
return
del _residents[owner]
+ _media_needs.pop(owner, None)
if owner == _owner:
if _residents:
_owner = next(reversed(_residents))
@@ -537,9 +540,10 @@ def acquire_for_request(
register = None,
**kwargs,
) -> Any:
+ from core.inference.gpu_residency import ChatLoadInFlightRefusal
try:
return acquire_for(owner, register, **kwargs)
- except GpuBusyForAnotherAccountError as exc:
+ except (GpuBusyForAnotherAccountError, ChatLoadInFlightRefusal) as exc:
raise exc.as_http_exception() from exc
@@ -555,6 +559,8 @@ def acquire_for(
policy: str = "off",
evicted: Optional[list] = None,
gpu_ids: Optional[Iterable[int]] = None,
+ need_mib: Optional[dict] = None,
+ evict_first: bool = False,
) -> Any:
"""Make ``owner`` resident, evicting every other resident under the default policy.
@@ -566,6 +572,10 @@ def acquire_for(
chat, image or video model. ``alongside`` is unrelated: it only decides whether the owner
account is re-claimed. ``evicted`` collects ``{kind, model}`` for each model an evictor
unloads. ``gpu_ids`` are the visible device ordinals the owner will use, when known.
+
+ Under "fit" and "always" only residents on the same GPUs can be evicted (gpu_residency.coreside_locked):
+ ``need_mib`` is what an image or video load will hold per physical GPU (None: unknown, so it evicts first),
+ and ``evict_first`` makes a chat load with no estimate before launch evict as "off" does.
"""
global _owner, _owner_epoch, _owner_account, _prior_account
if owner not in _EVICTORS:
@@ -576,11 +586,34 @@ def acquire_for(
with _lock:
if expected_current is not None and (_owner, _owner_epoch) != expected_current:
raise OwnerChangedError("The resident GPU model changed; retry the load.")
- if register is not None or replacing:
+ coreside = policy != "off" and owner in _KEPT_UNDER_ALWAYS
+ if (register is not None or replacing) and not coreside:
raise_if_other_accounts_active(acting)
+ elif (register is not None or replacing) and not (owner == CHAT and alongside):
+ # Only another account's work on this model blocks reloading it; residents it would evict are
+ # checked one by one.
+ from core.inference.gpu_residency import foreign_on
+ busy = foreign_on(owner, acting)
+ if busy:
+ raise GpuBusyForAnotherAccountError(owner, busy)
_sync_from_view()
prev_owner, prev_pair = _owner, (_owner_account, _prior_account)
+ # Read before evicting: dropping the current owner promotes a parked resident and pops its accounts.
+ parked = owner in _accounts
others = [name for name in _residents if name != owner]
+ if others and coreside:
+ from core.inference.gpu_residency import coreside_locked
+ coreside_locked(
+ owner,
+ policy = policy,
+ need_mib = need_mib,
+ gpu_ids = gpu_ids,
+ allow_evict = allow_evict,
+ evict_first = evict_first,
+ acting = acting,
+ evicted = evicted,
+ )
+ others = []
if others:
if not allow_evict:
raise GpuOwnerBusyError(others[-1])
@@ -603,11 +636,15 @@ def acquire_for(
_EVICTORS[name]()
note_evictions(unloading, sink)
_drop_resident(name)
- if owner != VIDEO:
+ if owner != VIDEO and not (coreside and VIDEO in _residents):
# A resident H3 sd-server sits outside every owner's teardown.
_release_idle_video_servers(f"GPU acquired for {owner}")
# Records who LOADED the model; a plain re-assert must not hand it to whoever asked last.
- claims = prev_owner != owner or ((register is not None or replacing) and not alongside)
+ # A resident parked behind a later claimant is a re-assert, not a new load.
+ rejoins = prev_owner != owner and owner in _residents and parked
+ claims = (prev_owner != owner and not rejoins) or (
+ (register is not None or replacing) and not alongside
+ )
if owner != _owner:
if _owner is not None:
# Another resident stays loaded; park its accounts while ``owner`` is the last claimant.
@@ -620,11 +657,34 @@ def acquire_for(
else:
_owner_account, _prior_account = None, None
_owner = owner
- _residents.pop(owner, None)
+ held_before = _residents.pop(owner, ...)
known = physical_gpu_ids(gpu_ids)
- _residents[owner] = known
+ gpus_now = known if known is not None or not need_mib else frozenset(need_mib)
+ if owner == CHAT and held_before is not ...:
+ from core.inference import model_slots
+ if register is None and not replacing and gpus_now is None:
+ # A re-assert places nothing new.
+ gpus_now = held_before
+ elif alongside or model_slots.holds_vram():
+ # Every chat model kept loaded counts, not just the newest; unknown (None) means every GPU.
+ gpus_now = (
+ None if held_before is None or gpus_now is None else held_before | gpus_now
+ )
+ _residents[owner] = gpus_now
_owner_epoch += 1
- result = register() if register is not None else None
+ try:
+ result = register() if register is not None else None
+ except BaseException:
+ if held_before is not ...:
+ # The model already loaded stays where it was.
+ _residents[owner] = held_before
+ raise
+ if owner in (DIFFUSION, VIDEO) and register is not None:
+ # The price media_held_mib() reports for this resident; unpriced residents are evicted to make room.
+ if coreside and need_mib:
+ _media_needs[owner] = need_mib
+ else:
+ _media_needs.pop(owner, None)
# A raising registration loaded nothing and must not take residency.
if claims:
_prior_account, _owner_account = _owner_account, acting
diff --git a/studio/backend/core/inference/gpu_residency.py b/studio/backend/core/inference/gpu_residency.py
new file mode 100644
index 00000000000..5b50e99d83e
--- /dev/null
+++ b/studio/backend/core/inference/gpu_residency.py
@@ -0,0 +1,812 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Whether an image or video model can stay on a GPU next to chat, and what to unload when it can't.
+
+Keep multiple models loaded set to When they fit or Always lets the GPU arbiter keep several residents. This module
+prices them: a media resident holds the weights plus generation headroom it was admitted with, a new media load is
+priced from the files already on disk, and live free VRAM decides whether it fits. Anything it cannot price is
+treated as not fitting, so the caller frees memory first (fit) or refuses (always). Modelled on training_vram.py.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any, Iterable, Optional
+
+from core.inference import gpu_arbiter as arb
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+# fits iff free_mib >= need_mib + FIT_MARGIN_MIB on every GPU the load uses. Live free VRAM already excludes resident
+# weights, so the margin only covers estimate error and allocator slack. An image load must also keep its
+# transformer on the GPU (the diffusion planner adds its own reserve on top, so this is not doubled there).
+FIT_MARGIN_MIB = 1024
+# A chat load in flight finishes on its own; the media auto-switch uses the same wait.
+K3_RETRY_AFTER_S = 15
+
+MEDIA_OWNERS = (arb.DIFFUSION, arb.VIDEO)
+_MEDIA_KIND = {arb.DIFFUSION: "image", arb.VIDEO: "video"}
+
+
+class MediaNeed(dict):
+ """{physical GPU id: weights + generation headroom MiB}, plus the split the planner rule needs."""
+
+ def __init__(self, per_gpu: dict, *, weights_mib: int, headroom_mib: int, kind: str):
+ super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})
+ self.weights_mib = int(weights_mib)
+ self.headroom_mib = int(headroom_mib)
+ self.kind = kind
+
+
+def current_policy() -> str:
+ """Keep multiple models loaded, read as the owner. Never call this under the arbiter lock (it reads a database)."""
+ try:
+ from utils.multi_model_settings import get_multi_model_mode
+ return get_multi_model_mode()
+ except Exception as exc: # noqa: BLE001 - an unreadable setting keeps today's behaviour
+ logger.warning("gpu_residency: could not read Keep multiple models loaded: %s", exc)
+ return "off"
+
+
+def _media_records() -> dict[str, dict]:
+ """Admitted needs of the media residents still on the GPU. Lock free: acquire_for calls this holding the lock."""
+ records = dict(getattr(arb, "_media_needs", {}) or {})
+ if not records:
+ return {}
+ resident = set(arb._view()) if hasattr(arb, "_view") else {arb._owner}
+ return {
+ owner: need
+ for owner, need in records.items()
+ if owner in resident and _loaded_or_loading(owner)
+ }
+
+
+def _loaded_or_loading(owner: str) -> bool:
+ """False once a media load failed and left nothing loaded, so its price stops counting. Unknown reads as True."""
+ try:
+ engine = _media_engine(owner)
+ if engine is None:
+ return True
+ return getattr(engine, "_state", None) is not None or bool(engine.loading_repo_ids())
+ except Exception: # noqa: BLE001
+ return True
+
+
+def media_held_mib() -> dict[int, int]:
+ """{physical GPU id: MiB} held by resident image and video models (weights + headroom). Never raises."""
+ try:
+ held: dict[int, int] = {}
+ for need in _media_records().values():
+ for gpu, mib in need.items():
+ held[gpu] = held.get(gpu, 0) + int(mib)
+ return held
+ except Exception as exc: # noqa: BLE001
+ logger.warning("gpu_residency: could not price the resident media models: %s", exc)
+ return {}
+
+
+def media_unallocated_mib() -> dict[int, int]:
+ """{physical GPU id: MiB} of the media price live free VRAM still shows as free: an idle resident's generation
+ headroom, or all of a resident still loading. Never raises."""
+ try:
+ held: dict[int, int] = {}
+ for owner, need in _media_records().items():
+ loading = _media_busy(owner)[0]
+ for gpu, mib in need.items():
+ unallocated = int(mib) if loading else int(getattr(need, "headroom_mib", 0))
+ held[gpu] = held.get(gpu, 0) + unallocated
+ return held
+ except Exception as exc: # noqa: BLE001
+ logger.warning("gpu_residency: could not price the resident media models: %s", exc)
+ return {}
+
+
+def _media_engine(owner: str) -> Any:
+ from core.inference.media_keepwarm import engine_if_imported
+ return engine_if_imported(owner)
+
+
+def _media_busy(owner: str) -> tuple[bool, bool]:
+ """(loading, generating) for a media owner; unknown reads as busy, so it is never freed as idle."""
+ try:
+ engine = _media_engine(owner)
+ if engine is None:
+ return False, False
+ loading = bool(engine.loading_repo_ids())
+ progress = engine.generate_progress() or {}
+ from core.inference.media_keepwarm import _TRACKERS
+
+ tracker = _TRACKERS.get(owner)
+ outstanding = bool(tracker is not None and tracker.outstanding())
+ return loading, bool(progress.get("active")) or outstanding
+ except Exception: # noqa: BLE001
+ return True, True
+
+
+def summarize_resident_media() -> dict[str, Any]:
+ """Report the resident image and video models. Never raises."""
+ summary: dict[str, Any] = {}
+ records = {}
+ try:
+ records = _media_records()
+ resident = arb.residents()
+ except Exception: # noqa: BLE001
+ resident = {}
+ for owner in MEDIA_OWNERS:
+ loading, generating = _media_busy(owner) if owner in resident else (False, False)
+ summary[owner] = {
+ "resident": owner in resident,
+ "gpus": sorted(resident.get(owner) or ()),
+ "loading": loading,
+ "generating": generating,
+ "priced": owner in records,
+ }
+ summary["loading"] = any(summary[owner]["loading"] for owner in MEDIA_OWNERS)
+ summary["held_mib"] = media_held_mib()
+ return summary
+
+
+def media_load_need_mib(kind: str, request: Any, gpu_ordinal: Optional[int]) -> Optional[MediaNeed]:
+ """What a new image or video load will hold, priced from files already on disk. None when unknown.
+
+ ``gpu_ordinal`` is the visible torch ordinal the route resolved; None (automatic) is only priceable on a single
+ visible GPU. Never touches the network and never raises."""
+ try:
+ from utils.hardware.hardware import get_parent_visible_gpu_ids
+
+ parent = list(get_parent_visible_gpu_ids() or ())
+ if gpu_ordinal is None:
+ physical = parent[0] if len(parent) == 1 else None
+ else:
+ physical = parent[gpu_ordinal] if 0 <= gpu_ordinal < len(parent) else None
+ if physical is None: # several GPUs and no pick, or a UUID/MIG mask with no numeric ids
+ return None
+ priced = {arb.DIFFUSION: _image_need, arb.VIDEO: _video_need}.get(kind, lambda *_: None)(
+ request, gpu_ordinal
+ )
+ if priced is None or not priced[0] or priced[0] <= 0:
+ return None
+ weights, headroom = int(priced[0]), int(priced[1])
+ return MediaNeed(
+ {physical: weights + headroom},
+ weights_mib = weights,
+ headroom_mib = headroom,
+ kind = _MEDIA_KIND[kind],
+ )
+ except Exception as exc: # noqa: BLE001 - an unpriced load is treated as not fitting
+ logger.warning("gpu_residency: could not price the %s load: %s", kind, exc)
+ return None
+
+
+def _cached_pick(repo: str, filename: Optional[str]) -> Optional[str]:
+ """The picked checkpoint file when it is already on disk (local path or either Hub cache root), else None."""
+ from pathlib import Path
+
+ local = Path(repo).expanduser()
+ if local.is_file():
+ return str(local)
+ if not filename or Path(filename).is_absolute() or ".." in Path(filename).parts:
+ return None
+ if local.is_dir():
+ return str(local / filename) if (local / filename).is_file() else None
+ from huggingface_hub import try_to_load_from_cache
+ from core.inference.diffusion import hub_cache_dir
+
+ for root in (hub_cache_dir(), None):
+ hit = try_to_load_from_cache(repo, filename, cache_dir = root)
+ if isinstance(hit, str) and Path(hit).is_file():
+ return hit
+ return None
+
+
+def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:
+ """(weights MiB, headroom MiB) for a diffusers image load, the way DiffusionBackend._plan_memory prices it."""
+ from core.inference.diffusion import (
+ DiffusionBackend as DB,
+ _image_variant_hint,
+ resolve_model_kind,
+ )
+ from core.inference.diffusion_auto_policy import resident_bytes_from_declared
+ from core.inference.diffusion_engine_router import predict_engine
+ from core.inference.diffusion_families import (
+ detect_family_for_pick,
+ local_pipeline_components_are_complete,
+ named_variant_base,
+ prefer_ungated_mirror,
+ resolve_base_repo,
+ )
+ from core.inference.diffusion_gguf_pipeline import whole_pipeline_gguf_resident_mib
+ from core.inference.diffusion_memory import (
+ estimate_gguf_resident_mib,
+ estimate_image_runtime_mib,
+ estimate_safetensors_dense_mib,
+ file_size_mib,
+ )
+ from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS
+
+ repo, name = request.model_path, request.gguf_filename
+ if getattr(request, "text_encoder_file", None) or getattr(request, "vae_file", None):
+ return None # supplied companions are priced from their headers at load time only
+ kind = resolve_model_kind(name, request.model_kind)
+ fam = detect_family_for_pick(repo, name, request.family_override)
+ if (
+ fam is None
+ or predict_engine(fam, model_kind = kind, gpu_ordinal = gpu_ordinal, gguf_filename = name)
+ != ENGINE_DIFFUSERS
+ ):
+ return None
+ mib = 1024 * 1024
+ if kind == "pipeline":
+ base = repo
+ files: dict[str, int] = {}
+ dirs: list = []
+
+ def grab(d):
+ dirs.append(d)
+ sizes = DB._local_dir_weight_sizes(d, exclude_transformer = False)
+ for rel, size in sizes.items():
+ files[rel] = max(files.get(rel, 0), size)
+ return sizes
+
+ DB._union_over_cached_revs(prefer_ungated_mirror(repo), grab)
+ # A partial download would be priced as the whole pipeline.
+ if not any(local_pipeline_components_are_complete(d, "model_index.json") for d in dirs):
+ return None
+ weights = (resident_bytes_from_declared(repo, files.items()) or 0) // mib
+ else:
+ path = _cached_pick(repo, name)
+ if path is None:
+ return None
+ whole = getattr(fam, "single_file_is_pipeline", False)
+ if kind == "single_file":
+ fp8 = not whole and "fp8" in str(name).lower()
+ weights = estimate_safetensors_dense_mib(file_size_mib(path), fp8_upcast = fp8)
+ elif whole:
+ weights = whole_pipeline_gguf_resident_mib(path) or estimate_gguf_resident_mib(
+ file_size_mib(path)
+ )
+ else:
+ weights = estimate_gguf_resident_mib(file_size_mib(path))
+ base = (request.base_repo or "").strip()
+ if (
+ not base
+ ): # offline only: the link a past load recorded, else the curated variant the name spells
+ from hub.utils.companion_assets import read_companion_links
+ linked = read_companion_links().get(str(repo).strip().lower()) or []
+ base = linked[0] if linked else (named_variant_base(fam, repo, name) or "")
+ base = resolve_base_repo(fam, base)
+ if not whole:
+ companions = max(
+ DB._companion_cache_bytes(prefer_ungated_mirror(base)),
+ DB._companion_cache_bytes(base),
+ )
+ if not companions:
+ return None # the text encoder and VAE are not downloaded yet
+ weights = (weights or 0) + companions // mib
+ hint = _image_variant_hint(fam.name, name, repo, base)
+ return weights, estimate_image_runtime_mib(width = None, height = None, family = hint)
+
+
+def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:
+ """(weights MiB, headroom MiB) for a video load: checkpoint file sizes plus the family's companion table."""
+ from core.inference.diffusion import DiffusionBackend as DB
+ from core.inference.diffusion_component_sizes import video_components_gb
+ from core.inference.diffusion_memory import (
+ estimate_gguf_resident_mib,
+ estimate_safetensors_dense_mib,
+ estimate_video_runtime_mib,
+ file_size_mib,
+ )
+ from core.inference.video import _detect_load_family, resolve_video_model_kind
+ from core.inference.video_families import resolve_video_base_repo
+ from core.inference.video_minimax_h3 import is_h3_native
+ from core.inference.video_moe_pair import moe_partner_filename
+
+ repo, name = request.model_path, request.gguf_filename
+ kind = resolve_video_model_kind(name, request.model_kind)
+ fam = _detect_load_family(
+ repo, name, request.family_override, request.display_repo_id, request.model_kind
+ )
+ if fam is None or is_h3_native(fam, kind):
+ return None # H3 native runs its own server processes, priced elsewhere
+ gb = 1000.0**3 / (1024 * 1024) # MiB per GB of the family tables
+ if kind == "pipeline":
+ components = video_components_gb(fam, repo, allow_hub = False)
+ if components is None or not DB._cache_bytes(repo):
+ return None
+ weights = int(sum(components) * gb)
+ else:
+ names = [name] + ([moe_partner_filename(name)] if getattr(fam, "is_moe", False) else [])
+ paths = [_cached_pick(repo, n) if n else None for n in names]
+ components = video_components_gb(
+ fam, resolve_video_base_repo(fam, request.base_repo, repo), allow_hub = False
+ )
+ if None in paths or components is None:
+ return None
+ sizes = [file_size_mib(p) for p in paths]
+ price = estimate_gguf_resident_mib if kind == "gguf" else estimate_safetensors_dense_mib
+ if None in sizes:
+ return None
+ weights = sum(price(s) for s in sizes) + int((components[1] + components[2]) * gb)
+ preset = fam.resolution_presets[0]
+ return weights, estimate_video_runtime_mib(
+ width = preset[0], height = preset[1], num_frames = fam.default_num_frames
+ )
+
+
+def _free_mib_by_gpu() -> tuple[Optional[dict[int, dict]], Optional[str]]:
+ """{physical id: {free_mib, total_mib}} or (None, reason) when the host can't be priced."""
+ try:
+ from utils.hardware import DeviceType, get_device, get_visible_gpu_utilization
+ if get_device() != DeviceType.CUDA:
+ # MLX and other unified-memory hosts share the pool with the CPU: co-residency there is deferred.
+ return None, "non_accelerator"
+ util = get_visible_gpu_utilization() or {}
+ except Exception as exc: # noqa: BLE001
+ logger.warning("gpu_residency: GPU probe failed: %s", exc)
+ return None, "probe_error"
+ if util.get("index_kind") != "physical":
+ return None, "unmapped_gpus"
+ rows: dict[int, dict] = {}
+ for device in util.get("devices") or ():
+ total_gb, used_gb = device.get("vram_total_gb"), device.get("vram_used_gb")
+ if device.get("unified_memory") or device.get("shared_memory") or not total_gb:
+ rows[device["index"]] = {"unified": True}
+ continue
+ if used_gb is None:
+ continue
+ total = int(total_gb * 1024)
+ rows[device["index"]] = {
+ "free_mib": max(int(total - used_gb * 1024), 0),
+ "total_mib": total,
+ }
+ return rows, None
+
+
+def _keeps_transformer_resident(need: MediaNeed, gpu: int, free_mib: int, total_mib: int) -> bool:
+ """The diffusion planner's verdict for this load at the free memory left: "fits" never means streaming weights."""
+ from core.inference.diffusion_memory import (
+ DeviceMemory,
+ plan_diffusion_memory,
+ plan_keeps_transformer_resident,
+ )
+
+ ordinal = _ordinal_for_physical(gpu)
+ plan = plan_diffusion_memory(
+ target = SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = DeviceMemory("cuda", f"cuda:{ordinal}", "discrete_vram", free_mib, total_mib),
+ model_dense_mib = need.weights_mib,
+ runtime_headroom_mib = need.headroom_mib,
+ # Planned as auto whatever the load asks: Balanced and Low VRAM hold less than an auto plan that stays resident.
+ requested_mode = None,
+ )
+ return plan_keeps_transformer_resident(plan)
+
+
+def _ordinal_for_physical(gpu: int) -> int:
+ try:
+ from utils.hardware.hardware import get_parent_visible_gpu_ids
+ parent = list(get_parent_visible_gpu_ids() or ())
+ return parent.index(gpu) if gpu in parent else gpu
+ except Exception: # noqa: BLE001
+ return gpu
+
+
+def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dict]:
+ """Whether ``owner`` fits next to the residents already loaded. Side-effect free; default-deny with a reason."""
+ if not need_mib:
+ return False, {"reason": "estimate_unavailable"}
+ rows, reason = _free_mib_by_gpu()
+ if rows is None:
+ return False, {"reason": reason}
+ records = _media_records()
+ own = records.get(owner)
+ # A resident still loading has not allocated its weights yet either, so live free VRAM misses all of it.
+ loading = {name for name in records if name != owner and _media_busy(name)[0]}
+ info: dict[str, Any] = {"need_mib": {}, "free_mib": {}, "short_mib": {}, "reason": "fits"}
+ for gpu, mib in need_mib.items():
+ row = rows.get(gpu)
+ if row is None:
+ return False, {"reason": "gpu_unknown", "gpu": gpu}
+ if row.get("unified"):
+ return False, {"reason": "unified_memory", "gpu": gpu}
+ free = row["free_mib"]
+ # An idle resident's generation headroom is not allocated yet; it will be when it generates.
+ free -= sum(
+ int(need[gpu]) if name in loading else getattr(need, "headroom_mib", 0)
+ for name, need in records.items()
+ if name != owner and gpu in need
+ )
+ if own is not None and gpu in own:
+ # Reloading the same owner frees what it holds now.
+ free += max(own[gpu] - getattr(own, "headroom_mib", 0), 0)
+ info["need_mib"][gpu], info["free_mib"][gpu] = int(mib), max(free, 0)
+ if free < mib + FIT_MARGIN_MIB:
+ info["short_mib"][gpu] = int(mib + FIT_MARGIN_MIB - free)
+ info["reason"] = "short"
+ if info["short_mib"]:
+ return False, info
+ if getattr(need_mib, "kind", None) == "image":
+ for gpu in need_mib:
+ if not _keeps_transformer_resident(
+ need_mib, gpu, info["free_mib"][gpu], rows[gpu]["total_mib"]
+ ):
+ info["reason"] = "would_stream"
+ return False, info
+ return True, info
+
+
+def chat_load_in_flight() -> bool:
+ """Whether a chat load (or swap) is running. Its GPUs aren't known before it launches, so callers treat it as
+ overlapping every GPU."""
+ try:
+ from core.inference import llama_keepwarm, model_slots
+ from core.inference.llama_cpp import chat_load_active
+ return bool(
+ llama_keepwarm._load_lock.locked()
+ or chat_load_active()
+ or model_slots.loading is not None
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+_CHAT_LOADING_MESSAGE = (
+ "Unsloth Studio is loading a chat model on this GPU. Retry once it has finished loading."
+)
+
+
+class ChatLoadInFlightRefusal(Exception):
+ """A media load under Always met a chat load whose size isn't known yet. Retryable, unlike the Always refusal."""
+
+ def __init__(self) -> None:
+ super().__init__(_CHAT_LOADING_MESSAGE)
+
+ def as_http_exception(
+ self,
+ path: Optional[str] = None,
+ *,
+ openai_errors: bool = False,
+ ):
+ return chat_loading_http_exception(path, openai_errors = openai_errors)
+
+
+def chat_loading_http_exception(path: Optional[str] = None, *, openai_errors: bool = False):
+ """The transient 409, shaped like GpuBusyForAnotherAccountError.as_http_exception."""
+ from fastapi import HTTPException
+ from utils.api_errors import error_body_for_path
+
+ retry_after = K3_RETRY_AFTER_S
+ if openai_errors and not (path and path.startswith("/v1/")):
+ path = "/v1/"
+ detail = (
+ error_body_for_path(path, _CHAT_LOADING_MESSAGE, status = 409, code = "gpu_busy", param = "model")
+ if path and path.startswith("/v1/")
+ else {"error": "gpu_busy", "message": _CHAT_LOADING_MESSAGE, "retry_after": retry_after}
+ )
+ return HTTPException(status_code = 409, detail = detail, headers = {"Retry-After": str(retry_after)})
+
+
+def _gb(mib: int) -> str:
+ return f"{mib / 1024:.1f}"
+
+
+def need_free_refusal(info: dict, slots: list[dict]) -> Exception:
+ """The Always refusal for a load that doesn't fit next to the models already loaded."""
+ from utils.account_context import is_owner_context
+
+ short = info.get("short_mib") or {}
+ gpu = max(short, key = short.get) if short else None
+ if gpu is None or not is_owner_context():
+ # No numbers (masked host, unified memory, probe error) or a managed account: the plain refusal.
+ return arb.refusal_for(slots)
+ message = (
+ f"This model needs about {_gb(info['need_mib'][gpu])} GB of GPU memory, and {_gb(info['free_mib'][gpu])} GB "
+ "is free next to the models already loaded. Keep multiple models loaded is set to Always, so nothing is "
+ "unloaded to make room. Unload one of them, or choose When they fit."
+ )
+ first = slots[0] if slots else {}
+ return arb.MultiModelAlwaysRefusal(
+ message, kind = first.get("kind"), model = first.get("model") or None
+ )
+
+
+def foreign_on(owner: str, account_id: Optional[str]) -> int:
+ """Generations another account runs on ``owner``'s model; only these block evicting it."""
+ try:
+ if owner == arb.CHAT:
+ from state import active_generations
+ return active_generations.foreign_count(account_id)
+ if owner == arb.DIFFUSION:
+ from hub.services.models import account_access
+ if account_access.generation_is_foreign("diffusion"):
+ return 1
+ return 0
+ if owner == arb.VIDEO:
+ import sys
+
+ video = sys.modules.get("core.inference.video")
+ running = video.generation_account_in_flight() if video is not None else None
+ return int(running is not None and running != account_id)
+ except Exception as exc: # noqa: BLE001 - unknown reads as busy, so nothing is torn down under a stream
+ logger.warning("gpu_residency: could not count generations on %s: %s", owner, exc)
+ return 1
+ return 0
+
+
+def _overlaps(gpus: Optional[Iterable[int]], target: Optional[Iterable[int]]) -> bool:
+ return gpus is None or target is None or bool(set(gpus) & set(target))
+
+
+def _record_evicted(slots: list[dict], evicted: Optional[list], freed: list[str]) -> None:
+ """Report evictions the way acquire_for does (to ``evicted`` or the route's sink) and log them."""
+ try:
+ from core.inference.api_monitor import api_monitor
+ except Exception: # noqa: BLE001
+ api_monitor = None
+ arb.note_evictions(slots, evicted)
+ for slot in slots:
+ model = slot.get("model") or slot.get("kind") or ""
+ freed.append(model)
+ if api_monitor is not None:
+ try:
+ api_monitor.record_lifecycle(event = "unload", model = model, reason = "evicted")
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _evict_resident(
+ name: str, acting: Optional[str], evicted: Optional[list], freed: list[str]
+) -> None:
+ """Evict one whole resident. Caller holds the arbiter lock."""
+ busy = foreign_on(name, acting)
+ if busy:
+ raise arb.GpuBusyForAnotherAccountError(name, busy)
+ slots = arb.resident_slots(name) or [{"kind": _MEDIA_KIND.get(name, "chat"), "model": ""}]
+ logger.info("gpu_arbiter: evicting %s to make room", name)
+ arb._EVICTORS[name]()
+ arb._drop_resident(name)
+ _record_evicted(slots, evicted, freed)
+
+
+def _free_steps(
+ owner: str,
+ gpu_ids: Optional[Iterable[int]],
+ reason: str,
+ evicted: Optional[list],
+ *,
+ need_mib: Optional[dict] = None,
+ short_mib: int = 0,
+ acting: Optional[str] = None,
+ media_only: bool = False,
+) -> list[str]:
+ """Free the cheapest residents on ``gpu_ids`` first, re-checking after each step. Caller holds the arbiter lock.
+
+ Order: anything still loading, idle video servers, the idle image or video model, idle chat models kept
+ alongside, then the active chat model. Residents on other GPUs are never touched. With ``need_mib`` it stops once
+ the load fits; without one (a chat launch prices itself) it stops after the first step that freed something.
+ ``media_only`` stops before any chat model. "Keep model in GPU memory" does not protect a model here, as with Off.
+ """
+ target = set(gpu_ids) if gpu_ids is not None else None
+ freed: list[str] = []
+
+ def done() -> bool:
+ if need_mib:
+ return can_load_alongside(owner, need_mib)[0]
+ return bool(freed)
+
+ def overlapping() -> list[str]:
+ return [n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)]
+
+ # 1. Residents still loading can't be sized.
+ for name in overlapping():
+ if media_only and name not in MEDIA_OWNERS:
+ continue
+ loading = (
+ chat_load_in_flight()
+ if name == arb.CHAT
+ else name in MEDIA_OWNERS and _media_busy(name)[0]
+ )
+ if loading:
+ _evict_resident(name, acting, evicted, freed)
+ if freed and done():
+ return freed
+ # 2. Idle H3 video servers sit outside every owner's teardown; one beside a video model on other GPUs stays.
+ video_gpus = arb._view().get(arb.VIDEO, ...)
+ if owner != arb.VIDEO and (video_gpus is ... or _overlaps(video_gpus, target)):
+ try:
+ from core.inference.video_minimax_h3 import release_h3_native_servers
+ if release_h3_native_servers(reason):
+ freed.append("video-server")
+ except Exception as exc: # noqa: BLE001
+ logger.warning("gpu_residency: could not release the idle video server: %s", exc)
+ if freed and done():
+ return freed
+ # 3. The idle image or video model on these GPUs.
+ for name in overlapping():
+ if name in MEDIA_OWNERS and not any(_media_busy(name)) and not foreign_on(name, acting):
+ _evict_resident(name, acting, evicted, freed)
+ if done():
+ return freed
+ if media_only:
+ return freed
+ # 4. Idle chat models kept alongside, least recently used first.
+ if arb.CHAT in overlapping() or owner == arb.CHAT:
+ from core.inference import model_slots
+
+ short = short_mib or (
+ max((can_load_alongside(owner, need_mib)[1].get("short_mib") or {0: 0}).values())
+ if need_mib
+ else 0
+ )
+ victims = model_slots.eviction_victims(None, short, gpu_ids)
+ if victims:
+ reports = {id(victim): _kept_slot_report(victim) for victim in victims}
+ # A slot a request reached since it was picked stays loaded, so it isn't reported.
+ dropped = model_slots.evict_these(victims) or []
+ _record_evicted([reports[id(victim)] for victim in dropped], evicted, freed)
+ if dropped and done():
+ return freed
+ # 5. The active chat model.
+ if owner != arb.CHAT and arb.CHAT in overlapping():
+ _evict_resident(arb.CHAT, acting, evicted, freed)
+ return freed
+
+
+def _kept_slot_report(slot: Any) -> dict:
+ try:
+ from routes.inference import _kept_slot_reports
+ reports = _kept_slot_reports(slot)
+ if reports:
+ return reports[0]
+ except Exception: # noqa: BLE001
+ pass
+ from core.inference.model_ids import public_model_id
+
+ request = getattr(slot, "request", None)
+ raw = str(getattr(request, "model_path", "") or "")
+ model = public_model_id(raw) or raw
+ return {"kind": "chat", "model": model, "account": getattr(slot, "account", None)}
+
+
+def media_fit_kwargs(
+ owner: str,
+ request: Any,
+ gpu_ordinal: Optional[int],
+ policy: Optional[str] = None,
+) -> dict:
+ """The acquire_for keywords for an image or video load: the policy and, when known, what it will hold."""
+ policy = policy or current_policy()
+ if policy == "off":
+ return {}
+ need = media_load_need_mib(owner, request, gpu_ordinal)
+ kwargs: dict = {"policy": policy}
+ if gpu_ordinal is not None:
+ # Known even when the load can't be priced, so residents on other GPUs stay out of it.
+ kwargs["gpu_ids"] = [gpu_ordinal]
+ if need:
+ kwargs["need_mib"] = need
+ return kwargs
+
+
+def free_for(
+ owner: str,
+ gpu_ids: Optional[Iterable[int]],
+ reason: str,
+ evicted: Optional[list] = None,
+ *,
+ need_mib: Optional[dict] = None,
+ short_mib: int = 0,
+ media_only: bool = False,
+) -> list[str]:
+ """Free residents on ``gpu_ids`` so ``owner`` fits, cheapest first. Appends {kind, model} to ``evicted``."""
+ from utils.account_context import current_account_id
+
+ acting = current_account_id()
+ with arb._lock:
+ arb._sync_from_view()
+ return _free_steps(
+ owner,
+ gpu_ids,
+ reason,
+ evicted,
+ need_mib = need_mib,
+ short_mib = short_mib,
+ acting = acting,
+ media_only = media_only,
+ )
+
+
+def _claim_holds_anything(name: str) -> bool:
+ """Whether a kept claim holds or is loading a model; unreadable counts as yes. Caller holds the lock."""
+ if arb.resident_slots(name):
+ return True
+ return arb._chat_occupied() if name == arb.CHAT else arb._media_occupied(name)
+
+
+def coreside_locked(
+ owner: str,
+ *,
+ policy: str,
+ need_mib: Optional[dict],
+ gpu_ids: Optional[Iterable[int]],
+ allow_evict: bool,
+ evict_first: bool,
+ acting: Optional[str],
+ evicted: Optional[list],
+) -> None:
+ """Make room for ``owner`` under When they fit / Always, evicting only what has to go. Caller holds the lock.
+
+ Residents on other GPUs stay. A chat load joins priced image and video models and lets its launch price them. An
+ image or video load joins when it fits, else When they fit frees the cheapest residents and Always refuses.
+ Anything that can't be priced is evicted first under When they fit, as with Off.
+ """
+ target = (
+ arb.physical_gpu_ids(gpu_ids)
+ if gpu_ids is not None
+ else (frozenset(need_mib) if need_mib else None)
+ )
+ overlap = [
+ name for name, gpus in arb._view().items() if name != owner and _overlaps(gpus, target)
+ ]
+ for name in [n for n in overlap if n not in arb._KEPT_UNDER_ALWAYS]:
+ # A Decision API server is evicted as before.
+ if not allow_evict:
+ raise arb.GpuOwnerBusyError(name)
+ _evict_resident(name, acting, evicted, [])
+ kept = [n for n in overlap if n in arb._KEPT_UNDER_ALWAYS]
+ if policy == "always":
+ # A claim whose load failed holds nothing, so Always evicts it rather than refusing (gpu_arbiter.acquire_for).
+ for name in [n for n in kept if not _claim_holds_anything(n)]:
+ if not allow_evict:
+ raise arb.GpuOwnerBusyError(name)
+ _evict_resident(name, acting, evicted, [])
+ kept.remove(name)
+
+ def evict_or_refuse(names: list[str]) -> None:
+ if not names:
+ return
+ if policy == "always":
+ raise arb.refusal_for(
+ [slot for n in names for slot in arb.resident_slots(n)], arb.KIND_LABELS[names[0]]
+ )
+ if not allow_evict:
+ raise arb.GpuOwnerBusyError(names[-1])
+ for name in names:
+ _evict_resident(name, acting, evicted, [])
+ kept.remove(name)
+
+ if not kept:
+ return
+ if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_load_in_flight():
+ # Its size isn't known until it launches.
+ if policy == "always":
+ raise ChatLoadInFlightRefusal()
+ evict_or_refuse([arb.CHAT])
+ if owner == arb.CHAT and not need_mib:
+ if evict_first:
+ # No estimate before launch (a non-GGUF load, or a load with no retry to make room).
+ return evict_or_refuse(list(kept))
+ records = _media_records()
+ evict_or_refuse([n for n in kept if n not in records or _media_busy(n)[0]])
+ if kept and _free_mib_by_gpu()[0] is None:
+ evict_or_refuse(list(kept))
+ return
+ if need_mib is None or evict_first:
+ return evict_or_refuse(list(kept))
+ fits, info = can_load_alongside(owner, need_mib)
+ if fits:
+ return
+ if policy == "always":
+ raise need_free_refusal(info, [slot for n in kept for slot in arb.resident_slots(n)])
+ if not allow_evict:
+ raise arb.GpuOwnerBusyError(kept[-1])
+ _free_steps(owner, target, f"GPU memory for {owner}", evicted, need_mib = need_mib, acting = acting)
+ if not can_load_alongside(owner, need_mib)[0]:
+ # Still short: unload the rest on these GPUs, as Off does, and let the planner offload.
+ for name in [
+ n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)
+ ]:
+ _evict_resident(name, acting, evicted, [])
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index e3c2dfcd47e..218236da468 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -639,6 +639,23 @@ def _net_of_held_vram(gpu_mem, held: dict[int, int]):
]
+def _net_of_unallocated_vram(gpu_mem, unallocated: dict[int, int]):
+ """Take from each GPU's live free MiB what a resident image or video model has not allocated yet (its generation
+ headroom): the total cap misses it once another process uses the card. Totals of 0 are left as is."""
+ if not unallocated:
+ return gpu_mem
+ return [
+ (idx, max(0, free - unallocated.get(idx, 0)) if total > 0 else free, total)
+ for idx, free, total in gpu_mem
+ ]
+
+
+def _net_of_other_models_vram(gpu_mem, held: dict[int, int], unallocated: dict[int, int]):
+ """Live free less unallocated media VRAM, then capped at total less ``held``. The other order charges the
+ headroom twice whenever the total cap binds (nothing else on the card)."""
+ return _net_of_held_vram(_net_of_unallocated_vram(gpu_mem, unallocated), held)
+
+
_serving_backends: "weakref.WeakSet[LlamaCppBackend]" = weakref.WeakSet()
# A WeakSet is not safe to iterate while another thread adds or drops a backend.
_serving_backends_lock = threading.Lock()
@@ -26717,7 +26734,9 @@ class LlamaCppBackend:
_gpu_mem = self._get_gpu_memory(binary, for_llama_server = not gpu_ids)
_held_by_others = self._other_planned_vram_mib()
_shared_gpus = frozenset(_held_by_others)
- _gpu_mem = _net_of_held_vram(_gpu_mem, _held_by_others)
+ _gpu_mem = _net_of_other_models_vram(
+ _gpu_mem, _held_by_others, self._media_unallocated_vram_mib()
+ )
# Every present device gated out (#7624). Left alone the launch
# takes the `--fit on` arm with `gpu_indices` still None, so no
# mask is written, the child enumerates every unsupported card and
@@ -36245,7 +36264,7 @@ class LlamaCppBackend:
)
def _other_planned_vram_mib(self) -> dict[int, int]:
- """MiB per GPU the other serving backends' running servers hold."""
+ """MiB per GPU the other serving backends' running servers and the resident image/video models hold."""
held: dict[int, int] = {}
serving = _serving_backends_snapshot()
if self not in serving:
@@ -36255,8 +36274,21 @@ class LlamaCppBackend:
continue
for idx, mib in getattr(backend, "_planned_vram_mib", {}).items():
held[idx] = held.get(idx, 0) + mib
+ # Weights plus generation headroom, the full figure _net_of_held_vram needs (empty under Off).
+ from core.inference.gpu_residency import media_held_mib
+
+ for idx, mib in media_held_mib().items():
+ held[idx] = held.get(idx, 0) + mib
return held
+ def _media_unallocated_vram_mib(self) -> dict[int, int]:
+ """MiB per GPU of the resident image/video models' price not yet allocated (empty under Off)."""
+ if self not in _serving_backends_snapshot():
+ return {}
+ from core.inference.gpu_residency import media_unallocated_mib
+
+ return media_unallocated_mib()
+
def _note_server_pid(self, pid: int) -> None:
if getattr(self, "_owns_pidfile", True):
self._record_server_pid(pid)
diff --git a/studio/backend/core/inference/media_auto_switch.py b/studio/backend/core/inference/media_auto_switch.py
index 28c50593361..ffc41e75a14 100644
--- a/studio/backend/core/inference/media_auto_switch.py
+++ b/studio/backend/core/inference/media_auto_switch.py
@@ -324,15 +324,26 @@ async def _gated_start_load(
re-plan and the load registration take.
"""
from fastapi import HTTPException
+ from core.inference import gpu_arbiter, gpu_residency
from core.inference.media_keepwarm import admission_gate
from core.inference.llama_keepwarm import inference_lifecycle_gate
- needed = (
- (inference_lifecycle_gate(), admission_gate(DIFFUSION), admission_gate(VIDEO))
- if takes_the_gpu
- else (admission_gate(owner),)
- )
try:
+ # Under Always a chat load's GPUs aren't known yet, so it overlaps every card: refuse now
+ # (retryable) instead of queueing on chat's gate behind it. Off and fit keep the wait, and
+ # so does a CPU chat load (no GPU claim) or one on known GPUs (the arbiter checks overlap).
+ if (
+ takes_the_gpu
+ and gpu_residency.chat_load_in_flight()
+ and gpu_arbiter.residents().get(gpu_arbiter.CHAT, ...) is None
+ and await asyncio.to_thread(gpu_residency.current_policy) == "always"
+ ):
+ raise gpu_residency.chat_loading_http_exception(openai_errors = openai_errors)
+ needed = (
+ (inference_lifecycle_gate(), admission_gate(DIFFUSION), admission_gate(VIDEO))
+ if takes_the_gpu
+ else (admission_gate(owner),)
+ )
async with contextlib.AsyncExitStack() as gates:
for gate in needed:
await bounded(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1d7f67b04ea..5c07e1c5530 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -12482,6 +12482,194 @@ def release_chat_after_kept_models() -> bool:
return model_slots.in_slot(None, lambda: release_if(CHAT, zero_vram_primary_only))
+def release_chat_after_cpu_fallback(backend, identifier) -> bool:
+ """A primary whose GPU start fell back to the CPU drops CHAT, unless something else still holds VRAM."""
+ from core.inference.gpu_arbiter import CHAT, release_if
+
+ def only_cpu_left() -> bool:
+ if getattr(backend, "active_model_name", None) != identifier:
+ return False
+ entry = (getattr(backend, "models", None) or {}).get(identifier)
+ if not isinstance(entry, dict) or entry.get("audio_device_fallback") != "cpu":
+ return False
+ llama = get_llama_cpp_backend()
+ return not model_slots.holds_vram() and (not llama.is_active or llama.holds_no_vram)
+
+ return model_slots.in_slot(None, lambda: release_if(CHAT, only_cpu_left))
+
+
+# True while load_model_gated can free memory and retry, so only there does a primary refuse a partial GPU fit.
+_primary_can_retry_fit: contextvars.ContextVar[bool] = contextvars.ContextVar(
+ "primary_can_retry_fit", default = False
+)
+
+
+def _placement_gpu_ids(placement) -> Optional[list[int]]:
+ # Only GGUF and native audio resolve their GPUs; any other chat load keeps the ids it asked for.
+ if placement.resolved_gpu_ids is not None:
+ return placement.resolved_gpu_ids
+ return placement.requested_gpu_ids
+
+
+def _media_on_target_gpus(placement) -> bool:
+ """Whether an image or video model kept loaded beside chat holds memory on the GPUs this load targets."""
+ from core.inference.gpu_residency import media_held_mib
+
+ held = media_held_mib()
+ gpus = None if placement.gpu_ids_are_vulkan_ordinals else _placement_gpu_ids(placement)
+ return bool(held) and (not gpus or bool(set(held) & set(gpus)))
+
+
+def _primary_refuses_partial_fit(serving: bool, policy: str, placement) -> bool:
+ """Whether a primary GGUF load refuses a partial GPU fit beside an image or video model, to retry once room is
+ made. Not under Always while a primary serves: the refusal lands after its teardown, leaving nothing loaded."""
+ return (
+ _primary_can_retry_fit.get()
+ and not (serving and policy == "always")
+ and _media_on_target_gpus(placement)
+ )
+
+
+def _visible_ordinals(physical_ids) -> Optional[list[int]]:
+ """Visible device ordinals for physical GPU ids, or None when one isn't visible (then every GPU counts)."""
+ try:
+ from utils.hardware.hardware import get_parent_visible_gpu_ids
+ parent = list(get_parent_visible_gpu_ids() or ())
+ except Exception: # noqa: BLE001
+ return None
+ if not parent:
+ return [int(g) for g in physical_ids]
+ if any(int(g) not in parent for g in physical_ids):
+ return None
+ return [parent.index(int(g)) for g in physical_ids]
+
+
+def _chat_fit_kwargs(
+ config,
+ placement,
+ policy: Optional[str] = None,
+ credit_gb: Optional[dict] = None,
+) -> dict:
+ """How the CHAT acquire shares the GPU under When they fit / Always.
+
+ audio.cpp and native audio are priced from the placement estimate. A GGUF launch prices itself (it refuses a
+ partial fit and load_model_gated makes room). Anything else has no estimate before launch: When they fit
+ evicts first, Always tries the load and turns an out-of-memory failure into its refusal. ``credit_gb`` is what
+ the outgoing primary frees per GPU when this load replaces it.
+ """
+ if policy is None:
+ from core.inference.gpu_residency import current_policy
+ policy = current_policy()
+ if policy == "off":
+ return {}
+ kwargs: dict = {"policy": policy}
+ gpus = _placement_gpu_ids(placement)
+ ordinals = (
+ _visible_ordinals(gpus) if gpus and not placement.gpu_ids_are_vulkan_ordinals else None
+ )
+ if ordinals is not None:
+ # Only residents on these GPUs are weighed or unloaded; the arbiter maps them back to physical ids.
+ kwargs["gpu_ids"] = ordinals
+ if placement.native_required_gb and gpus and not placement.gpu_ids_are_vulkan_ordinals:
+ credit = credit_gb or {}
+ # Net of the outgoing primary, so it may be negative: a smaller replacement needs no extra room.
+ kwargs["need_mib"] = {
+ int(g): int((placement.native_required_gb / len(gpus) - credit.get(int(g), 0.0)) * 1024)
+ for g in gpus
+ }
+ elif policy == "fit" and not (config.is_gguf and _primary_can_retry_fit.get()):
+ kwargs["evict_first"] = True
+ return kwargs
+
+
+def _outgoing_chat_credit_gb(
+ placement, policy: str, replacing: bool, availability: "Optional[_NativeAudioAvailability]"
+) -> tuple[Optional[dict], Optional[dict]]:
+ """VRAM the primary chat model frees per GPU when a priced native load replaces it beside an image or video model.
+
+ Returns (credit, wait): the CHAT acquire charges the new load less the credit, and the load waits for ``wait`` GB
+ to be free after the teardown. (None, None) when nothing is credited.
+ """
+ if not (
+ replacing
+ and policy != "off"
+ and placement.native_required_gb
+ and placement.resolved_gpu_ids
+ and _media_on_target_gpus(placement)
+ ):
+ return None, None
+ from core.inference.gpu_arbiter import CHAT
+
+ if availability is not None and availability.credited == CHAT:
+ # The automatic placement already measured the outgoing chat model and waits for it.
+ return availability.reclaimable_gpu_gb, None
+ chat = _native_audio_post_handoff_free_gb(CHAT)
+ if chat is None:
+ return None, None
+ target = list(placement.resolved_gpu_ids)
+ share = float(placement.native_required_gb) / len(target)
+ return chat.reclaimable_gpu_gb, {int(g): share for g in target}
+
+
+async def _make_room_for_primary(request: LoadRequest, exc, evicted: list) -> LoadRequest:
+ """A primary chat load refused a partial GPU fit beside an image or video model: return the request to retry.
+
+ When they fit frees the image or video model (cheapest first), else lets llama-server fit what it can. Always
+ only accepts a smaller context; otherwise it refuses, since nothing may be unloaded.
+ """
+ from core.inference import gpu_arbiter, gpu_residency
+
+ if exc.capped:
+ return request.model_copy(update = {"force_alongside": True})
+ if await asyncio.to_thread(gpu_residency.current_policy) == "always":
+ rows = (await asyncio.to_thread(gpu_residency._free_mib_by_gpu))[0] or {}
+ gpus = [g for g in exc.gpu_indices or () if "free_mib" in rows.get(g, {})]
+ free = sum(rows[g]["free_mib"] for g in gpus)
+ info = {
+ "need_mib": {0: free + exc.short_mib},
+ "free_mib": {0: free},
+ "short_mib": {0: exc.short_mib},
+ }
+ held, short = gpu_arbiter.residents(), set(exc.gpu_indices or ())
+ # Name only models that could be in the way: skip one known to sit on other GPUs.
+ slots = [
+ s
+ for o in gpu_residency.MEDIA_OWNERS
+ if not (short and held.get(o) and not held[o] & short)
+ for s in gpu_arbiter.resident_slots(o)
+ ]
+ raise gpu_residency.need_free_refusal(info if gpus else {}, slots)
+ freed = await _free_for_chat(exc, evicted)
+ # Anything freed (an idle video server counts) earns a retry at full size; else llama-server fits what it can.
+ return request if freed else request.model_copy(update = {"force_alongside": True})
+
+
+async def _free_for_chat(
+ exc,
+ evicted: list,
+ media_only: bool = False,
+) -> list[str]:
+ """Free the cheapest residents on the GPUs a chat load came up short on; ``media_only`` keeps every chat model."""
+ from core.inference import gpu_arbiter, gpu_residency
+
+ # The route's eviction sink when it keeps one (it reports the slots itself), else the plain list.
+ sink = gpu_arbiter.eviction_sink.get()
+ slots: list = sink if sink is not None else []
+ before = len(slots)
+ freed = await asyncio.to_thread(
+ gpu_residency.free_for,
+ gpu_arbiter.CHAT,
+ exc.gpu_indices,
+ "a chat model",
+ slots,
+ short_mib = exc.short_mib,
+ media_only = media_only,
+ )
+ if sink is None:
+ evicted += [s["model"] for s in slots[before:] if s.get("model")]
+ return freed
+
+
def reap_dead_managed_engine(backend) -> None:
"""A crashed engine leaves nothing resident, so its claim and sharers go with it."""
from core.inference.gpu_arbiter import CHAT, holds
@@ -16207,20 +16395,38 @@ class _NativeAudioAvailability(NamedTuple):
effective_free_gb: dict[int, float]
reclaimable_gpu_gb: dict[int, float]
owner_snapshot: tuple[Optional[str], int] = (None, 0)
+ # The owner whose memory is credited, when not the last claimant (owner_snapshot[0]).
+ credited_owner: Optional[str] = None
+
+ @property
+ def credited(self) -> Optional[str]:
+ return self.credited_owner or self.owner_snapshot[0]
-def _native_audio_post_handoff_free_gb() -> Optional[_NativeAudioAvailability]:
- """Free VRAM after the outgoing Studio owner is torn down, or ``None``.
+def _native_audio_post_handoff_free_gb(
+ outgoing: Optional[str] = None,
+) -> Optional[_NativeAudioAvailability]:
+ """Free VRAM after the outgoing Studio owner (``outgoing``, default the last claimant) is torn down, or ``None``.
Only memory attributable to a backend this CHAT handoff provably evicts is
credited. Training, STT, export and unknown/external processes remain in
the live-used figure. Missing attribution fails closed.
"""
import utils.hardware as hardware
- from core.inference.gpu_arbiter import CHAT, DIFFUSION, VIDEO, owner_snapshot
+ from core.inference.gpu_arbiter import CHAT, DIFFUSION, VIDEO, holds, owner_snapshot
outgoing_owner = owner_snapshot()
- owner = outgoing_owner[0]
+ if (
+ outgoing is None
+ and outgoing_owner[0] in (DIFFUSION, VIDEO)
+ and routed_slot.get() is None
+ and holds(CHAT)
+ ):
+ # A primary load replaces the chat model; an image or video model claimed after it may stay beside.
+ chat = _native_audio_post_handoff_free_gb(CHAT)
+ if chat is not None:
+ return chat
+ owner = outgoing or outgoing_owner[0]
live: dict[int, tuple[float, float]] = {}
reclaimable: dict[int, float] = {}
@@ -16378,6 +16584,7 @@ def _native_audio_post_handoff_free_gb() -> Optional[_NativeAudioAvailability]:
},
reclaimable_gpu_gb = reclaimable,
owner_snapshot = outgoing_owner,
+ credited_owner = owner,
)
@@ -18575,6 +18782,7 @@ async def load_model_gated(
action = "Loading a model",
)
+ _primary_can_retry_fit.set(True)
while True:
try:
response = await _run_tracked_load_model_impl(
@@ -18589,6 +18797,8 @@ async def load_model_gated(
except (GpuMemoryShortError, HTTPException) as exc:
measured = isinstance(exc, GpuMemoryShortError)
if not measured:
+ if extra is None and _ran_out_of_memory(exc):
+ await asyncio.to_thread(_refuse_oom_beside_media_under_always)
# A non-GGUF load reports no fit up front: its out-of-memory failure
# makes room the same way, one idle kept model at a time.
if extra is None or not _ran_out_of_memory(exc):
@@ -18599,6 +18809,10 @@ async def load_model_gated(
if isinstance(detail, dict)
else str(detail)
)
+ if extra is None:
+ # Only a primary beside an image or video model refuses a partial fit.
+ request = await _make_room_for_primary(request, exc, evicted)
+ continue
from utils.multi_model_settings import get_multi_model_mode
if await asyncio.to_thread(get_multi_model_mode) == "always":
@@ -18612,6 +18826,9 @@ async def load_model_gated(
raise refusal_for(
await asyncio.to_thread(chat_eviction_reports)
) from exc
+ # An idle image or video model goes before any chat model, as gpu_residency orders it.
+ if await _free_for_chat(exc, evicted, media_only = True):
+ continue
# No chat may start on a victim between its pick and teardown.
async with inference_lifecycle_gate() if new_slot else nullcontext():
victims = await asyncio.to_thread(
@@ -18652,6 +18869,8 @@ async def load_model_gated(
# the new one is placed on a GPU it no longer shares.
await asyncio.to_thread(_unload_primary)
note_keyed(replaced_reports)
+ # No retry follows this load, so it must not refuse a partial fit beside media.
+ _primary_can_retry_fit.set(False)
response = await _run_tracked_load_model_impl(
request,
fastapi_request,
@@ -18715,12 +18934,30 @@ async def load_model_gated(
await asyncio.to_thread(model_slots.drop, extra)
await asyncio.to_thread(release_chat_gpu_claim)
finally:
+ _primary_can_retry_fit.set(False)
# A slot that would not stop stays stuck, but this attempt is over either way.
with _scoped_load_attempts_lock:
_pending_load_attempts.pop(attempt.token, None)
_finish_load_attempt(attempt)
+def _refuse_oom_beside_media_under_always() -> None:
+ """Under Always a chat load that ran out of memory beside an image or video model is refused, not retried."""
+ from core.inference import gpu_arbiter, gpu_residency
+
+ held = gpu_arbiter.residents()
+ chat_gpus = held.get(gpu_arbiter.CHAT)
+ # Only a model sharing a GPU with chat could have caused it; None (unknown) shares every GPU.
+ slots = [
+ s
+ for o in gpu_residency.MEDIA_OWNERS
+ if o in held and (chat_gpus is None or held[o] is None or held[o] & chat_gpus)
+ for s in gpu_arbiter.resident_slots(o)
+ ]
+ if slots and gpu_residency.current_policy() == "always":
+ raise gpu_arbiter.refusal_for(slots)
+
+
def _ran_out_of_memory(exc: HTTPException) -> bool:
if isinstance(exc.detail, dict): # a classified audio.cpp failure
# Out of CPU RAM: evicting GPU models frees none of it.
@@ -19807,7 +20044,9 @@ async def _load_model_impl(
)
if speech_codec_path is not None:
gguf_intent = replace(gguf_intent, audio_codec_path = speech_codec_path)
- if not replacing and not request.force_alongside:
+ if not request.force_alongside and (
+ not replacing or _primary_refuses_partial_fit(serving, multi_model_mode, placement)
+ ):
gguf_intent = replace(gguf_intent, refuse_partial_gpu_fit = True)
same_loaded_model = llama_backend.matches_load_source(gguf_intent)
_loaded_companion_roots = tuple(
@@ -20041,10 +20280,10 @@ async def _load_model_impl(
post_handoff_needed_gb = float(
(final_training_info or initial_training_info or {}).get("needed_gb") or required
)
- if expected_native_owner[0] in (DIFFUSION, VIDEO):
+ if final_availability.credited in (DIFFUSION, VIDEO):
post_media_handoff_gpu = selected_gpu
post_media_handoff_needed_gb = post_handoff_needed_gb
- elif expected_native_owner[0] == CHAT:
+ elif final_availability.credited == CHAT:
post_chat_handoff_expected_free_gb = {selected_gpu: post_handoff_needed_gb}
if chat_load_needs_gpu:
@@ -20057,6 +20296,18 @@ async def _load_model_impl(
if not replacing:
handoff_kwargs["alongside"] = True
handoff_kwargs.update(policy_kwargs(multi_model_mode))
+ outgoing_chat_gb, chat_wait_gb = await asyncio.to_thread(
+ _outgoing_chat_credit_gb,
+ placement,
+ multi_model_mode,
+ replacing,
+ final_availability if expected_native_owner is not None else None,
+ )
+ if post_chat_handoff_expected_free_gb is None:
+ post_chat_handoff_expected_free_gb = chat_wait_gb
+ handoff_kwargs.update(
+ _chat_fit_kwargs(config, placement, multi_model_mode, outgoing_chat_gb)
+ )
await asyncio.to_thread(
acquire_for_request,
CHAT,
@@ -20068,13 +20319,9 @@ async def _load_model_impl(
if isinstance(exc, OwnerChangedError):
raise HTTPException(status_code = 409, detail = str(exc)) from exc
raise
- # Wait for the freed VRAM only when the media model was actually evicted.
- if (
- post_media_handoff_gpu is not None
- and post_media_handoff_needed_gb is not None
- and not holds(expected_native_owner[0])
- ):
- settled = await asyncio.to_thread(
+ if post_media_handoff_gpu is not None and post_media_handoff_needed_gb is not None:
+ # Wait for the freed VRAM only when the media model was actually evicted.
+ settled = holds(expected_native_owner[0]) or await asyncio.to_thread(
_wait_for_native_audio_gpu_free_gb,
post_media_handoff_gpu,
post_media_handoff_needed_gb,
@@ -20087,9 +20334,10 @@ async def _load_model_impl(
"was stopped. Retry the audio-model load."
),
)
- # The outgoing media allocation is gone now. Re-run the training
- # policy against live free VRAM only, so a concurrent training
- # change cannot hide behind the earlier reclaimable snapshot.
+ # Re-run the training policy against live free VRAM only: the
+ # media allocation is gone, or was kept beside and never freed,
+ # and a concurrent training change cannot hide behind the
+ # earlier reclaimable snapshot.
live_placement = placement._replace(
# Automatic selection is already fixed at this point. Bind
# the live-only training recheck to that exact card, or a
@@ -20535,6 +20783,10 @@ async def _load_model_impl(
except Exception:
pass
+ if replacing and chat_load_needs_gpu:
+ # A GPU start that fell back to the CPU holds no VRAM, so the claim it took would evict media for nothing.
+ await asyncio.to_thread(release_chat_after_cpu_fallback, backend, config.identifier)
+
return LoadResponse(
engine = request.engine,
engine_parallelism = request.engine_parallelism,
@@ -50601,7 +50853,6 @@ async def load_diffusion_model_gated(
begin_media_load,
guarded_media_begin,
note_evictions,
- policy_kwargs,
refuse_replacing_media,
require_no_foreign_generations,
release,
@@ -50945,6 +51196,12 @@ async def load_diffusion_model_gated(
)
if needs_gpu:
+ from core.inference.gpu_residency import media_fit_kwargs
+
+ fit_kwargs = await asyncio.to_thread(
+ media_fit_kwargs, DIFFUSION, request, gpu_ordinal, multi_model_mode
+ )
+
# Register the in-flight load UNDER the arbiter lock: otherwise a competing acquire in that gap evicts DIFFUSION before
# the load is marked, finds nothing to cancel, and both allocate at once. The training admission wraps the same span.
def _acquire_and_begin():
@@ -50956,7 +51213,7 @@ async def load_diffusion_model_gated(
guarded_media_begin(
DIFFUSION, multi_model_mode, media_names, _begin_load
),
- **policy_kwargs(multi_model_mode),
+ **fit_kwargs,
)
finally:
eviction_sink.reset(sink_token)
diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.py
index 7b9ad7347f3..b4bcd90c4ef 100644
--- a/studio/backend/routes/video.py
+++ b/studio/backend/routes/video.py
@@ -343,7 +343,6 @@ async def load_video_model_gated(
acquire_for_request,
eviction_sink,
note_evictions,
- policy_kwargs,
begin_media_load,
guarded_media_begin,
refusal_for,
@@ -509,7 +508,13 @@ async def load_video_model_gated(
# Register the in-flight load UNDER the arbiter lock: otherwise a competing acquire in that gap evicts VIDEO
# before the load is marked, finds nothing to cancel, and both allocate at once. The training admission wraps
# the same span.
+ from core.inference.gpu_residency import media_fit_kwargs
from routes.inference import _diffusion_training_admission
+
+ fit_kwargs = await asyncio.to_thread(
+ media_fit_kwargs, VIDEO, request, gpu_ordinal, multi_model_mode
+ )
+
def _acquire_and_begin():
token = eviction_sink.set(evicted)
try:
@@ -517,7 +522,7 @@ async def load_video_model_gated(
return acquire_for_request(
VIDEO,
guarded_media_begin(VIDEO, multi_model_mode, media_names, _begin_load),
- **policy_kwargs(multi_model_mode),
+ **fit_kwargs,
)
finally:
eviction_sink.reset(token)
diff --git a/studio/backend/tests/test_audio_cpu_fallback_claim.py b/studio/backend/tests/test_audio_cpu_fallback_claim.py
new file mode 100644
index 00000000000..7d93106d809
--- /dev/null
+++ b/studio/backend/tests/test_audio_cpu_fallback_claim.py
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A chat load that took the GPU but fell back to the CPU drops CHAT once nothing holds VRAM.
+
+Otherwise the next image or video load evicts a model that never used the GPU.
+"""
+
+import inspect
+
+import pytest
+
+import core.inference.model_slots as model_slots
+import routes.inference as inf
+from core.inference import gpu_arbiter
+
+
+class FakeLlama:
+ holds_no_vram = False
+
+ def __init__(self, active = False):
+ self.is_loaded = self.is_active = active
+ self.model_identifier = "org/chat-GGUF" if active else None
+
+
+class FakeOrchestrator:
+ def __init__(
+ self,
+ active = None,
+ entry = None,
+ ):
+ self.active_model_name = active
+ self.models = {active: dict(entry or {})} if active else {}
+ self.loading_models = ()
+
+
+PRIMARY = "org/tts-audio-cpp"
+
+
+def evicted(monkeypatch):
+ calls = []
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ monkeypatch.setitem(gpu_arbiter._EVICTORS, gpu_arbiter.CHAT, lambda: calls.append("chat"))
+ monkeypatch.setitem(
+ gpu_arbiter._EVICTORS, gpu_arbiter.DIFFUSION, lambda: calls.append("diffusion")
+ )
+ monkeypatch.setattr(gpu_arbiter, "_release_idle_video_servers", lambda reason: None)
+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama())
+ monkeypatch.setattr(model_slots, "slots", [])
+ monkeypatch.setattr(model_slots, "stuck", [])
+ monkeypatch.setattr(model_slots, "loading", None)
+ return calls
+
+
+def _primary(entry):
+ return FakeOrchestrator(PRIMARY, entry)
+
+
+def test_a_cpu_fallback_primary_drops_chat_so_media_evicts_nothing(evicted):
+ assert inf.release_chat_after_cpu_fallback(_primary({"audio_device_fallback": "cpu"}), PRIMARY)
+ assert gpu_arbiter.current_owner() is None
+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)
+ assert evicted == [] and gpu_arbiter.current_owner() == gpu_arbiter.DIFFUSION
+
+
+ "entry", [{}, {"audio_device_fallback": None}, {"audio_device_fallback": "gpu"}]
+)
+def test_no_recorded_fallback_keeps_chat(evicted, entry):
+ assert not inf.release_chat_after_cpu_fallback(_primary(entry), PRIMARY)
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+
+
+def test_a_replaced_primary_keeps_chat(evicted):
+ backend = _primary({"audio_device_fallback": "cpu"})
+ backend.active_model_name = "org/other"
+ assert not inf.release_chat_after_cpu_fallback(backend, PRIMARY)
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+
+
+def test_a_kept_model_loading_keeps_chat(evicted, monkeypatch):
+ slot = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+ monkeypatch.setattr(model_slots, "loading", (slot, "org/kept-GGUF"))
+ assert not inf.release_chat_after_cpu_fallback(
+ _primary({"audio_device_fallback": "cpu"}), PRIMARY
+ )
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+
+
+def test_a_kept_model_on_the_gpu_keeps_chat(evicted, monkeypatch):
+ kept = model_slots.ExtraSlot(FakeLlama(active = True), FakeOrchestrator(), "owner")
+ monkeypatch.setattr(model_slots, "slots", [kept])
+ assert model_slots.holds_vram()
+ assert not inf.release_chat_after_cpu_fallback(
+ _primary({"audio_device_fallback": "cpu"}), PRIMARY
+ )
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+ # A CPU-only kept model holds no VRAM, so the claim goes.
+ kept.llama._gpu_offload_active = False
+ assert inf.release_chat_after_cpu_fallback(_primary({"audio_device_fallback": "cpu"}), PRIMARY)
+
+
+def test_a_primary_llama_server_on_the_gpu_keeps_chat(evicted, monkeypatch):
+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama(active = True))
+ assert not inf.release_chat_after_cpu_fallback(
+ _primary({"audio_device_fallback": "cpu"}), PRIMARY
+ )
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+
+
+def test_a_kept_audio_cpp_fallback_beside_a_gpu_primary_keeps_chat(evicted, monkeypatch):
+ # The kept model's own orchestrator is not the primary: the call site is gated on `replacing`,
+ # and even if it ran, the primary llama-server's VRAM keeps the claim.
+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama(active = True))
+ kept = FakeOrchestrator("org/kept-audio-cpp", {"audio_device_fallback": "cpu"})
+ monkeypatch.setattr(model_slots, "slots", [model_slots.ExtraSlot(FakeLlama(), kept, "owner")])
+ assert not inf.release_chat_after_cpu_fallback(kept, "org/kept-audio-cpp")
+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT
+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)
+ assert evicted == ["chat"]
+
+
+def test_the_release_runs_for_a_primary_gpu_load_before_the_response():
+ source = inspect.getsource(inf._load_model_impl)
+ call = "await asyncio.to_thread(release_chat_after_cpu_fallback, backend, config.identifier)"
+ assert source.count(call) == 1
+ gate = "if replacing and chat_load_needs_gpu:\n"
+ assert source.count(gate) == 1
+ at = source.index(call)
+ # The gate opens the block, then only the comment line sits between it and the call.
+ assert source[source.index(gate) + len(gate) : at].strip().startswith("#")
+ assert source[source.index(gate) + len(gate) : at].count("\n") == 1
+ # The standard path's response statement follows directly, so the call is never inside its arguments.
+ after = source[at + len(call) :]
+ assert after.lstrip().startswith("return LoadResponse(")
diff --git a/studio/backend/tests/test_gpu_arbiter.py b/studio/backend/tests/test_gpu_arbiter.py
index 84b51914744..35035e5dc87 100644
--- a/studio/backend/tests/test_gpu_arbiter.py
+++ b/studio/backend/tests/test_gpu_arbiter.py
@@ -61,6 +61,347 @@ def test_reacquiring_same_owner_does_not_evict(calls):
assert arb.current_owner() == arb.CHAT
+# Keep multiple models loaded set to When they fit / Always (gpu_residency.coreside_locked) next to Off.
+
+
+def cores(calls, monkeypatch):
+ import sys
+ import types
+
+ import core.inference.gpu_residency as res
+ import utils.hardware.hardware # noqa: F401
+ from core.inference import model_slots, video_minimax_h3
+ from core.inference.api_monitor import api_monitor
+ from state import active_generations
+
+ env = types.SimpleNamespace(calls = calls, lifecycle = [], h3 = [], probes = [])
+ for name, value in {
+ "_residents": {},
+ "_media_needs": {},
+ "_accounts": {},
+ "_owner_account": None,
+ "_prior_account": None,
+ }.items():
+ monkeypatch.setattr(arb, name, value)
+ monkeypatch.setitem(arb._EVICTORS, arb.VIDEO, lambda: calls.append("evict-video"))
+ for owner, kind in ((arb.CHAT, "chat"), (arb.DIFFUSION, "image"), (arb.VIDEO, "video")):
+ monkeypatch.setitem(
+ arb._SLOT_REPORTERS, owner, lambda kind = kind: [{"kind": kind, "model": f"org/{kind}"}]
+ )
+ monkeypatch.setattr(arb, "_release_idle_video_servers", env.h3.append)
+ monkeypatch.setattr(
+ sys.modules["utils.hardware.hardware"], "get_parent_visible_gpu_ids", lambda: []
+ )
+ monkeypatch.setattr(res, "chat_load_in_flight", lambda: False)
+ monkeypatch.setattr(res, "_media_busy", lambda owner: (False, False))
+ monkeypatch.setattr(res, "_loaded_or_loading", lambda owner: True)
+ rows = {gpu: {"free_mib": 8192, "total_mib": 24576} for gpu in (0, 1)}
+ monkeypatch.setattr(res, "_free_mib_by_gpu", lambda: (rows, None))
+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *args, **kwargs: [])
+ monkeypatch.setattr(video_minimax_h3, "release_h3_native_servers", lambda reason: 0)
+ monkeypatch.setattr(
+ api_monitor, "record_lifecycle", lambda **event: env.lifecycle.append(event)
+ )
+
+ def fits(*blockers):
+ # Short while any of ``blockers`` is still resident, so a re-check after an eviction passes.
+ def probe(owner, need_mib):
+ env.probes.append(owner)
+ short = any(name in arb._view() for name in blockers)
+ return not short, {
+ "need_mib": {0: 6144},
+ "free_mib": {0: 2048},
+ "short_mib": {0: 5120} if short else {},
+ "reason": "short" if short else "fits",
+ }
+
+ monkeypatch.setattr(res, "can_load_alongside", probe)
+
+ def no_probe(owner, need_mib):
+ raise AssertionError("can_load_alongside must not run here")
+
+ env.fits = fits
+ env.no_probe = lambda: monkeypatch.setattr(res, "can_load_alongside", no_probe)
+ active_generations.reset_for_tests()
+ yield env
+ active_generations.reset_for_tests()
+
+
+CHAT_SLOT, IMAGE_SLOT = (
+ {"kind": "chat", "model": "org/chat"},
+ {"kind": "image", "model": "org/image"},
+)
+
+
+ "policy, need, short, evicts",
+ [
+ ("off", {0: 4096}, False, True), # today's swap
+ ("fit", {0: 4096}, False, False), # fits: both stay
+ ("fit", {0: 4096}, True, True), # short: chat freed, cheapest first
+ ("fit", None, False, True), # unpriced: evict first, as Off does
+ ("always", {0: 4096}, False, False), # fits: joins
+ ],
+)
+def test_image_load_beside_a_resident_chat_model_policy_matrix(cores, policy, need, short, evicts):
+ cores.fits(*([arb.CHAT] if short else []))
+ arb.acquire_for(arb.CHAT, gpu_ids = [0])
+ evicted: list = []
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, need_mib = need, evicted = evicted)
+ assert cores.calls == (["evict-chat"] if evicts else [])
+ assert set(arb.residents()) == ({arb.DIFFUSION} if evicts else {arb.CHAT, arb.DIFFUSION})
+ assert evicted == ([CHAT_SLOT] if evicts else [])
+ # The fit path records each eviction in the API monitor; Off keeps today's silent swap.
+ assert [event["reason"] for event in cores.lifecycle] == (
+ ["evicted"] if evicts and policy != "off" else []
+ )
+ assert arb._media_needs.get(arb.DIFFUSION) == (need if policy != "off" else None)
+
+
[email protected]("managed", [False, True])
+def test_always_refuses_an_image_load_that_does_not_fit_without_evicting(cores, managed):
+ from utils.account_context import AccountContext, run_as
+
+ cores.fits(arb.CHAT)
+ arb.acquire_for(arb.CHAT, gpu_ids = [0])
+
+ def load():
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "always", need_mib = {0: 6144})
+
+ with pytest.raises(arb.MultiModelAlwaysRefusal) as refused:
+ run_as(AccountContext("b" * 32, "bob"), load) if managed else load()
+ message = str(refused.value)
+ if managed:
+ assert "is set to keep loaded" in message and "GB" not in message
+ else:
+ assert message.startswith("This model needs about 6.0 GB of GPU memory, and 2.0 GB is free")
+ assert "is set to Always" in message
+ assert cores.calls == [] and cores.lifecycle == []
+ assert set(arb.residents()) == {arb.CHAT}
+
+
[email protected]("policy", ["off", "fit", "always"])
+def test_image_load_on_another_gpu_never_evicts_chat_unless_off(cores, policy):
+ cores.no_probe()
+ arb.acquire_for(arb.CHAT, gpu_ids = [0])
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, need_mib = {1: 4096})
+ if policy == "off":
+ assert cores.calls == ["evict-chat"]
+ assert arb.residents() == {arb.DIFFUSION: frozenset({1})}
+ else:
+ assert cores.calls == []
+ assert arb.residents() == {arb.CHAT: frozenset({0}), arb.DIFFUSION: frozenset({1})}
+
+
[email protected]("policy", ["fit", "always"])
+def test_chat_load_on_another_gpu_keeps_an_unpriced_image_model(cores, policy):
+ # Unpriced media on GPU 1 can't be counted, but a chat load pinned to GPU 0 never touches it.
+ arb.acquire_for(arb.DIFFUSION, lambda: None, gpu_ids = [1])
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = policy, evict_first = True)
+ assert cores.calls == []
+ assert arb.residents() == {arb.DIFFUSION: frozenset({1}), arb.CHAT: frozenset({0})}
+
+
[email protected]("policy", ["fit", "always"])
+def test_unpriced_image_load_on_another_gpu_keeps_chat(cores, policy):
+ # sd.cpp or H3 can't be priced, but a load pinned to GPU 1 still leaves chat on GPU 0 alone.
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = policy)
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, gpu_ids = [1])
+ assert cores.calls == []
+ assert arb.residents() == {arb.CHAT: frozenset({0}), arb.DIFFUSION: frozenset({1})}
+
+
+def test_media_fit_kwargs_keep_the_gpu_without_a_price(monkeypatch):
+ import core.inference.gpu_residency as res
+
+ priced = res.MediaNeed({1: 4096}, weights_mib = 3000, headroom_mib = 1096, kind = "image")
+ need = {"value": None}
+ monkeypatch.setattr(res, "media_load_need_mib", lambda *a: need["value"])
+ assert res.media_fit_kwargs(arb.DIFFUSION, None, 1, "off") == {}
+ assert res.media_fit_kwargs(arb.DIFFUSION, None, 1, "fit") == {"policy": "fit", "gpu_ids": [1]}
+ assert res.media_fit_kwargs(arb.VIDEO, None, None, "always") == {"policy": "always"}
+ need["value"] = priced
+ assert res.media_fit_kwargs(arb.DIFFUSION, None, 1, "fit") == {
+ "policy": "fit",
+ "gpu_ids": [1],
+ "need_mib": priced,
+ }
+
+
+def test_chat_gpus_add_up_across_loads_and_survive_a_reassert(cores):
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = "fit")
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [1], policy = "fit", alongside = True)
+ assert arb.residents() == {arb.CHAT: frozenset({0, 1})}
+ arb.acquire_for(arb.CHAT, policy = "fit") # a re-assert places nothing new
+ assert arb.residents() == {arb.CHAT: frozenset({0, 1})}
+ arb.acquire_for(
+ arb.CHAT, lambda: None, policy = "fit"
+ ) # a load on unknown GPUs counts as every GPU
+ assert arb.residents() == {arb.CHAT: None}
+
+
+def test_primary_chat_replacement_drops_the_gpus_it_left(cores, monkeypatch):
+ from core.inference import model_slots
+
+ # Replacing the primary with nothing kept alongside leaves no chat model on the old GPU.
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = "fit")
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [1], policy = "fit")
+ assert arb.residents() == {arb.CHAT: frozenset({1})}
+ arb.acquire_for(arb.CHAT, lambda: None, policy = "fit")
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [1], policy = "fit")
+ assert arb.residents() == {arb.CHAT: frozenset({1})}
+ # A model kept alongside still holds GPU 1, so a primary moved to GPU 0 adds up.
+ monkeypatch.setattr(model_slots, "holds_vram", lambda: True)
+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = "fit")
+ assert arb.residents() == {arb.CHAT: frozenset({0, 1})}
+
+
+ "policy, priced, kwargs, short, outcome",
+ [
+ ("fit", True, {}, False, "join"), # the launch prices the idle image model
+ ("always", True, {}, False, "join"),
+ ("fit", False, {}, False, "evict"), # unpriced media can't be counted
+ ("always", False, {}, False, "refuse"),
+ ("fit", True, {"evict_first": True}, False, "evict"), # no estimate before launch
+ (
+ "fit",
+ True,
+ {"need_mib": {0: 6144}},
+ False,
+ "join",
+ ), # audio.cpp, priced from native_required_gb
+ ("fit", True, {"need_mib": {0: 6144}}, True, "evict"),
+ ],
+)
+def test_chat_load_beside_an_idle_image_model(cores, policy, priced, kwargs, short, outcome):
+ cores.fits(*([arb.DIFFUSION] if short else []))
+ arb.acquire_for(
+ arb.DIFFUSION,
+ lambda: None,
+ policy = "fit" if priced else "off",
+ need_mib = {0: 4096} if priced else None,
+ gpu_ids = [0],
+ )
+ assert (arb.DIFFUSION in arb._media_needs) is priced
+ evicted: list = []
+
+ def load():
+ arb.acquire_for(arb.CHAT, gpu_ids = [0], policy = policy, evicted = evicted, **kwargs)
+
+ if outcome == "refuse":
+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always"):
+ load()
+ else:
+ load()
+ assert cores.calls == (["evict-diffusion"] if outcome == "evict" else [])
+ assert evicted == ([IMAGE_SLOT] if outcome == "evict" else [])
+ expected = {"join": {arb.CHAT, arb.DIFFUSION}, "evict": {arb.CHAT}, "refuse": {arb.DIFFUSION}}[
+ outcome
+ ]
+ assert set(arb.residents()) == expected
+
+
+ "policy, need, refused",
+ [
+ ("fit", {1: 4096}, False), # Bob's chat is on GPU 0: nothing of his is evicted
+ ("fit", {0: 4096}, True), # short on Bob's GPU: his stream blocks the eviction
+ ("off", {1: 4096}, True), # Off keeps the global guard
+ ],
+)
+def test_another_accounts_stream_only_blocks_evicting_its_own_resident(
+ cores, policy, need, refused
+):
+ import threading
+
+ from state.active_generations import ActiveGeneration
+ from utils.account_context import AccountContext, run_as
+
+ alice, bob = AccountContext("a" * 32, "alice"), AccountContext("b" * 32, "bob")
+ cores.fits(arb.CHAT)
+ run_as(bob, arb.acquire_for, arb.CHAT, lambda: None, gpu_ids = [0])
+ with ActiveGeneration(threading.Event(), run_id = "bob-chat", account_id = bob.account_id):
+
+ def load():
+ run_as(
+ alice, arb.acquire_for, arb.DIFFUSION, lambda: None, policy = policy, need_mib = need
+ )
+
+ if refused:
+ with pytest.raises(arb.GpuBusyForAnotherAccountError):
+ load()
+ else:
+ load()
+ assert cores.calls == []
+ assert arb.holds(arb.CHAT) and arb.holds(arb.DIFFUSION) is not refused
+ assert arb.owner_account(arb.CHAT) == bob.account_id
+
+
+def test_free_for_collects_what_it_evicts_and_spares_other_gpus(cores):
+ import core.inference.gpu_residency as res
+
+ cores.fits(arb.CHAT)
+ arb.acquire_for(arb.CHAT, gpu_ids = [0])
+ arb.acquire_for(arb.VIDEO, lambda: None, policy = "fit", need_mib = {1: 4096})
+ evicted: list = []
+ freed = res.free_for(arb.DIFFUSION, [0], "test", evicted, need_mib = {0: 6144})
+ assert freed == ["org/chat"] and evicted == [CHAT_SLOT]
+ assert cores.calls == ["evict-chat"]
+ assert set(arb.residents()) == {arb.VIDEO}
+ assert cores.lifecycle == [{"event": "unload", "model": "org/chat", "reason": "evicted"}]
+
+
[email protected]("video_resident", [True, False])
+def test_a_fit_join_keeps_the_video_server_while_video_is_resident(cores, video_resident):
+ cores.fits()
+ arb.acquire_for(arb.CHAT, gpu_ids = [0])
+ if video_resident:
+ arb.acquire_for(arb.VIDEO, lambda: None, policy = "fit", need_mib = {1: 4096})
+ cores.h3.clear()
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})
+ assert cores.calls == []
+ assert cores.h3 == ([] if video_resident else [f"GPU acquired for {arb.DIFFUSION}"])
+
+
[email protected]("policy", ["always", "fit"])
+def test_image_load_behind_an_in_flight_chat_load(cores, monkeypatch, policy):
+ import core.inference.gpu_residency as res
+ from fastapi import HTTPException
+
+ cores.fits(arb.CHAT)
+ monkeypatch.setattr(res, "chat_load_in_flight", lambda: True)
+ arb.acquire_for(arb.CHAT, gpu_ids = [0])
+
+ def load():
+ arb.acquire_for_request(arb.DIFFUSION, lambda: None, policy = policy, need_mib = {0: 4096})
+
+ if policy == "always":
+ # Its size isn't known yet, so Always answers the retryable K3 409 instead of the Always refusal.
+ with pytest.raises(HTTPException) as refused:
+ load()
+ assert refused.value.status_code == 409
+ detail = refused.value.detail
+ assert detail["error"] == "gpu_busy" and detail["retry_after"] == 15
+ assert detail["message"].startswith("Unsloth Studio is loading a chat model")
+ assert refused.value.headers == {"Retry-After": "15"}
+ assert cores.calls == [] and set(arb.residents()) == {arb.CHAT}
+ else:
+ load() # today's handoff
+ assert cores.calls == ["evict-chat"] and set(arb.residents()) == {arb.DIFFUSION}
+
+
[email protected]("owner", [arb.DIFFUSION, arb.VIDEO])
+def test_a_lone_media_load_under_fit_never_probes(cores, owner):
+ cores.no_probe()
+ for _ in range(2): # a reload of the same owner has no other resident either
+ arb.acquire_for(owner, lambda: None, policy = "fit", need_mib = {0: 4096})
+ assert cores.calls == []
+ assert arb.residents() == {owner: frozenset({0})}
+ assert arb._media_needs[owner] == {0: 4096}
+
+
def test_release_clears_owner(calls):
arb.acquire_for(arb.DIFFUSION)
arb.release(arb.DIFFUSION)
diff --git a/studio/backend/tests/test_gpu_arbitration_sim.py b/studio/backend/tests/test_gpu_arbitration_sim.py
index 13ea2e482ae..7bd7ac96354 100644
--- a/studio/backend/tests/test_gpu_arbitration_sim.py
+++ b/studio/backend/tests/test_gpu_arbitration_sim.py
@@ -10,6 +10,7 @@ import itertools
import sqlite3
import threading
import time
+from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -716,6 +717,411 @@ def test_cpu_media_drain_does_not_refuse_unrelated_gpu_work(sim, media_switch, m
sim.finish("a")
+# Keep multiple models loaded (When they fit / Always) against a fake VRAM ledger: the arbiter, gpu_residency and the
+# chat pricing helpers are real; only the GPU probe, the evictors and the slot reports are fakes.
+GIB = 1024
+
+
+class VramSim:
+ def __init__(self, monkeypatch, totals_gb):
+ import utils.hardware as hardware
+ from core.inference import gpu_residency, model_slots, video_minimax_h3
+ from utils.hardware import hardware as hardware_impl
+
+ self.residency = gpu_residency
+ self.totals = {gpu: gb * GIB for gpu, gb in enumerate(totals_gb)}
+ self.footprint: dict[str, dict[int, int]] = {}
+ self.evictions: list[str] = []
+ self.probes = 0
+ self.index_kind = "physical"
+ self.parent = sorted(self.totals)
+ self.policy = "off"
+ for owner in BACKENDS:
+ monkeypatch.setitem(arb._EVICTORS, owner, lambda o = owner: self._evict(o))
+ monkeypatch.setitem(
+ arb._SLOT_REPORTERS,
+ owner,
+ lambda o = owner: [
+ {"kind": arb.KIND_LABELS[o], "model": f"{arb.KIND_LABELS[o]}-model"}
+ ],
+ )
+ monkeypatch.setattr(arb, "_residents", {})
+ monkeypatch.setattr(arb, "_accounts", {})
+ monkeypatch.setattr(arb, "_media_needs", {})
+ monkeypatch.setattr(arb, "_release_idle_video_servers", lambda reason: None)
+ monkeypatch.setattr(video_minimax_h3, "release_h3_native_servers", lambda reason: 0)
+ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
+ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", self._utilization)
+ monkeypatch.setattr(hardware_impl, "get_parent_visible_gpu_ids", lambda: list(self.parent))
+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: self.policy)
+ monkeypatch.setattr(gpu_residency, "_media_busy", lambda owner: (False, False))
+ monkeypatch.setattr(gpu_residency, "_loaded_or_loading", lambda owner: True)
+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: False)
+ # The diffusion planner's own verdict is covered in test_gpu_residency_coexistence.py.
+ monkeypatch.setattr(gpu_residency, "_keeps_transformer_resident", lambda *a: True)
+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *a, **k: [])
+
+ def _utilization(self):
+ self.probes += 1
+ return {
+ "index_kind": self.index_kind,
+ "devices": [
+ {"index": gpu, "vram_total_gb": total / GIB, "vram_used_gb": self.used(gpu) / GIB}
+ for gpu, total in self.totals.items()
+ ],
+ }
+
+ def used(self, gpu):
+ return sum(held.get(gpu, 0) for held in self.footprint.values())
+
+ def _evict(self, owner):
+ self.evictions.append(owner)
+ self.footprint.pop(owner, None)
+
+ def load(self, owner, held, **kwargs):
+ evicted: list = []
+
+ def register():
+ self.footprint[owner] = dict(held)
+
+ arb.acquire_for(owner, register, evicted = evicted, **kwargs)
+ return evicted
+
+ def load_image(
+ self,
+ need_gb,
+ gpu = 0,
+ headroom_gb = 4,
+ **kwargs,
+ ):
+ """An image load priced at ``need_gb``; idle, only its weights are allocated (the headroom is not)."""
+ need, headroom = need_gb * GIB, headroom_gb * GIB
+ priced = self.residency.MediaNeed(
+ {gpu: need}, weights_mib = need - headroom, headroom_mib = headroom, kind = "image"
+ )
+ return self.load(arb.DIFFUSION, {gpu: need - headroom}, need_mib = priced, **kwargs)
+
+ def load_chat(
+ self,
+ gb = 10,
+ gpu_ids = (0,),
+ **kwargs,
+ ):
+ held = {gpu: gb * GIB // max(len(gpu_ids or (0,)), 1) for gpu in (gpu_ids or (0,))}
+ return self.load(
+ arb.CHAT, held, gpu_ids = list(gpu_ids) if gpu_ids is not None else None, **kwargs
+ )
+
+
+def vram(monkeypatch):
+ return lambda *totals_gb: VramSim(monkeypatch, totals_gb)
+
+
+CHAT_SLOT = {"kind": "chat", "model": "chat-model"}
+IMAGE_SLOT = {"kind": "image", "model": "image-model"}
+
+
[email protected]("policy", ["fit", "always"])
+def test_image_joins_chat_on_one_gpu_when_it_fits(vram, policy):
+ v = vram(80)
+ v.load_chat(10)
+ assert v.load_image(20, policy = policy) == []
+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}
+ assert v.residency.media_held_mib() == {0: 20 * GIB}
+ assert v.used(0) == 26 * GIB # 10 GB chat + 16 GB idle image weights
+
+
+def test_image_evicts_chat_on_one_gpu_when_short_under_fit(vram):
+ v = vram(80)
+ v.load_chat(10)
+ assert v.load_image(75, policy = "fit") == [CHAT_SLOT]
+ assert v.evictions == [arb.CHAT] and set(arb.residents()) == {arb.DIFFUSION}
+ assert arb.CHAT not in v.footprint
+
+
+def test_image_short_on_one_gpu_under_always_refuses_and_keeps_chat(vram):
+ v = vram(80)
+ v.load_chat(10)
+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always"):
+ v.load_image(75, policy = "always")
+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT}
+ assert v.footprint == {arb.CHAT: {0: 10 * GIB}} and v.residency.media_held_mib() == {}
+
+
[email protected]("policy", ["fit", "always"])
+def test_image_on_another_gpu_never_evicts_or_probes(vram, policy):
+ v = vram(80, 80)
+ v.load_chat(10, gpu_ids = (0,))
+ assert v.load_image(75, gpu = 1, policy = policy) == []
+ assert v.evictions == [] and v.probes == 0
+ assert arb.residents() == {arb.CHAT: frozenset({0}), arb.DIFFUSION: frozenset({1})}
+
+
+def test_a_refused_image_registration_keeps_the_loaded_image_on_its_gpu(vram):
+ v = vram(80, 80)
+ v.load_image(20, gpu = 0, policy = "always")
+
+ def refuse():
+ raise arb.MultiModelAlwaysRefusal("another image model is loaded")
+
+ with pytest.raises(arb.MultiModelAlwaysRefusal):
+ arb.acquire_for(arb.DIFFUSION, refuse, policy = "always", gpu_ids = [1])
+ assert arb.residents()[arb.DIFFUSION] == frozenset({0})
+ # A chat load on GPU 0 still sees the image there.
+ with pytest.raises(arb.MultiModelAlwaysRefusal):
+ v.load_chat(70, gpu_ids = (0,), policy = "always", evict_first = True)
+ assert v.evictions == [] and set(arb.residents()) == {arb.DIFFUSION}
+
+
+def test_image_on_another_gpu_under_off_still_swaps(vram):
+ v = vram(80, 80)
+ v.load_chat(10, gpu_ids = (0,))
+ assert v.load_image(20, gpu = 1) == [CHAT_SLOT]
+ assert v.evictions == [arb.CHAT] and set(arb.residents()) == {arb.DIFFUSION}
+
+
+def _chat_kwargs(
+ config,
+ placement,
+ *,
+ primary_can_retry = False,
+):
+ from routes import inference as inf
+ token = inf._primary_can_retry_fit.set(primary_can_retry)
+ try:
+ return inf._chat_fit_kwargs(config, placement)
+ finally:
+ inf._primary_can_retry_fit.reset(token)
+
+
+def _placement(
+ gpus = (0,),
+ *,
+ vulkan = False,
+ native_gb = None,
+):
+ from routes.inference import _LoadPlacement
+ resolved = list(gpus) if gpus is not None else None
+ return _LoadPlacement(resolved, resolved, vulkan, None, native_required_gb = native_gb)
+
+
+def test_gguf_chat_joins_an_idle_image_and_its_launch_prices_the_image(vram):
+ from core.inference import llama_cpp
+
+ v = vram(80, 80)
+ v.policy = "fit"
+ v.load_image(20, policy = "fit")
+ placement = _placement((0,))
+ kwargs = _chat_kwargs(SimpleNamespace(is_gguf = True), placement, primary_can_retry = True)
+ assert kwargs == {"policy": "fit", "gpu_ids": [0]}
+ assert v.load_chat(10, **kwargs) == []
+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}
+
+ backend = llama_cpp.LlamaCppBackend(manages_processes = False)
+ llama_cpp.register_serving_backend(backend)
+ try:
+ held = backend._other_planned_vram_mib()
+ finally:
+ llama_cpp.unregister_serving_backend(backend)
+ assert held == {
+ 0: 20 * GIB
+ } # weights plus the headroom the idle image will allocate when it generates
+ # A stale free reading on GPU 0 is capped at total less what the image holds; GPU 1 is left alone.
+ probed = [(0, 80 * GIB, 80 * GIB), (1, 70 * GIB, 80 * GIB)]
+ assert llama_cpp._net_of_held_vram(probed, held) == [
+ (0, 60 * GIB, 80 * GIB),
+ (1, 70 * GIB, 80 * GIB),
+ ]
+
+
+def test_gguf_chat_sizing_keeps_an_idle_image_headroom_when_another_process_uses_the_gpu(vram):
+ from core.inference import llama_cpp
+
+ v = vram(80)
+ v.policy = "fit"
+ v.load_image(24, headroom_gb = 8, policy = "fit") # 16 GB weights allocated, 8 GB headroom not yet
+ v.footprint["other process"] = {0: 20 * GIB}
+ probed = [(0, v.totals[0] - v.used(0), v.totals[0])]
+ assert probed == [(0, 44 * GIB, 80 * GIB)]
+ backend = llama_cpp.LlamaCppBackend(manages_processes = False)
+ llama_cpp.register_serving_backend(backend)
+ try:
+ held = backend._other_planned_vram_mib()
+ unallocated = backend._media_unallocated_vram_mib()
+ finally:
+ llama_cpp.unregister_serving_backend(backend)
+ sized = llama_cpp._net_of_other_models_vram(probed, held, unallocated)
+ # 44 GB is free, but 8 GB of it is the image's generation budget.
+ assert sized == [(0, 36 * GIB, 80 * GIB)]
+ launch = " ".join(Path(llama_cpp.__file__).read_text(encoding = "utf-8").split())
+ assert (
+ "_gpu_mem = _net_of_other_models_vram( _gpu_mem, _held_by_others, self._media_unallocated_vram_mib() )"
+ in launch
+ )
+
+
[email protected]("loading", [False, True])
+def test_gguf_chat_sizing_charges_the_image_once_on_an_otherwise_empty_gpu(
+ vram, monkeypatch, loading
+):
+ """Nothing else on the card: the total cap alone charges the image's full price, never twice."""
+ from core.inference import llama_cpp
+
+ v = vram(80)
+ v.policy = "fit"
+ v.load_image(24, headroom_gb = 8, policy = "fit")
+ if loading:
+ monkeypatch.setattr(v.residency, "_media_busy", lambda owner: (True, False))
+ v.footprint.pop(arb.DIFFUSION) # nothing allocated yet
+ probed = [(0, v.totals[0] - v.used(0), v.totals[0])]
+ backend = llama_cpp.LlamaCppBackend(manages_processes = False)
+ llama_cpp.register_serving_backend(backend)
+ try:
+ held = backend._other_planned_vram_mib()
+ unallocated = backend._media_unallocated_vram_mib()
+ finally:
+ llama_cpp.unregister_serving_backend(backend)
+ sized = llama_cpp._net_of_other_models_vram(probed, held, unallocated)
+ assert sized == [(0, 56 * GIB, 80 * GIB)]
+
+
+def test_a_loading_image_is_held_in_full_by_chat_sizing(vram, monkeypatch):
+ from core.inference import llama_cpp
+
+ v = vram(80)
+ v.load_image(24, headroom_gb = 8, policy = "fit")
+ monkeypatch.setattr(v.residency, "_media_busy", lambda owner: (True, False))
+ assert v.residency.media_unallocated_mib() == {0: 24 * GIB}
+ assert llama_cpp._net_of_unallocated_vram([(0, 60 * GIB, 0)], {0: 24 * GIB}) == [
+ (0, 60 * GIB, 0)
+ ] # an integrated GPU's reading already counts shared memory
+
+
[email protected]("alongside", [False, True])
+def test_chat_reasserted_behind_an_image_keeps_the_account_that_loaded_it(vram, alongside):
+ """Another account reusing the chat model (or adding a slot) while an image sits beside it does not take it."""
+ v = vram(80)
+ v.policy = "fit"
+ run_as(ALICE, v.load_chat, 10)
+ assert run_as(ALICE, v.load_image, 20, policy = "fit") == []
+ assert arb.owner_account(arb.CHAT) == ALICE.account_id
+ if alongside:
+ run_as(BOB, v.load_chat, 5, policy = "fit", alongside = True)
+ else:
+ run_as(BOB, arb.acquire_for_request, arb.CHAT, policy = "fit")
+ assert set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}
+ assert arb.owner_account(arb.CHAT) == ALICE.account_id
+ assert arb.owner_account(arb.DIFFUSION) == ALICE.account_id
+
+
+def test_gguf_chat_with_no_fit_retry_evicts_first_under_fit(vram):
+ v = vram(80)
+ v.policy = "fit"
+ assert _chat_kwargs(SimpleNamespace(is_gguf = True), _placement()) == {
+ "policy": "fit",
+ "gpu_ids": [0],
+ "evict_first": True,
+ }
+
+
+ ("policy", "expected", "evicted"),
+ [
+ ("fit", {"policy": "fit", "gpu_ids": [0], "evict_first": True}, [IMAGE_SLOT]),
+ ("always", {"policy": "always", "gpu_ids": [0]}, []),
+ ],
+)
+def test_non_gguf_chat_beside_an_image(vram, policy, expected, evicted):
+ v = vram(80)
+ v.policy = policy
+ v.load_image(20, policy = policy)
+ placement = _placement((0,))
+ kwargs = _chat_kwargs(SimpleNamespace(is_gguf = False), placement, primary_can_retry = True)
+ assert kwargs == expected
+ assert v.load_chat(10, **kwargs) == evicted
+ assert v.evictions == [arb.DIFFUSION for _ in evicted]
+ assert arb.CHAT in arb.residents() and (arb.DIFFUSION in arb.residents()) == (not evicted)
+
+
+def _audio_cpp_chat(v, native_gb, **placement_kw):
+ placement = _placement(native_gb = native_gb, **placement_kw)
+ kwargs = _chat_kwargs(SimpleNamespace(is_gguf = False), placement)
+ gpus = None if placement.gpu_ids_are_vulkan_ordinals else placement.resolved_gpu_ids
+ return kwargs, v.load(arb.CHAT, {0: int(native_gb * GIB)}, **{"gpu_ids": gpus, **kwargs})
+
+
[email protected]("policy", ["fit", "always"])
+def test_audio_cpp_chat_joins_a_resident_image_when_native_estimate_fits(vram, policy):
+ v = vram(80)
+ v.policy = policy
+ v.load_image(20, policy = policy)
+ kwargs, evicted = _audio_cpp_chat(v, 6.0)
+ assert kwargs == {"policy": policy, "gpu_ids": [0], "need_mib": {0: 6144}}
+ assert evicted == [] and v.evictions == [] and set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}
+
+
+def test_audio_cpp_chat_short_beside_an_image_evicts_it_under_fit(vram):
+ v = vram(80)
+ v.policy = "fit"
+ v.load_image(20, policy = "fit")
+ kwargs, evicted = _audio_cpp_chat(v, 70.0)
+ assert kwargs["need_mib"] == {0: 70 * GIB}
+ assert evicted == [IMAGE_SLOT] and v.evictions == [arb.DIFFUSION]
+ assert set(arb.residents()) == {arb.CHAT} and v.residency.media_held_mib() == {}
+
+
+def test_audio_cpp_chat_short_beside_an_image_refuses_under_always(vram):
+ v = vram(80)
+ v.policy = "always"
+ v.load_image(20, policy = "always")
+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always"):
+ _audio_cpp_chat(v, 70.0)
+ assert v.evictions == [] and set(arb.residents()) == {arb.DIFFUSION}
+
+
[email protected]("placement_kw", [{"vulkan": True}, {"gpus": None}])
+def test_audio_cpp_chat_without_physical_gpus_evicts_first_under_fit(vram, placement_kw):
+ v = vram(80)
+ v.policy = "fit"
+ v.load_image(20, policy = "fit")
+ kwargs, evicted = _audio_cpp_chat(v, 6.0, **placement_kw)
+ assert kwargs == {"policy": "fit", "evict_first": True}
+ assert evicted == [IMAGE_SLOT] and v.evictions == [arb.DIFFUSION]
+
+
+def _masked(v):
+ # A UUID/MIG mask: rows carry relative indices and there are no numeric parent ids.
+ v.index_kind = "relative"
+ v.parent = []
+ v.load_chat(10, gpu_ids = None)
+
+
[email protected]("priced", [False, True])
+def test_masked_host_image_beside_chat_evicts_first_under_fit(vram, priced):
+ v = vram(80)
+ _masked(v)
+ assert v.residency.media_load_need_mib(arb.DIFFUSION, SimpleNamespace(), None) is None
+ if priced:
+ assert v.load_image(20, policy = "fit") == [CHAT_SLOT]
+ else:
+ assert v.load(arb.DIFFUSION, {0: 16 * GIB}, policy = "fit") == [CHAT_SLOT]
+ assert v.evictions == [arb.CHAT] and set(arb.residents()) == {arb.DIFFUSION}
+
+
[email protected]("priced", [False, True])
+def test_masked_host_image_beside_chat_refuses_under_always(vram, priced):
+ v = vram(80)
+ _masked(v)
+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always") as refused:
+ if priced:
+ v.load_image(20, policy = "always")
+ else:
+ v.load(arb.DIFFUSION, {0: 16 * GIB}, policy = "always")
+ assert refused.value.kind == "chat" and refused.value.model == "chat-model"
+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT}
+
+
@pytest.mark.parametrize("owner", [arb.DIFFUSION, arb.VIDEO])
@pytest.mark.parametrize("resident_owner", [arb.CHAT, arb.DIFFUSION, arb.VIDEO])
def test_actual_media_load_routes_refuse_before_engine_activation_or_load(
@@ -816,3 +1222,96 @@ def test_chat_resident_fast_path_adds_no_account_policy_work(route, chat_switch,
asyncio.run(
arun_as(OWNER, route._maybe_auto_switch_model("org/one-GGUF", request, OWNER.username))
)
+
+
[email protected]("policy", ["fit", "always"])
+def test_automatic_native_placement_credits_the_chat_model_an_image_sits_beside(
+ vram, monkeypatch, policy
+):
+ """Chat (10 GB) then an image (2 GB) on 16 GB: a primary native load replaces the chat model, so its 10 GB
+ count, not the image's 2 GB (the image stays loaded beside it)."""
+ from routes import inference as inf
+
+ v = vram(16)
+ v.policy = policy
+ v.load_chat(10)
+ assert v.load_image(4, headroom_gb = 2, policy = policy) == []
+ assert arb.owner_snapshot()[0] == arb.DIFFUSION
+ monkeypatch.setattr(
+ inf, "get_inference_backend", lambda: SimpleNamespace(active_model_name = None)
+ )
+ llama = SimpleNamespace(is_loaded = True, reclaimable_gpu_memory_gb = lambda: {0: 10.0})
+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: llama)
+ availability = inf._native_audio_post_handoff_free_gb()
+ assert availability is not None
+ assert availability.credited == arb.CHAT
+ assert availability.effective_free_gb == {0: 14.0}
+ assert availability.owner_snapshot == arb.owner_snapshot()
+ placement = _placement((0,), native_gb = 8.0)
+ # The CHAT acquire reuses this measurement and keeps the image beside the new model.
+ credit, wait = inf._outgoing_chat_credit_gb(placement, policy, True, availability)
+ assert credit == {0: 10.0} and wait is None
+
+
[email protected]("policy", ["fit", "always"])
+def test_native_load_replacing_the_primary_is_credited_with_its_memory(vram, monkeypatch, policy):
+ """A priced native load that replaces a 40 GB primary beside an idle image fits once the primary goes."""
+ from routes import inference as inf
+
+ v = vram(80)
+ v.policy = policy
+ v.load_chat(40)
+ assert v.load_image(20, policy = policy) == []
+ outgoing = inf._NativeAudioAvailability({0: 64.0}, {0: 40.0}, arb.owner_snapshot())
+ monkeypatch.setattr(
+ inf,
+ "_native_audio_post_handoff_free_gb",
+ lambda outgoing_owner = None: outgoing if outgoing_owner == arb.CHAT else None,
+ )
+ placement = _placement((0,), native_gb = 30.0)
+ credit, wait = inf._outgoing_chat_credit_gb(placement, policy, True, None)
+ assert credit == {0: 40.0} and wait == {0: 30.0}
+ kwargs = inf._chat_fit_kwargs(SimpleNamespace(is_gguf = False), placement, policy, credit)
+ assert v.load(arb.CHAT, {0: 30 * GIB}, replacing = True, **kwargs) == []
+ assert v.evictions == [] and arb.DIFFUSION in arb.residents()
+ # Loaded alongside, nothing is torn down, so nothing is credited.
+ assert inf._outgoing_chat_credit_gb(placement, policy, False, None) == (None, None)
+
+
+def test_extra_chat_load_that_evicts_the_image_keeps_the_chat_loader(vram):
+ """Alice's chat model sits behind her image; Bob's extra (unpriced) chat model evicts the image under When they
+ fit, but the primary chat model is still Alice's."""
+ v = vram(24)
+ run_as(ALICE, v.load_chat, 10, policy = "fit")
+ assert run_as(ALICE, v.load_image, 8, policy = "fit") == []
+ assert arb.owner_snapshot()[0] == arb.DIFFUSION
+ run_as(
+ BOB,
+ v.load,
+ arb.CHAT,
+ {0: 20 * GIB},
+ alongside = True,
+ policy = "fit",
+ gpu_ids = [0],
+ evict_first = True,
+ )
+ assert v.evictions == [arb.DIFFUSION]
+ assert arb.owner_account(arb.CHAT) == ALICE.account_id
+
+
[email protected]("policy", ["fit", "always"])
+def test_smaller_native_load_replacing_the_primary_fits_beside_a_tightly_packed_image(
+ vram, monkeypatch, policy
+):
+ """24 GB: a 14 GB chat model and an image admitted with no slack (6.5 GB weights + 2.5 GB headroom). A 2 GB
+ native model replacing the chat model frees 12 GB, so the image stays loaded."""
+ from routes import inference as inf
+
+ v = vram(24)
+ v.policy = policy
+ v.load_chat(14)
+ assert v.load_image(9, headroom_gb = 2.5, policy = policy) == []
+ placement = _placement((0,), native_gb = 2.0)
+ kwargs = inf._chat_fit_kwargs(SimpleNamespace(is_gguf = False), placement, policy, {0: 14.0})
+ assert v.load(arb.CHAT, {0: 2 * GIB}, replacing = True, **kwargs) == []
+ assert v.evictions == [] and arb.DIFFUSION in arb.residents()
diff --git a/studio/backend/tests/test_gpu_residency_coexistence.py b/studio/backend/tests/test_gpu_residency_coexistence.py
new file mode 100644
index 00000000000..dde3fef1e42
--- /dev/null
+++ b/studio/backend/tests/test_gpu_residency_coexistence.py
@@ -0,0 +1,688 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for core/inference/gpu_residency.py: pricing image and video loads that stay on a GPU beside chat."""
+
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import utils.hardware.hardware as _hw_module
+from core.inference import gpu_arbiter as arb
+from core.inference import gpu_residency as gr
+from core.inference.diffusion import DiffusionBackend
+from core.inference.diffusion_memory import estimate_image_runtime_mib, estimate_video_runtime_mib
+from utils.hardware import DeviceType
+
+MIB = 1024 * 1024
+
+
+class _GpuCacheResetMixin:
+ """Reset module-level GPU caches between tests to prevent state leaks."""
+
+ def tearDown(self):
+ _hw_module._physical_gpu_count = None
+ _hw_module._visible_gpu_count = None
+
+
+def _parent_ids(ids):
+ return patch("utils.hardware.hardware.get_parent_visible_gpu_ids", return_value = list(ids))
+
+
+def _image_request(
+ model_path,
+ gguf_filename = "z-image-turbo-Q4_K_M.gguf",
+ **kw,
+):
+ fields = dict(
+ model_kind = None, base_repo = None, family_override = None, text_encoder_file = None, vae_file = None
+ )
+ fields.update(kw)
+ return SimpleNamespace(model_path = str(model_path), gguf_filename = gguf_filename, **fields)
+
+
+def _video_request(model_path, gguf_filename, **kw):
+ fields = dict(model_kind = None, base_repo = None, family_override = None, display_repo_id = None)
+ fields.update(kw)
+ return SimpleNamespace(model_path = str(model_path), gguf_filename = gguf_filename, **fields)
+
+
+def _sparse(path: Path, mib: int) -> Path:
+ path.parent.mkdir(parents = True, exist_ok = True)
+ with open(path, "wb") as f:
+ f.truncate(mib * MIB)
+ return path
+
+
+class TestMediaLoadNeed(_GpuCacheResetMixin, unittest.TestCase):
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self._tmp.name) / "Z-Image-Turbo-GGUF"
+ _sparse(self.root / "z-image-turbo-Q4_K_M.gguf", 4000)
+ self._patches = [
+ # Native engines are host-dependent; the diffusers path is what these tests price.
+ patch(
+ "core.inference.diffusion_engine_router.predict_engine", return_value = "diffusers"
+ ),
+ patch.object(
+ DiffusionBackend,
+ "_companion_cache_bytes",
+ staticmethod(lambda base, *a, **k: 6000 * MIB),
+ ),
+ ]
+ for p in self._patches:
+ p.start()
+
+ def tearDown(self):
+ for p in reversed(self._patches):
+ p.stop()
+ self._tmp.cleanup()
+ super().tearDown()
+
+ def _image(
+ self,
+ ordinal,
+ parent = (2, 3),
+ request = None,
+ ):
+ with _parent_ids(parent):
+ return gr.media_load_need_mib(
+ arb.DIFFUSION, request or _image_request(self.root), ordinal
+ )
+
+ def test_gguf_image_prices_weights_plus_headroom_on_the_physical_gpu(self):
+ need = self._image(1)
+ self.assertIsNotNone(need)
+ weights = int(4000 * 1.05) + 6000
+ self.assertEqual(need.weights_mib, weights)
+ self.assertEqual(need.kind, "image")
+ self.assertEqual(list(need), [3]) # ordinal 1 of parent [2, 3]
+ self.assertEqual(need[3], weights + need.headroom_mib)
+ self.assertGreaterEqual(need.headroom_mib, 1024)
+ self.assertLessEqual(need.headroom_mib, estimate_image_runtime_mib(width = None, height = None))
+
+ def test_uncached_checkpoint_is_unknown(self):
+ self.assertIsNone(
+ self._image(0, request = _image_request(self.root, "z-image-turbo-Q8_0.gguf"))
+ )
+
+ def test_uncached_companions_are_unknown(self):
+ with patch.object(
+ DiffusionBackend, "_companion_cache_bytes", staticmethod(lambda *a, **k: 0)
+ ):
+ self.assertIsNone(self._image(0))
+
+ def test_unknown_family_is_unknown(self):
+ with patch("core.inference.diffusion_families.detect_family_for_pick", return_value = None):
+ self.assertIsNone(self._image(0))
+
+ def test_native_engine_is_unknown(self):
+ with patch("core.inference.diffusion_engine_router.predict_engine", return_value = "sd_cpp"):
+ self.assertIsNone(self._image(0))
+
+ def test_automatic_gpu_with_several_visible_is_unknown(self):
+ self.assertIsNone(self._image(None, parent = (0, 1)))
+
+ def test_automatic_gpu_with_one_visible_uses_it(self):
+ need = self._image(None, parent = (5,))
+ self.assertEqual(list(need), [5])
+
+ def test_masked_gpus_are_unknown(self):
+ self.assertIsNone(self._image(0, parent = ()))
+ self.assertIsNone(self._image(4, parent = (0, 1)))
+
+ def test_internal_error_is_unknown(self):
+ with patch("core.inference.diffusion.resolve_model_kind", side_effect = RuntimeError("boom")):
+ self.assertIsNone(self._image(0))
+ with patch(
+ "utils.hardware.hardware.get_parent_visible_gpu_ids", side_effect = OSError("smi")
+ ):
+ self.assertIsNone(gr.media_load_need_mib(arb.DIFFUSION, _image_request(self.root), 0))
+
+ def test_unknown_kind_is_unknown(self):
+ with _parent_ids((0,)):
+ self.assertIsNone(gr.media_load_need_mib("chat", _image_request(self.root), 0))
+
+ def _video_family(self, name = "wan-2.1"):
+ return SimpleNamespace(
+ name = name,
+ base_repo = "Wan-AI/Wan2.1-T2V-14B-Diffusers",
+ is_moe = False,
+ resolution_presets = ((832, 480),),
+ default_num_frames = 81,
+ )
+
+ def test_video_gguf_prices_checkpoint_plus_companion_table(self):
+ root = Path(self._tmp.name) / "Wan-GGUF"
+ _sparse(root / "wan-Q4_K_M.gguf", 3000)
+ fam = self._video_family()
+ with (
+ patch("core.inference.video._detect_load_family", return_value = fam),
+ patch(
+ "core.inference.diffusion_component_sizes.video_components_gb",
+ return_value = (28.0, 10.0, 0.5),
+ ),
+ _parent_ids((2, 3)),
+ ):
+ need = gr.media_load_need_mib(arb.VIDEO, _video_request(root, "wan-Q4_K_M.gguf"), 0)
+ companions = int(10.5 * 1000.0**3 / MIB)
+ self.assertEqual(need.weights_mib, int(3000 * 1.05) + companions)
+ self.assertEqual(
+ need.headroom_mib, estimate_video_runtime_mib(width = 832, height = 480, num_frames = 81)
+ )
+ self.assertEqual(need.kind, "video")
+ self.assertEqual(dict(need), {2: need.weights_mib + need.headroom_mib})
+
+ def test_video_h3_native_is_unknown(self):
+ root = Path(self._tmp.name) / "H3-GGUF"
+ _sparse(root / "h3-Q4_K_M.gguf", 10)
+ with (
+ patch(
+ "core.inference.video._detect_load_family",
+ return_value = self._video_family("minimax-h3"),
+ ),
+ patch(
+ "core.inference.diffusion_component_sizes.video_components_gb",
+ return_value = (1.0, 1.0, 1.0),
+ ),
+ _parent_ids((0,)),
+ ):
+ self.assertIsNone(
+ gr.media_load_need_mib(arb.VIDEO, _video_request(root, "h3-Q4_K_M.gguf"), 0)
+ )
+
+
+def _util(*devices, index_kind = "physical"):
+ rows = [dict(index = i, vram_total_gb = t, vram_used_gb = u) for i, t, u in devices]
+ return patch(
+ "utils.hardware.get_visible_gpu_utilization",
+ return_value = {"index_kind": index_kind, "devices": rows},
+ )
+
+
+def _cuda(device = DeviceType.CUDA):
+ return patch("utils.hardware.get_device", return_value = device)
+
+
+def _need(
+ per_gpu,
+ weights = 3000,
+ headroom = 2000,
+ kind = "video",
+):
+ return gr.MediaNeed(per_gpu, weights_mib = weights, headroom_mib = headroom, kind = kind)
+
+
+class _ArbiterResetMixin:
+ """Empty arbiter state for each test, restored afterwards."""
+
+ _FIELDS = ("_owner", "_residents", "_media_needs", "_accounts")
+
+ def setUp(self):
+ super().setUp()
+ self._saved = {name: getattr(arb, name) for name in self._FIELDS}
+ arb._owner, arb._residents, arb._media_needs, arb._accounts = None, {}, {}, {}
+ # Residents set by hand have no engine behind them; TestFailedLoadPrice covers the check.
+ self._engine_patch = patch.object(gr, "_loaded_or_loading", return_value = True)
+ self._engine_patch.start()
+ self.addCleanup(self._engine_patch.stop)
+
+ def tearDown(self):
+ for name, value in self._saved.items():
+ setattr(arb, name, value)
+ super().tearDown()
+
+ def _resident(
+ self,
+ owner,
+ gpus,
+ need = None,
+ ):
+ arb._residents[owner] = frozenset(gpus)
+ arb._owner = owner
+ if need is not None:
+ arb._media_needs[owner] = need
+
+
+FREE_80_20 = 80 * 1024 - 20 * 1024 # free MiB on an 80 GB card with 20 GB used
+
+
+class TestCanLoadAlongside(_ArbiterResetMixin, unittest.TestCase):
+ def _check(
+ self,
+ need,
+ owner = arb.VIDEO,
+ device = DeviceType.CUDA,
+ util = None,
+ ):
+ with _cuda(device), util or _util((0, 80.0, 20.0)):
+ return gr.can_load_alongside(owner, need)
+
+ def test_default_deny_reasons(self):
+ need = _need({0: 1000})
+ self.assertEqual(
+ gr.can_load_alongside(arb.VIDEO, None), (False, {"reason": "estimate_unavailable"})
+ )
+ self.assertEqual(gr.can_load_alongside(arb.VIDEO, {})[1]["reason"], "estimate_unavailable")
+ for device in (DeviceType.MLX, DeviceType.CPU, DeviceType.XPU):
+ self.assertEqual(
+ self._check(need, device = device), (False, {"reason": "non_accelerator"})
+ )
+ with (
+ _cuda(),
+ patch("utils.hardware.get_visible_gpu_utilization", side_effect = RuntimeError("smi")),
+ ):
+ self.assertEqual(
+ gr.can_load_alongside(arb.VIDEO, need), (False, {"reason": "probe_error"})
+ )
+ for kind in ("relative", "unresolved"):
+ self.assertEqual(
+ self._check(need, util = _util((0, 80.0, 20.0), index_kind = kind))[1]["reason"],
+ "unmapped_gpus",
+ )
+ self.assertEqual(self._check(_need({5: 1000}))[1], {"reason": "gpu_unknown", "gpu": 5})
+
+ def test_unified_memory_is_denied(self):
+ need = _need({0: 1000})
+ for flag in ("unified_memory", "shared_memory"):
+ util = {
+ "index_kind": "physical",
+ "devices": [{"index": 0, "vram_total_gb": 80.0, "vram_used_gb": 1.0, flag: True}],
+ }
+ with _cuda(), patch("utils.hardware.get_visible_gpu_utilization", return_value = util):
+ self.assertEqual(
+ gr.can_load_alongside(arb.VIDEO, need),
+ (False, {"reason": "unified_memory", "gpu": 0}),
+ )
+ self.assertEqual(self._check(need, util = _util((0, 0, 0.0)))[1]["reason"], "unified_memory")
+
+ def test_margin_boundary(self):
+ fits, info = self._check(_need({0: FREE_80_20 - gr.FIT_MARGIN_MIB}))
+ self.assertTrue(fits)
+ self.assertEqual(info["reason"], "fits")
+ fits, info = self._check(_need({0: FREE_80_20 - gr.FIT_MARGIN_MIB + 1}))
+ self.assertFalse(fits)
+ self.assertEqual((info["reason"], info["short_mib"]), ("short", {0: 1}))
+
+ def test_one_short_gpu_fails_the_whole_load(self):
+ util = _util((0, 80.0, 20.0), (1, 24.0, 20.0))
+ fits, info = self._check(_need({0: 1000, 1: 4000}), util = util)
+ self.assertFalse(fits)
+ self.assertEqual(list(info["short_mib"]), [1])
+ self.assertEqual(info["free_mib"], {0: FREE_80_20, 1: 4 * 1024})
+
+ def test_other_resident_headroom_is_subtracted(self):
+ self._resident(arb.VIDEO, {0}, _need({0: 5000}, weights = 3000, headroom = 2000))
+ _, info = self._check(_need({0: 1000}, kind = "other"), owner = arb.DIFFUSION)
+ self.assertEqual(info["free_mib"], {0: FREE_80_20 - 2000})
+
+ def test_a_resident_still_loading_counts_in_full(self):
+ # Its weights aren't in live free VRAM yet, so a second media load can't be admitted into that gap.
+ self._resident(arb.VIDEO, {0}, _need({0: 5000}, weights = 3000, headroom = 2000))
+ with patch.object(gr, "_media_busy", lambda owner: (owner == arb.VIDEO, False)):
+ _, info = self._check(_need({0: 1000}, kind = "other"), owner = arb.DIFFUSION)
+ self.assertEqual(info["free_mib"], {0: FREE_80_20 - 5000})
+
+ def test_own_record_adds_back_its_weights(self):
+ self._resident(arb.VIDEO, {0}, _need({0: 5000}, weights = 3000, headroom = 2000))
+ _, info = self._check(_need({0: 1000}), owner = arb.VIDEO)
+ self.assertEqual(info["free_mib"], {0: FREE_80_20 + 3000})
+
+ def test_image_that_would_stream_is_denied(self):
+ plan = MagicMock(return_value = "plan")
+ with (
+ _parent_ids((0,)),
+ patch("core.inference.diffusion_memory.plan_diffusion_memory", plan),
+ patch(
+ "core.inference.diffusion_memory.plan_keeps_transformer_resident",
+ return_value = False,
+ ),
+ ):
+ fits, info = self._check(_need({0: 1000}, kind = "image"), owner = arb.DIFFUSION)
+ self.assertFalse(fits)
+ self.assertEqual(info["reason"], "would_stream")
+ kw = plan.call_args.kwargs
+ self.assertIsNone(kw["requested_mode"])
+ self.assertEqual((kw["model_dense_mib"], kw["runtime_headroom_mib"]), (3000, 2000))
+ self.assertEqual(
+ (kw["device_memory"].device, kw["device_memory"].free_mib), ("cuda:0", FREE_80_20)
+ )
+
+ def test_image_kept_resident_fits_and_video_skips_the_planner(self):
+ with (
+ _parent_ids((0,)),
+ patch(
+ "core.inference.diffusion_memory.plan_keeps_transformer_resident", return_value = True
+ ),
+ ):
+ self.assertTrue(self._check(_need({0: 1000}, kind = "image"), owner = arb.DIFFUSION)[0])
+ with patch(
+ "core.inference.diffusion_memory.plan_diffusion_memory",
+ side_effect = AssertionError("planner"),
+ ):
+ self.assertTrue(self._check(_need({0: 1000}, kind = "video"))[0])
+
+
+class TestPipelineNeed(_GpuCacheResetMixin, unittest.TestCase):
+ def setUp(self):
+ import json
+
+ self._tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self._tmp.name) / "Z-Image-Turbo"
+ self.root.mkdir()
+ (self.root / "model_index.json").write_text(
+ json.dumps(
+ {
+ "_class_name": "ZImagePipeline",
+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
+ "text_encoder": ["transformers", "Qwen3Model"],
+ "tokenizer": ["transformers", "Qwen2Tokenizer"],
+ "transformer": ["diffusers", "ZImageTransformer2DModel"],
+ "vae": ["diffusers", "AutoencoderKL"],
+ }
+ )
+ )
+ (self.root / "vae").mkdir()
+ (self.root / "vae" / "config.json").write_text("{}")
+ _sparse(self.root / "vae" / "diffusion_pytorch_model.safetensors", 100)
+ self._patches = [
+ patch(
+ "core.inference.diffusion_engine_router.predict_engine", return_value = "diffusers"
+ ),
+ patch(
+ "core.inference.diffusion_families.detect_family_for_pick",
+ return_value = SimpleNamespace(name = "z-image"),
+ ),
+ ]
+ for p in self._patches:
+ p.start()
+
+ def tearDown(self):
+ for p in reversed(self._patches):
+ p.stop()
+ self._tmp.cleanup()
+ super().tearDown()
+
+ def _need(self):
+ request = _image_request(self.root, None, model_kind = "pipeline")
+ with _parent_ids((0,)):
+ return gr.media_load_need_mib(arb.DIFFUSION, request, 0)
+
+ def test_a_partly_downloaded_pipeline_is_unknown(self):
+ self.assertIsNone(self._need())
+
+ def test_a_complete_pipeline_is_priced_from_every_component(self):
+ (self.root / "scheduler").mkdir()
+ (self.root / "scheduler" / "scheduler_config.json").write_text("{}")
+ (self.root / "tokenizer").mkdir()
+ (self.root / "tokenizer" / "tokenizer_config.json").write_text("{}")
+ (self.root / "tokenizer" / "tokenizer.json").write_text("{}")
+ for name, stem, mib in (
+ ("text_encoder", "model", 8000),
+ ("transformer", "diffusion_pytorch_model", 12000),
+ ):
+ (self.root / name).mkdir()
+ (self.root / name / "config.json").write_text("{}")
+ _sparse(self.root / name / f"{stem}.safetensors", mib)
+ need = self._need()
+ self.assertIsNotNone(need)
+ self.assertGreaterEqual(need.weights_mib, 20000)
+
+
+class TestFailedLoadPrice(_ArbiterResetMixin, unittest.TestCase):
+ def test_a_price_counts_only_while_its_model_is_loaded_or_loading(self):
+ self._engine_patch.stop()
+ self._resident(arb.VIDEO, {0}, _need({0: 5000}))
+ for state, loading, counted in (
+ (None, (), False),
+ ("s", (), True),
+ (None, ("org/v",), True),
+ ):
+ engine = SimpleNamespace(_state = state, loading_repo_ids = lambda loading = loading: loading)
+ with patch.object(gr, "_media_engine", return_value = engine):
+ self.assertEqual(gr.media_held_mib(), {0: 5000} if counted else {})
+ with patch.object(gr, "_media_engine", return_value = None):
+ self.assertEqual(gr.media_held_mib(), {0: 5000})
+
+
+class TestMediaHeldMib(_ArbiterResetMixin, unittest.TestCase):
+ def test_nothing_resident_is_empty(self):
+ self.assertEqual(gr.media_held_mib(), {})
+ arb._media_needs[arb.VIDEO] = _need({0: 5000}) # a record with no resident
+ self.assertEqual(gr.media_held_mib(), {})
+
+ def test_weights_plus_headroom_summed_per_gpu(self):
+ self._resident(arb.CHAT, {0, 1})
+ self._resident(arb.VIDEO, {0}, _need({0: 5000}))
+ self._resident(arb.DIFFUSION, {0, 1}, _need({0: 7000, 1: 1000}, kind = "image"))
+ self.assertEqual(gr.media_held_mib(), {0: 12000, 1: 1000})
+
+ def test_non_resident_record_is_ignored(self):
+ self._resident(arb.DIFFUSION, {0}, _need({0: 7000}, kind = "image"))
+ arb._media_needs[arb.VIDEO] = _need({0: 5000})
+ self.assertEqual(gr.media_held_mib(), {0: 7000})
+
+ def test_corrupt_record_never_raises(self):
+ self._resident(arb.VIDEO, {0})
+ arb._media_needs[arb.VIDEO] = {0: "lots"}
+ self.assertEqual(gr.media_held_mib(), {})
+
+
+class TestFreeForOrder(_ArbiterResetMixin, unittest.TestCase):
+ def setUp(self):
+ super().setUp()
+ self.log, self.evicted = [], []
+ self.loading, self.generating, self.foreign, self.fits = (
+ set(),
+ set(),
+ {},
+ MagicMock(return_value = (False, {})),
+ )
+ recorder = lambda name: (lambda: self.log.append(name))
+ self.dropped = list
+ self.monitor = MagicMock()
+ self._patches = [
+ patch.dict(
+ arb._EVICTORS,
+ {
+ arb.CHAT: recorder("chat"),
+ arb.DIFFUSION: recorder("image"),
+ arb.VIDEO: recorder("video"),
+ },
+ ),
+ patch.object(
+ arb,
+ "resident_slots",
+ side_effect = lambda n: [
+ {"kind": gr._MEDIA_KIND.get(n, "chat"), "model": f"{n}-model"}
+ ],
+ ),
+ patch.object(
+ gr, "_media_busy", side_effect = lambda n: (n in self.loading, n in self.generating)
+ ),
+ patch.object(gr, "chat_load_in_flight", side_effect = lambda: arb.CHAT in self.loading),
+ patch.object(gr, "foreign_on", side_effect = lambda n, a: self.foreign.get(n, 0)),
+ patch.object(gr, "can_load_alongside", self.fits),
+ patch.object(
+ gr, "_kept_slot_report", return_value = {"kind": "chat", "model": "kept-model"}
+ ),
+ patch(
+ "core.inference.video_minimax_h3.release_h3_native_servers",
+ side_effect = lambda reason: self.log.append("h3") or 1,
+ ),
+ patch(
+ "core.inference.model_slots.eviction_victims",
+ side_effect = lambda *a: self.log.append("kept") or ["slot"],
+ ),
+ patch(
+ "core.inference.model_slots.evict_these",
+ side_effect = lambda v: self.log.append("evict_these") or self.dropped(v),
+ ),
+ patch("core.inference.api_monitor.api_monitor", self.monitor),
+ patch("utils.account_context.current_account_id", return_value = "me"),
+ ]
+ for p in self._patches:
+ p.start()
+
+ def tearDown(self):
+ for p in reversed(self._patches):
+ p.stop()
+ super().tearDown()
+
+ def _free(
+ self,
+ owner = arb.DIFFUSION,
+ gpus = (0,),
+ ):
+ return gr.free_for(
+ owner, gpus, "test", self.evicted, need_mib = _need({0: 1000}, kind = "image")
+ )
+
+ def test_an_h3_server_beside_video_on_another_gpu_stays(self):
+ self._resident(arb.VIDEO, {1})
+ self._resident(arb.CHAT, {0})
+ self._free()
+ self.assertNotIn("h3", self.log)
+
+ def test_a_kept_slot_a_request_reached_is_not_reported(self):
+ self._resident(arb.CHAT, {0})
+ self.dropped = lambda victims: []
+ self._free()
+ self.assertNotIn({"kind": "chat", "model": "kept-model"}, self.evicted)
+
+ def test_idle_media_before_kept_then_active_chat(self):
+ self._resident(arb.VIDEO, {0})
+ self._resident(arb.CHAT, {0})
+ freed = self._free()
+ self.assertEqual(self.log, ["h3", "video", "kept", "evict_these", "chat"])
+ self.assertEqual(freed, ["video-server", "video-model", "kept-model", "chat-model"])
+ self.assertEqual(arb._view(), {})
+ self.assertEqual(
+ self.evicted,
+ [
+ {"kind": "video", "model": "video-model"},
+ {"kind": "chat", "model": "kept-model"},
+ {"kind": "chat", "model": "chat-model"},
+ ],
+ )
+ self.monitor.record_lifecycle.assert_any_call(
+ event = "unload", model = "video-model", reason = "evicted"
+ )
+ self.assertEqual(self.monitor.record_lifecycle.call_count, 3)
+
+ def test_loading_resident_goes_first(self):
+ self._resident(arb.VIDEO, {0})
+ self._resident(arb.CHAT, {0})
+ self.loading = {arb.VIDEO, arb.CHAT}
+ self._free()
+ self.assertEqual(self.log[:3], ["video", "chat", "h3"])
+ self.assertNotIn("kept", self.log) # CHAT is gone, so no kept slots overlap
+
+ def test_stops_once_it_fits(self):
+ self._resident(arb.VIDEO, {0})
+ self._resident(arb.CHAT, {0})
+ self.fits.side_effect = [(False, {}), (True, {})]
+ self.assertEqual(self._free(), ["video-server", "video-model"])
+ self.assertEqual(self.log, ["h3", "video"])
+ self.assertIn(arb.CHAT, arb._view())
+
+ def test_disjoint_gpus_and_busy_media_are_untouched(self):
+ self._resident(arb.VIDEO, {1})
+ self._resident(arb.CHAT, {0})
+ self._free()
+ self.assertNotIn("video", self.log)
+ self._resident(arb.VIDEO, {0})
+ self.generating = {arb.VIDEO}
+ self.log.clear()
+ self._resident(arb.CHAT, {0})
+ self._free()
+ self.assertEqual(self.log, ["h3", "kept", "evict_these", "chat"])
+ self.assertIn(arb.VIDEO, arb._view())
+
+ def test_foreign_generation_blocks_the_eviction(self):
+ self._resident(arb.CHAT, {0})
+ self.foreign = {arb.CHAT: 2}
+ with self.assertRaises(arb.GpuBusyForAnotherAccountError) as ctx:
+ self._free()
+ self.assertEqual((ctx.exception.owner, ctx.exception.active), (arb.CHAT, 2))
+ self.assertNotIn("chat", self.log)
+ self.assertIn(arb.CHAT, arb._view())
+
+
+class TestOtherPlannedIncludesMedia(_ArbiterResetMixin, unittest.TestCase):
+ def test_media_held_is_added_for_a_registered_backend(self):
+ from core.inference import llama_cpp
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ mine, other, stray = (LlamaCppBackend(manages_processes = False) for _ in range(3))
+ other._process = object()
+ other._planned_vram_mib = {0: 4000}
+ self._resident(arb.DIFFUSION, {0, 1}, _need({0: 7000, 1: 500}, kind = "image"))
+ for backend in (mine, other):
+ llama_cpp.register_serving_backend(backend)
+ try:
+ self.assertEqual(mine._other_planned_vram_mib(), {0: 11000, 1: 500})
+ self.assertEqual(stray._other_planned_vram_mib(), {})
+ finally:
+ for backend in (mine, other):
+ llama_cpp.unregister_serving_backend(backend)
+
+
+class TestChatLoadingRefusal(unittest.TestCase):
+ def test_api_shape(self):
+ exc = gr.chat_loading_http_exception("/api/inference/images/load")
+ self.assertEqual(exc.status_code, 409)
+ self.assertEqual(exc.headers, {"Retry-After": "15"})
+ self.assertEqual(set(exc.detail), {"error", "message", "retry_after"})
+ self.assertEqual((exc.detail["error"], exc.detail["retry_after"]), ("gpu_busy", 15))
+ self.assertEqual(gr.ChatLoadInFlightRefusal().as_http_exception().detail, exc.detail)
+
+ def test_openai_shape(self):
+ for exc in (
+ gr.chat_loading_http_exception("/v1/images/generations"),
+ gr.chat_loading_http_exception("/api/x", openai_errors = True),
+ ):
+ self.assertEqual(exc.headers, {"Retry-After": "15"})
+ self.assertEqual(exc.detail["error"]["code"], "gpu_busy")
+ message = exc.detail["error"]["message"]
+ self.assertNotIn("is set to Always", message)
+ self.assertNotIn("is set to keep loaded", message)
+
+
+class TestKeptSlotReport(unittest.TestCase):
+ """An evicted kept slot reports the public id the status code shows, never a host path."""
+
+ def test_loaded_gguf_slot_reports_public_id(self):
+ llama = SimpleNamespace(
+ is_loaded = True,
+ model_identifier = "/srv/models/Qwen3-4B-Q4_K_M.gguf",
+ _openai_advertised_id = None,
+ )
+ orchestrator = SimpleNamespace(active_model_name = None)
+ slot = SimpleNamespace(
+ llama = llama,
+ orchestrator = orchestrator,
+ account = "me",
+ request = SimpleNamespace(model_path = "/srv/models/Qwen3-4B-Q4_K_M.gguf"),
+ )
+ report = gr._kept_slot_report(slot)
+ self.assertEqual(report["model"], "Qwen3-4B-Q4_K_M")
+ self.assertEqual(report["kind"], "chat")
+ self.assertEqual(report["account"], "me")
+
+ def test_fallback_never_reports_a_path(self):
+ slot = SimpleNamespace(
+ llama = None,
+ orchestrator = None,
+ account = None,
+ request = SimpleNamespace(model_path = "/srv/models/Qwen3-4B-Q4_K_M.gguf"),
+ )
+ with patch("routes.inference._kept_slot_reports", side_effect = RuntimeError("x")):
+ report = gr._kept_slot_report(slot)
+ self.assertEqual(report["model"], "Qwen3-4B-Q4_K_M")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/studio/backend/tests/test_media_auto_switch.py b/studio/backend/tests/test_media_auto_switch.py
index 87a4c796fb1..0fb48574066 100644
--- a/studio/backend/tests/test_media_auto_switch.py
+++ b/studio/backend/tests/test_media_auto_switch.py
@@ -1744,6 +1744,117 @@ def test_a_cpu_switch_does_not_hold_the_gates_it_cannot_evict_behind(
assert held == []
+def _chat_loading():
+ """A chat load in flight, as the real one shows it: chat's load lock held."""
+ assert chat._load_lock.acquire(blocking = False)
+ try:
+ yield
+ finally:
+ chat._load_lock.release()
+
+
[email protected]("openai_errors", [False, True])
+def test_always_refuses_behind_a_chat_load_without_waiting(
+ flux, enabled, takes_the_gpu, backend, loads, monkeypatch, openai_errors
+):
+ # Under Always the chat load's GPUs aren't known yet, so it overlaps every card: the switch
+ # answers the retryable 409 at once instead of queueing on chat's gate behind the load.
+ import core.inference.gpu_residency as residency
+
+ monkeypatch.setattr(residency, "current_policy", lambda: "always")
+ # The chat load has claimed the GPU.
+ monkeypatch.setattr(arb, "_owner", arb.CHAT)
+ monkeypatch.setattr(arb, "_residents", {arb.CHAT: None})
+
+ def _gate():
+ raise AssertionError("an Always switch must not wait behind a chat load")
+
+ monkeypatch.setattr(chat, "inference_lifecycle_gate", _gate)
+
+ with _chat_loading():
+ began = time.monotonic()
+ with pytest.raises(HTTPException) as excinfo:
+ _switch("black-forest-labs/FLUX.1-dev", openai_errors = openai_errors)
+ elapsed = time.monotonic() - began
+
+ assert excinfo.value.status_code == 409
+ assert excinfo.value.headers["Retry-After"] == "15"
+ detail = excinfo.value.detail
+ if openai_errors:
+ assert detail["error"]["code"] == "gpu_busy"
+ else:
+ assert detail["error"] == "gpu_busy"
+ assert detail["retry_after"] == 15
+ assert isinstance(detail["message"], str)
+ assert set(detail) == {"error", "message", "retry_after"}
+ assert "is set to Always" not in detail["message"]
+ assert "is set to keep loaded" not in detail["message"]
+ assert loads == []
+ assert elapsed < 5.0
+ # The switch's locks went back with the refusal.
+ assert not asyncio.run(_lock_is_held(arb.DIFFUSION))
+
+
+def test_fit_keeps_todays_wait_behind_a_chat_load(
+ flux, enabled, takes_the_gpu, backend, loads, monkeypatch
+):
+ # Only Always refuses: under fit the switch still queues on chat's lifecycle gate, and a
+ # load that holds it past the budget ends in today's 503.
+ import core.inference.gpu_residency as residency
+
+ monkeypatch.setattr(residency, "current_policy", lambda: "fit")
+ monkeypatch.setattr(mas, "_SWITCH_BUDGET_S", 0.4)
+ assert chat._lifecycle_lock.acquire(blocking = False)
+ try:
+ with _chat_loading():
+ with pytest.raises(HTTPException) as excinfo:
+ _switch("black-forest-labs/FLUX.1-dev")
+ finally:
+ chat._lifecycle_lock.release()
+
+ assert excinfo.value.status_code == 503
+ assert loads == []
+
+
[email protected]("claim", [{}, {arb.CHAT: frozenset({0})}])
+def test_always_waits_behind_a_cpu_or_placed_chat_load(
+ flux, enabled, takes_the_gpu, backend, loads, monkeypatch, claim
+):
+ # A CPU chat load never claims the GPU, and one on known GPUs is checked for overlap by the
+ # arbiter, so the switch queues on chat's gate as under fit instead of answering the 409.
+ import core.inference.gpu_residency as residency
+
+ monkeypatch.setattr(residency, "current_policy", lambda: "always")
+ monkeypatch.setattr(arb, "_owner", arb.CHAT if claim else None)
+ monkeypatch.setattr(arb, "_residents", dict(claim))
+ monkeypatch.setattr(mas, "_SWITCH_BUDGET_S", 0.4)
+ assert chat._lifecycle_lock.acquire(blocking = False)
+ try:
+ with _chat_loading():
+ with pytest.raises(HTTPException) as excinfo:
+ _switch("black-forest-labs/FLUX.1-dev")
+ finally:
+ chat._lifecycle_lock.release()
+
+ assert excinfo.value.status_code == 503
+ assert loads == []
+
+
+def test_cpu_switch_ignores_a_chat_load_under_always(flux, enabled, backend, loads, monkeypatch):
+ # A CPU load evicts nothing on the GPU, so a chat load there is no reason to refuse it.
+ import core.inference.gpu_residency as residency
+
+ monkeypatch.setattr(backends, "load_takes_the_gpu", lambda: False)
+ monkeypatch.setattr(mas, "load_takes_the_gpu", lambda: False)
+ monkeypatch.setattr(residency, "current_policy", lambda: "always")
+
+ with _chat_loading():
+ _switch("black-forest-labs/FLUX.1-dev")
+
+ assert [pick.model_id for _owner, pick in loads] == ["black-forest-labs/FLUX.1-dev"]
+
+
def test_the_video_route_refuses_a_flow_control_the_target_does_not_expose(monkeypatch):
# LTX-2 exposes no audio_flow_shift, and _resolve_flow_shifts would only say so after the
# switch had evicted the resident pipeline and loaded the target.
diff --git a/studio/backend/tests/test_multi_model_primary_fit.py b/studio/backend/tests/test_multi_model_primary_fit.py
new file mode 100644
index 00000000000..160b23da1aa
--- /dev/null
+++ b/studio/backend/tests/test_multi_model_primary_fit.py
@@ -0,0 +1,415 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A primary chat load beside a kept image or video model: make room under When they fit, refuse under Always."""
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+
+import core.inference.model_slots as model_slots
+import core.inference.orchestrator as orchestrator
+import routes.inference as inf
+from core.inference import gpu_arbiter, gpu_residency
+from core.inference.llama_cpp import GpuMemoryShortError
+from models.inference import LoadRequest, LoadResponse
+from utils import multi_model_settings
+from utils.account_context import AccountContext, arun_as
+
+BOB = AccountContext("b" * 32, "bob")
+IMG = {"kind": "image", "model": "org/Img"}
+
+
+class FakeLlama:
+ holds_no_vram, is_vision, context_length = False, False, 4096
+
+ def __init__(
+ self,
+ identifier = None,
+ variant = None,
+ ):
+ self.model_identifier, self.hf_variant = identifier, variant
+ self.is_loaded = self.is_active = identifier is not None
+
+
+class FakeOrchestrator:
+ active_model_name, models = None, {}
+
+
+def env(monkeypatch):
+ """Primary-only Unsloth Studio with spies on everything that could unload a model."""
+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama("org/A-GGUF", "Q4_K_M"))
+ monkeypatch.setattr(orchestrator, "_inference_backend", FakeOrchestrator())
+ monkeypatch.setattr(model_slots, "slots", [])
+ monkeypatch.setattr(model_slots, "stuck", [])
+ monkeypatch.setattr(inf, "LlamaCppBackend", FakeLlama)
+ monkeypatch.setattr(inf, "InferenceOrchestrator", FakeOrchestrator)
+ monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", lambda: None)
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
+ monkeypatch.setattr(multi_model_settings, "get_multi_model_enabled", lambda: True)
+ state = SimpleNamespace(
+ policy = "fit",
+ requests = [],
+ drops = [],
+ evicts = [],
+ unloads = [],
+ free_calls = [],
+ frees = [IMG],
+ errors = [],
+ retry_flags = [],
+ )
+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: state.policy)
+ monkeypatch.setattr(model_slots, "drop", lambda slot: state.drops.append(slot))
+ monkeypatch.setattr(model_slots, "evict", lambda *a: state.evicts.append(a) or [])
+ monkeypatch.setattr(inf, "_unload_primary", lambda: state.unloads.append(True))
+
+ def free_for(
+ owner,
+ gpu_ids,
+ reason,
+ evicted = None,
+ *,
+ need_mib = None,
+ short_mib = 0,
+ media_only = False,
+ ):
+ call = {"owner": owner, "gpu_ids": tuple(gpu_ids or ()), "short_mib": short_mib}
+ state.free_calls.append({**call, "media_only": True} if media_only else call)
+ evicted.extend(dict(s) for s in state.frees)
+ return [s["model"] for s in state.frees]
+
+ monkeypatch.setattr(gpu_residency, "free_for", free_for)
+ rows = {0: {"free_mib": 4096, "total_mib": 81920}}
+ monkeypatch.setattr(gpu_residency, "_free_mib_by_gpu", lambda: (rows, None))
+ monkeypatch.setattr(
+ gpu_arbiter, "resident_slots", lambda o: [dict(IMG)] if o == gpu_arbiter.DIFFUSION else []
+ )
+
+ async def load(request, *args, **kwargs):
+ state.requests.append(request)
+ state.retry_flags.append(inf._primary_can_retry_fit.get())
+ if state.errors and not request.force_alongside:
+ raise state.errors.pop(0)
+ return LoadResponse(
+ status = "loaded", model = request.model_path, display_name = "A", inference = {}
+ )
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+ return state
+
+
+def _short(capped = False):
+ return GpuMemoryShortError(
+ "needs 13 GB, 8 GB free", capped = capped, short_mib = 5000, gpu_indices = (0,)
+ )
+
+
+def _gated(account = None):
+ async def run():
+ coro = inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF"), None, "s")
+ response = await (arun_as(account, coro) if account else coro)
+ return response, inf._primary_can_retry_fit.get()
+
+ return asyncio.run(run())
+
+
+def _nothing_unloaded(state):
+ assert state.drops == [] and state.evicts == [] and state.unloads == []
+ assert model_slots.stuck == []
+
+
+def test_fit_frees_the_image_model_and_retries(env):
+ assert inf._primary_can_retry_fit.get() is False
+ env.errors = [_short()]
+ response, flag_after = _gated()
+ assert len(env.requests) == 2 and not env.requests[1].force_alongside
+ assert response.evicted == ["org/Img"]
+ assert env.free_calls == [{"owner": gpu_arbiter.CHAT, "gpu_ids": (0,), "short_mib": 5000}]
+ assert env.drops == [] and model_slots.stuck == [] # model_slots.drop(None) never runs
+ assert env.retry_flags == [True, True] and flag_after is False
+
+
+def test_freed_media_is_reported_once(env):
+ env.errors = [_short()]
+ response, _ = _gated()
+ # The route's eviction sink carries it into both fields; the retry must not add it a second time.
+ assert response.evicted == ["org/Img"] and response.evicted_slots == [IMG]
+
+
+def test_an_idle_video_server_freed_alone_still_earns_a_full_size_retry(env, monkeypatch):
+ env.errors = [_short()]
+ monkeypatch.setattr(
+ gpu_residency, "free_for", lambda *a, **k: env.free_calls.append(a) or ["video-server"]
+ )
+ response, _ = _gated()
+ assert [r.force_alongside for r in env.requests] == [False, False] and not response.evicted
+
+
+def test_fit_with_nothing_freed_lets_llama_server_fit_what_it_can(env):
+ env.errors, env.frees = [_short()], []
+ response, _ = _gated()
+ assert [r.force_alongside for r in env.requests] == [False, True]
+ assert response.status == "loaded" and not response.evicted
+ _nothing_unloaded(env)
+
+
[email protected]("policy", ["fit", "always"])
+def test_capped_fit_retries_at_a_smaller_context(env, policy):
+ env.policy, env.errors = policy, [_short(capped = True)]
+ response, _ = _gated()
+ assert [r.force_alongside for r in env.requests] == [False, True]
+ assert response.status == "loaded"
+ assert env.free_calls == []
+ _nothing_unloaded(env)
+
+
+def _refused(account = None) -> str:
+ """The Always refusal reaches the caller as the K2 409, header included."""
+ with pytest.raises(HTTPException) as info:
+ _gated(account)
+ assert info.value.status_code == 409
+ assert info.value.headers.get("X-Unsloth-Refusal") == "multi-model-always"
+ return str(info.value.detail)
+
+
+def test_always_refuses_with_the_numbers_for_the_owner(env):
+ env.policy, env.errors = "always", [_short()]
+ message = _refused()
+ assert "is set to Always" in message
+ # 5000 MiB short beside 4096 MiB free: needs 8.9 GB, 4.0 GB is free.
+ assert "needs about 8.9 GB of GPU memory, and 4.0 GB is free" in message
+ assert env.free_calls == [] and len(env.requests) == 1
+ _nothing_unloaded(env)
+
+
+def test_always_refuses_a_managed_account_with_the_plain_copy(env):
+ env.policy, env.errors = "always", [_short()]
+ message = _refused(BOB)
+ assert "GB" not in message
+ assert "the image model org/Img, which Unsloth Studio is set to keep loaded" in message
+ assert env.free_calls == []
+ _nothing_unloaded(env)
+
+
+def test_always_names_only_media_on_the_short_gpu(env, monkeypatch):
+ # The image on GPU 1 isn't in the way of a chat load short on GPU 0; the video model there is.
+ vid = {"kind": "video", "model": "org/Vid"}
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.VIDEO)
+ monkeypatch.setattr(
+ gpu_arbiter,
+ "_residents",
+ {gpu_arbiter.DIFFUSION: frozenset({1}), gpu_arbiter.VIDEO: frozenset({0})},
+ )
+ monkeypatch.setattr(
+ gpu_arbiter,
+ "resident_slots",
+ lambda o: {gpu_arbiter.DIFFUSION: [dict(IMG)], gpu_arbiter.VIDEO: [dict(vid)]}.get(o, []),
+ )
+ env.policy, env.errors = "always", [_short()]
+ message = _refused(BOB)
+ assert "org/Vid" in message and "org/Img" not in message
+ _nothing_unloaded(env)
+
+
+def _image_resident(monkeypatch):
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)
+ monkeypatch.setattr(gpu_arbiter, "_residents", {gpu_arbiter.DIFFUSION: frozenset({0})})
+ assert gpu_arbiter.holds(gpu_arbiter.DIFFUSION)
+
+
+def _oom():
+ return HTTPException(status_code = 500, detail = "CUDA out of memory")
+
+
+def test_non_gguf_oom_under_always_beside_an_image_model_is_refused(env, monkeypatch):
+ _image_resident(monkeypatch)
+ env.policy, env.errors = "always", [_oom()]
+ message = _refused()
+ assert "is set to Always" in message and "org/Img" in message
+ assert env.free_calls == []
+ _nothing_unloaded(env)
+
+
+def test_non_gguf_oom_under_always_beside_media_on_another_gpu_keeps_the_original_error(
+ env, monkeypatch
+):
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ monkeypatch.setattr(
+ gpu_arbiter,
+ "_residents",
+ {gpu_arbiter.DIFFUSION: frozenset({0}), gpu_arbiter.CHAT: frozenset({1})},
+ )
+ env.policy, env.errors = "always", [_oom()]
+ with pytest.raises(HTTPException) as info:
+ _gated()
+ assert info.value.status_code == 500 and info.value.detail == "CUDA out of memory"
+ _nothing_unloaded(env)
+
+
+def test_non_gguf_oom_under_fit_keeps_the_original_error(env, monkeypatch):
+ _image_resident(monkeypatch)
+ env.errors = [_oom()]
+ with pytest.raises(HTTPException) as info:
+ _gated()
+ assert info.value.status_code == 500 and info.value.detail == "CUDA out of memory"
+ assert env.free_calls == [] and len(env.requests) == 1
+ _nothing_unloaded(env)
+
+
+def test_non_gguf_oom_under_always_without_media_keeps_the_original_error(env, monkeypatch):
+ monkeypatch.setattr(gpu_arbiter, "_owner", None)
+ env.policy, env.errors = "always", [_oom()]
+ with pytest.raises(HTTPException) as info:
+ _gated()
+ assert info.value.status_code == 500
+ _nothing_unloaded(env)
+
+
+def _placement(
+ resolved,
+ vulkan = False,
+ native_gb = None,
+):
+ return SimpleNamespace(
+ requested_gpu_ids = None,
+ resolved_gpu_ids = resolved,
+ gpu_ids_are_vulkan_ordinals = vulkan,
+ native_required_gb = native_gb,
+ )
+
+
+ "held, placement, expected",
+ [
+ ({}, _placement([0]), False),
+ ({1: 4096}, _placement([0]), False),
+ ({1: 4096}, _placement([1]), True),
+ ({1: 4096}, _placement(None), True),
+ ({1: 4096}, _placement([0], vulkan = True), True),
+ ({}, _placement([0], vulkan = True), False),
+ ],
+)
+def test_media_on_target_gpus(monkeypatch, held, placement, expected):
+ monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: held)
+ assert inf._media_on_target_gpus(placement) is expected
+
+
+def test_chat_fit_kwargs(monkeypatch):
+ policy = SimpleNamespace(value = "off")
+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: policy.value)
+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: [])
+ gguf, native = SimpleNamespace(is_gguf = True), SimpleNamespace(is_gguf = False)
+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {}
+ policy.value = "fit"
+ token = inf._primary_can_retry_fit.set(True)
+ try:
+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {"policy": "fit", "gpu_ids": [0]}
+ # Anything but GGUF has no fit estimate before launch, even inside the retry.
+ assert inf._chat_fit_kwargs(native, _placement([0])) == {
+ "policy": "fit",
+ "gpu_ids": [0],
+ "evict_first": True,
+ }
+ finally:
+ inf._primary_can_retry_fit.reset(token)
+ # Auto-switch calls the tracked impl without load_model_gated, so nothing would make room after.
+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {
+ "policy": "fit",
+ "gpu_ids": [0],
+ "evict_first": True,
+ }
+ native_need = inf._chat_fit_kwargs(native, _placement([0, 1], native_gb = 8.0))
+ assert native_need == {"policy": "fit", "gpu_ids": [0, 1], "need_mib": {0: 4096, 1: 4096}}
+ policy.value = "always"
+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {"policy": "always", "gpu_ids": [0]}
+ # No target GPUs (or Vulkan ordinals) leaves every GPU in play.
+ assert inf._chat_fit_kwargs(gguf, _placement(None)) == {"policy": "always"}
+ assert inf._chat_fit_kwargs(gguf, _placement([0], vulkan = True)) == {"policy": "always"}
+
+
+ "parent, physical, expected",
+ [
+ ([], [3], [3]), # no mask: physical ids are the ordinals
+ ([2, 3], [3], [1]), # CUDA_VISIBLE_DEVICES=2,3
+ ([2, 3], [3, 2], [1, 0]),
+ ([2, 3], [5], None), # not visible: every GPU counts
+ ],
+)
+def test_visible_ordinals(monkeypatch, parent, physical, expected):
+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: parent)
+ assert inf._visible_ordinals(physical) == expected
+
+
+def test_the_replace_fallback_never_refuses_a_partial_fit():
+ # Nothing retries the fallback load, so a refusal beside media there would escape as a raw GpuMemoryShortError.
+ import inspect
+
+ source = inspect.getsource(inf.load_model_gated)
+ unload = source.index("await asyncio.to_thread(_unload_primary)")
+ assert (
+ unload
+ < source.index("_primary_can_retry_fit.set(False)", unload)
+ < source.index("_run_tracked_load_model_impl", unload)
+ )
+
+
+def test_a_pinned_non_gguf_chat_load_keeps_its_requested_gpus(monkeypatch):
+ # Only GGUF and native audio resolve their GPUs: a Transformers load pinned to GPU 1 still targets GPU 1.
+ pinned = inf._LoadPlacement([1], None, False, None)
+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: "always")
+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: [])
+ monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: {0: 4096})
+ assert inf._chat_fit_kwargs(SimpleNamespace(is_gguf = False), pinned) == {
+ "policy": "always",
+ "gpu_ids": [1],
+ }
+ assert inf._media_on_target_gpus(pinned) is False
+
+
+ ("serving", "policy", "expected"),
+ [(False, "always", True), (True, "always", False), (True, "fit", True), (False, "fit", True)],
+)
+def test_a_serving_primary_reloaded_under_always_is_not_torn_down_for_a_refusal(
+ monkeypatch, serving, policy, expected
+):
+ # Always only refuses; the partial-fit check runs after the old llama-server is killed.
+ monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: {0: 4096})
+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: [])
+ placement = inf._LoadPlacement([0], [0], False, None)
+ token = inf._primary_can_retry_fit.set(True)
+ try:
+ assert inf._primary_refuses_partial_fit(serving, policy, placement) is expected
+ finally:
+ inf._primary_can_retry_fit.reset(token)
+
+
+def test_an_extra_chat_load_frees_an_idle_image_before_replacing_the_primary(env, monkeypatch):
+ # Chat A + an idle image under When they fit: chat B loaded alongside comes up short. The image goes and B
+ # loads beside A, instead of A being unloaded (and the image after it, by the replacement primary load).
+ extra = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")
+
+ async def select(_request):
+ return extra
+
+ monkeypatch.setattr(inf, "_select_load_slot", select)
+ monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: "fit")
+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *a, **k: [])
+ monkeypatch.setattr(inf, "_raise_or_cancel_active_generations", lambda **k: None)
+ env.errors = [_short()]
+
+ async def run():
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ return await inf.load_model_gated(request, None, "s")
+
+ response = asyncio.run(run())
+ assert env.unloads == []
+ assert env.free_calls == [
+ {"owner": gpu_arbiter.CHAT, "gpu_ids": (0,), "short_mib": 5000, "media_only": True}
+ ]
+ assert len(env.requests) == 2 and env.requests[1].alongside
+ assert response.evicted == ["org/Img"]