Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: image and video models stay loaded beside chat when they fit (#14106)
Etherl9bac1786d3f8
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.pyindex 500167fc9ea..21ebcaae7dd 100644--- a/studio/backend/core/inference/gpu_arbiter.py+++ b/studio/backend/core/inference/gpu_arbiter.py@@ -47,6 +47,8 @@ _prior_account: Optional[str] = None_residents: dict[str, Optional[frozenset[int]]] = {}# (account, prior account) for residents other than ``_owner``, whose pair lives in the globals above._accounts: dict[str, tuple[Optional[str], Optional[str]]] = {}+# What each image or video resident was admitted with under When they fit / Always (gpu_residency.MediaNeed).+_media_needs: dict[str, dict] = {}# Phrases the refusal copy always contains, for readers that lose the X-Unsloth-Refusal header.MULTI_MODEL_REFUSAL_MARKERS = ("is set to Always", "is set to keep loaded")@@ -444,6 +446,7 @@ def _drop_resident(owner: str) -> None:if owner not in _residents:returndel _residents[owner]+ _media_needs.pop(owner, None)if owner == _owner:if _residents:_owner = next(reversed(_residents))@@ -537,9 +540,10 @@ def acquire_for_request(register = None,**kwargs,) -> Any:+ from core.inference.gpu_residency import ChatLoadInFlightRefusaltry:return acquire_for(owner, register, **kwargs)- except GpuBusyForAnotherAccountError as exc:+ except (GpuBusyForAnotherAccountError, ChatLoadInFlightRefusal) as exc:raise exc.as_http_exception() from exc@@ -555,6 +559,8 @@ def acquire_for(policy: str = "off",evicted: Optional[list] = None,gpu_ids: Optional[Iterable[int]] = None,+ need_mib: Optional[dict] = None,+ evict_first: bool = False,) -> Any:"""Make ``owner`` resident, evicting every other resident under the default policy.@@ -566,6 +572,10 @@ def acquire_for(chat, image or video model. ``alongside`` is unrelated: it only decides whether the owneraccount is re-claimed. ``evicted`` collects ``{kind, model}`` for each model an evictorunloads. ``gpu_ids`` are the visible device ordinals the owner will use, when known.++ Under "fit" and "always" only residents on the same GPUs can be evicted (gpu_residency.coreside_locked):+ ``need_mib`` is what an image or video load will hold per physical GPU (None: unknown, so it evicts first),+ and ``evict_first`` makes a chat load with no estimate before launch evict as "off" does."""global _owner, _owner_epoch, _owner_account, _prior_accountif owner not in _EVICTORS:@@ -576,11 +586,34 @@ def acquire_for(with _lock:if expected_current is not None and (_owner, _owner_epoch) != expected_current:raise OwnerChangedError("The resident GPU model changed; retry the load.")- if register is not None or replacing:+ coreside = policy != "off" and owner in _KEPT_UNDER_ALWAYS+ if (register is not None or replacing) and not coreside:raise_if_other_accounts_active(acting)+ elif (register is not None or replacing) and not (owner == CHAT and alongside):+ # Only another account's work on this model blocks reloading it; residents it would evict are+ # checked one by one.+ from core.inference.gpu_residency import foreign_on+ busy = foreign_on(owner, acting)+ if busy:+ raise GpuBusyForAnotherAccountError(owner, busy)_sync_from_view()prev_owner, prev_pair = _owner, (_owner_account, _prior_account)+ # Read before evicting: dropping the current owner promotes a parked resident and pops its accounts.+ parked = owner in _accountsothers = [name for name in _residents if name != owner]+ if others and coreside:+ from core.inference.gpu_residency import coreside_locked+ coreside_locked(+ owner,+ policy = policy,+ need_mib = need_mib,+ gpu_ids = gpu_ids,+ allow_evict = allow_evict,+ evict_first = evict_first,+ acting = acting,+ evicted = evicted,+ )+ others = []if others:if not allow_evict:raise GpuOwnerBusyError(others[-1])@@ -603,11 +636,15 @@ def acquire_for(_EVICTORS[name]()note_evictions(unloading, sink)_drop_resident(name)- if owner != VIDEO:+ if owner != VIDEO and not (coreside and VIDEO in _residents):# A resident H3 sd-server sits outside every owner's teardown._release_idle_video_servers(f"GPU acquired for {owner}")# Records who LOADED the model; a plain re-assert must not hand it to whoever asked last.- claims = prev_owner != owner or ((register is not None or replacing) and not alongside)+ # A resident parked behind a later claimant is a re-assert, not a new load.+ rejoins = prev_owner != owner and owner in _residents and parked+ claims = (prev_owner != owner and not rejoins) or (+ (register is not None or replacing) and not alongside+ )if owner != _owner:if _owner is not None:# Another resident stays loaded; park its accounts while ``owner`` is the last claimant.@@ -620,11 +657,34 @@ def acquire_for(else:_owner_account, _prior_account = None, None_owner = owner- _residents.pop(owner, None)+ held_before = _residents.pop(owner, ...)known = physical_gpu_ids(gpu_ids)- _residents[owner] = known+ gpus_now = known if known is not None or not need_mib else frozenset(need_mib)+ if owner == CHAT and held_before is not ...:+ from core.inference import model_slots+ if register is None and not replacing and gpus_now is None:+ # A re-assert places nothing new.+ gpus_now = held_before+ elif alongside or model_slots.holds_vram():+ # Every chat model kept loaded counts, not just the newest; unknown (None) means every GPU.+ gpus_now = (+ None if held_before is None or gpus_now is None else held_before | gpus_now+ )+ _residents[owner] = gpus_now_owner_epoch += 1- result = register() if register is not None else None+ try:+ result = register() if register is not None else None+ except BaseException:+ if held_before is not ...:+ # The model already loaded stays where it was.+ _residents[owner] = held_before+ raise+ if owner in (DIFFUSION, VIDEO) and register is not None:+ # The price media_held_mib() reports for this resident; unpriced residents are evicted to make room.+ if coreside and need_mib:+ _media_needs[owner] = need_mib+ else:+ _media_needs.pop(owner, None)# A raising registration loaded nothing and must not take residency.if claims:_prior_account, _owner_account = _owner_account, actingdiff --git a/studio/backend/core/inference/gpu_residency.py b/studio/backend/core/inference/gpu_residency.pynew file mode 100644index 00000000000..5b50e99d83e--- /dev/null+++ b/studio/backend/core/inference/gpu_residency.py@@ -0,0 +1,812 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Whether an image or video model can stay on a GPU next to chat, and what to unload when it can't.++Keep multiple models loaded set to When they fit or Always lets the GPU arbiter keep several residents. This module+prices them: a media resident holds the weights plus generation headroom it was admitted with, a new media load is+priced from the files already on disk, and live free VRAM decides whether it fits. Anything it cannot price is+treated as not fitting, so the caller frees memory first (fit) or refuses (always). Modelled on training_vram.py.+"""++from __future__ import annotations++from types import SimpleNamespace+from typing import Any, Iterable, Optional++from core.inference import gpu_arbiter as arb+from loggers import get_logger++logger = get_logger(__name__)++# fits iff free_mib >= need_mib + FIT_MARGIN_MIB on every GPU the load uses. Live free VRAM already excludes resident+# weights, so the margin only covers estimate error and allocator slack. An image load must also keep its+# transformer on the GPU (the diffusion planner adds its own reserve on top, so this is not doubled there).+FIT_MARGIN_MIB = 1024+# A chat load in flight finishes on its own; the media auto-switch uses the same wait.+K3_RETRY_AFTER_S = 15++MEDIA_OWNERS = (arb.DIFFUSION, arb.VIDEO)+_MEDIA_KIND = {arb.DIFFUSION: "image", arb.VIDEO: "video"}+++class MediaNeed(dict):+ """{physical GPU id: weights + generation headroom MiB}, plus the split the planner rule needs."""++ def __init__(self, per_gpu: dict, *, weights_mib: int, headroom_mib: int, kind: str):+ super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})+ self.weights_mib = int(weights_mib)+ self.headroom_mib = int(headroom_mib)+ self.kind = kind+++def current_policy() -> str:+ """Keep multiple models loaded, read as the owner. Never call this under the arbiter lock (it reads a database)."""+ try:+ from utils.multi_model_settings import get_multi_model_mode+ return get_multi_model_mode()+ except Exception as exc: # noqa: BLE001 - an unreadable setting keeps today's behaviour+ logger.warning("gpu_residency: could not read Keep multiple models loaded: %s", exc)+ return "off"+++def _media_records() -> dict[str, dict]:+ """Admitted needs of the media residents still on the GPU. Lock free: acquire_for calls this holding the lock."""+ records = dict(getattr(arb, "_media_needs", {}) or {})+ if not records:+ return {}+ resident = set(arb._view()) if hasattr(arb, "_view") else {arb._owner}+ return {+ owner: need+ for owner, need in records.items()+ if owner in resident and _loaded_or_loading(owner)+ }+++def _loaded_or_loading(owner: str) -> bool:+ """False once a media load failed and left nothing loaded, so its price stops counting. Unknown reads as True."""+ try:+ engine = _media_engine(owner)+ if engine is None:+ return True+ return getattr(engine, "_state", None) is not None or bool(engine.loading_repo_ids())+ except Exception: # noqa: BLE001+ return True+++def media_held_mib() -> dict[int, int]:+ """{physical GPU id: MiB} held by resident image and video models (weights + headroom). Never raises."""+ try:+ held: dict[int, int] = {}+ for need in _media_records().values():+ for gpu, mib in need.items():+ held[gpu] = held.get(gpu, 0) + int(mib)+ return held+ except Exception as exc: # noqa: BLE001+ logger.warning("gpu_residency: could not price the resident media models: %s", exc)+ return {}+++def media_unallocated_mib() -> dict[int, int]:+ """{physical GPU id: MiB} of the media price live free VRAM still shows as free: an idle resident's generation+ headroom, or all of a resident still loading. Never raises."""+ try:+ held: dict[int, int] = {}+ for owner, need in _media_records().items():+ loading = _media_busy(owner)[0]+ for gpu, mib in need.items():+ unallocated = int(mib) if loading else int(getattr(need, "headroom_mib", 0))+ held[gpu] = held.get(gpu, 0) + unallocated+ return held+ except Exception as exc: # noqa: BLE001+ logger.warning("gpu_residency: could not price the resident media models: %s", exc)+ return {}+++def _media_engine(owner: str) -> Any:+ from core.inference.media_keepwarm import engine_if_imported+ return engine_if_imported(owner)+++def _media_busy(owner: str) -> tuple[bool, bool]:+ """(loading, generating) for a media owner; unknown reads as busy, so it is never freed as idle."""+ try:+ engine = _media_engine(owner)+ if engine is None:+ return False, False+ loading = bool(engine.loading_repo_ids())+ progress = engine.generate_progress() or {}+ from core.inference.media_keepwarm import _TRACKERS++ tracker = _TRACKERS.get(owner)+ outstanding = bool(tracker is not None and tracker.outstanding())+ return loading, bool(progress.get("active")) or outstanding+ except Exception: # noqa: BLE001+ return True, True+++def summarize_resident_media() -> dict[str, Any]:+ """Report the resident image and video models. Never raises."""+ summary: dict[str, Any] = {}+ records = {}+ try:+ records = _media_records()+ resident = arb.residents()+ except Exception: # noqa: BLE001+ resident = {}+ for owner in MEDIA_OWNERS:+ loading, generating = _media_busy(owner) if owner in resident else (False, False)+ summary[owner] = {+ "resident": owner in resident,+ "gpus": sorted(resident.get(owner) or ()),+ "loading": loading,+ "generating": generating,+ "priced": owner in records,+ }+ summary["loading"] = any(summary[owner]["loading"] for owner in MEDIA_OWNERS)+ summary["held_mib"] = media_held_mib()+ return summary+++def media_load_need_mib(kind: str, request: Any, gpu_ordinal: Optional[int]) -> Optional[MediaNeed]:+ """What a new image or video load will hold, priced from files already on disk. None when unknown.++ ``gpu_ordinal`` is the visible torch ordinal the route resolved; None (automatic) is only priceable on a single+ visible GPU. Never touches the network and never raises."""+ try:+ from utils.hardware.hardware import get_parent_visible_gpu_ids++ parent = list(get_parent_visible_gpu_ids() or ())+ if gpu_ordinal is None:+ physical = parent[0] if len(parent) == 1 else None+ else:+ physical = parent[gpu_ordinal] if 0 <= gpu_ordinal < len(parent) else None+ if physical is None: # several GPUs and no pick, or a UUID/MIG mask with no numeric ids+ return None+ priced = {arb.DIFFUSION: _image_need, arb.VIDEO: _video_need}.get(kind, lambda *_: None)(+ request, gpu_ordinal+ )+ if priced is None or not priced[0] or priced[0] <= 0:+ return None+ weights, headroom = int(priced[0]), int(priced[1])+ return MediaNeed(+ {physical: weights + headroom},+ weights_mib = weights,+ headroom_mib = headroom,+ kind = _MEDIA_KIND[kind],+ )+ except Exception as exc: # noqa: BLE001 - an unpriced load is treated as not fitting+ logger.warning("gpu_residency: could not price the %s load: %s", kind, exc)+ return None+++def _cached_pick(repo: str, filename: Optional[str]) -> Optional[str]:+ """The picked checkpoint file when it is already on disk (local path or either Hub cache root), else None."""+ from pathlib import Path++ local = Path(repo).expanduser()+ if local.is_file():+ return str(local)+ if not filename or Path(filename).is_absolute() or ".." in Path(filename).parts:+ return None+ if local.is_dir():+ return str(local / filename) if (local / filename).is_file() else None+ from huggingface_hub import try_to_load_from_cache+ from core.inference.diffusion import hub_cache_dir++ for root in (hub_cache_dir(), None):+ hit = try_to_load_from_cache(repo, filename, cache_dir = root)+ if isinstance(hit, str) and Path(hit).is_file():+ return hit+ return None+++def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:+ """(weights MiB, headroom MiB) for a diffusers image load, the way DiffusionBackend._plan_memory prices it."""+ from core.inference.diffusion import (+ DiffusionBackend as DB,+ _image_variant_hint,+ resolve_model_kind,+ )+ from core.inference.diffusion_auto_policy import resident_bytes_from_declared+ from core.inference.diffusion_engine_router import predict_engine+ from core.inference.diffusion_families import (+ detect_family_for_pick,+ local_pipeline_components_are_complete,+ named_variant_base,+ prefer_ungated_mirror,+ resolve_base_repo,+ )+ from core.inference.diffusion_gguf_pipeline import whole_pipeline_gguf_resident_mib+ from core.inference.diffusion_memory import (+ estimate_gguf_resident_mib,+ estimate_image_runtime_mib,+ estimate_safetensors_dense_mib,+ file_size_mib,+ )+ from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS++ repo, name = request.model_path, request.gguf_filename+ if getattr(request, "text_encoder_file", None) or getattr(request, "vae_file", None):+ return None # supplied companions are priced from their headers at load time only+ kind = resolve_model_kind(name, request.model_kind)+ fam = detect_family_for_pick(repo, name, request.family_override)+ if (+ fam is None+ or predict_engine(fam, model_kind = kind, gpu_ordinal = gpu_ordinal, gguf_filename = name)+ != ENGINE_DIFFUSERS+ ):+ return None+ mib = 1024 * 1024+ if kind == "pipeline":+ base = repo+ files: dict[str, int] = {}+ dirs: list = []++ def grab(d):+ dirs.append(d)+ sizes = DB._local_dir_weight_sizes(d, exclude_transformer = False)+ for rel, size in sizes.items():+ files[rel] = max(files.get(rel, 0), size)+ return sizes++ DB._union_over_cached_revs(prefer_ungated_mirror(repo), grab)+ # A partial download would be priced as the whole pipeline.+ if not any(local_pipeline_components_are_complete(d, "model_index.json") for d in dirs):+ return None+ weights = (resident_bytes_from_declared(repo, files.items()) or 0) // mib+ else:+ path = _cached_pick(repo, name)+ if path is None:+ return None+ whole = getattr(fam, "single_file_is_pipeline", False)+ if kind == "single_file":+ fp8 = not whole and "fp8" in str(name).lower()+ weights = estimate_safetensors_dense_mib(file_size_mib(path), fp8_upcast = fp8)+ elif whole:+ weights = whole_pipeline_gguf_resident_mib(path) or estimate_gguf_resident_mib(+ file_size_mib(path)+ )+ else:+ weights = estimate_gguf_resident_mib(file_size_mib(path))+ base = (request.base_repo or "").strip()+ if (+ not base+ ): # offline only: the link a past load recorded, else the curated variant the name spells+ from hub.utils.companion_assets import read_companion_links+ linked = read_companion_links().get(str(repo).strip().lower()) or []+ base = linked[0] if linked else (named_variant_base(fam, repo, name) or "")+ base = resolve_base_repo(fam, base)+ if not whole:+ companions = max(+ DB._companion_cache_bytes(prefer_ungated_mirror(base)),+ DB._companion_cache_bytes(base),+ )+ if not companions:+ return None # the text encoder and VAE are not downloaded yet+ weights = (weights or 0) + companions // mib+ hint = _image_variant_hint(fam.name, name, repo, base)+ return weights, estimate_image_runtime_mib(width = None, height = None, family = hint)+++def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:+ """(weights MiB, headroom MiB) for a video load: checkpoint file sizes plus the family's companion table."""+ from core.inference.diffusion import DiffusionBackend as DB+ from core.inference.diffusion_component_sizes import video_components_gb+ from core.inference.diffusion_memory import (+ estimate_gguf_resident_mib,+ estimate_safetensors_dense_mib,+ estimate_video_runtime_mib,+ file_size_mib,+ )+ from core.inference.video import _detect_load_family, resolve_video_model_kind+ from core.inference.video_families import resolve_video_base_repo+ from core.inference.video_minimax_h3 import is_h3_native+ from core.inference.video_moe_pair import moe_partner_filename++ repo, name = request.model_path, request.gguf_filename+ kind = resolve_video_model_kind(name, request.model_kind)+ fam = _detect_load_family(+ repo, name, request.family_override, request.display_repo_id, request.model_kind+ )+ if fam is None or is_h3_native(fam, kind):+ return None # H3 native runs its own server processes, priced elsewhere+ gb = 1000.0**3 / (1024 * 1024) # MiB per GB of the family tables+ if kind == "pipeline":+ components = video_components_gb(fam, repo, allow_hub = False)+ if components is None or not DB._cache_bytes(repo):+ return None+ weights = int(sum(components) * gb)+ else:+ names = [name] + ([moe_partner_filename(name)] if getattr(fam, "is_moe", False) else [])+ paths = [_cached_pick(repo, n) if n else None for n in names]+ components = video_components_gb(+ fam, resolve_video_base_repo(fam, request.base_repo, repo), allow_hub = False+ )+ if None in paths or components is None:+ return None+ sizes = [file_size_mib(p) for p in paths]+ price = estimate_gguf_resident_mib if kind == "gguf" else estimate_safetensors_dense_mib+ if None in sizes:+ return None+ weights = sum(price(s) for s in sizes) + int((components[1] + components[2]) * gb)+ preset = fam.resolution_presets[0]+ return weights, estimate_video_runtime_mib(+ width = preset[0], height = preset[1], num_frames = fam.default_num_frames+ )+++def _free_mib_by_gpu() -> tuple[Optional[dict[int, dict]], Optional[str]]:+ """{physical id: {free_mib, total_mib}} or (None, reason) when the host can't be priced."""+ try:+ from utils.hardware import DeviceType, get_device, get_visible_gpu_utilization+ if get_device() != DeviceType.CUDA:+ # MLX and other unified-memory hosts share the pool with the CPU: co-residency there is deferred.+ return None, "non_accelerator"+ util = get_visible_gpu_utilization() or {}+ except Exception as exc: # noqa: BLE001+ logger.warning("gpu_residency: GPU probe failed: %s", exc)+ return None, "probe_error"+ if util.get("index_kind") != "physical":+ return None, "unmapped_gpus"+ rows: dict[int, dict] = {}+ for device in util.get("devices") or ():+ total_gb, used_gb = device.get("vram_total_gb"), device.get("vram_used_gb")+ if device.get("unified_memory") or device.get("shared_memory") or not total_gb:+ rows[device["index"]] = {"unified": True}+ continue+ if used_gb is None:+ continue+ total = int(total_gb * 1024)+ rows[device["index"]] = {+ "free_mib": max(int(total - used_gb * 1024), 0),+ "total_mib": total,+ }+ return rows, None+++def _keeps_transformer_resident(need: MediaNeed, gpu: int, free_mib: int, total_mib: int) -> bool:+ """The diffusion planner's verdict for this load at the free memory left: "fits" never means streaming weights."""+ from core.inference.diffusion_memory import (+ DeviceMemory,+ plan_diffusion_memory,+ plan_keeps_transformer_resident,+ )++ ordinal = _ordinal_for_physical(gpu)+ plan = plan_diffusion_memory(+ target = SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = DeviceMemory("cuda", f"cuda:{ordinal}", "discrete_vram", free_mib, total_mib),+ model_dense_mib = need.weights_mib,+ runtime_headroom_mib = need.headroom_mib,+ # Planned as auto whatever the load asks: Balanced and Low VRAM hold less than an auto plan that stays resident.+ requested_mode = None,+ )+ return plan_keeps_transformer_resident(plan)+++def _ordinal_for_physical(gpu: int) -> int:+ try:+ from utils.hardware.hardware import get_parent_visible_gpu_ids+ parent = list(get_parent_visible_gpu_ids() or ())+ return parent.index(gpu) if gpu in parent else gpu+ except Exception: # noqa: BLE001+ return gpu+++def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dict]:+ """Whether ``owner`` fits next to the residents already loaded. Side-effect free; default-deny with a reason."""+ if not need_mib:+ return False, {"reason": "estimate_unavailable"}+ rows, reason = _free_mib_by_gpu()+ if rows is None:+ return False, {"reason": reason}+ records = _media_records()+ own = records.get(owner)+ # A resident still loading has not allocated its weights yet either, so live free VRAM misses all of it.+ loading = {name for name in records if name != owner and _media_busy(name)[0]}+ info: dict[str, Any] = {"need_mib": {}, "free_mib": {}, "short_mib": {}, "reason": "fits"}+ for gpu, mib in need_mib.items():+ row = rows.get(gpu)+ if row is None:+ return False, {"reason": "gpu_unknown", "gpu": gpu}+ if row.get("unified"):+ return False, {"reason": "unified_memory", "gpu": gpu}+ free = row["free_mib"]+ # An idle resident's generation headroom is not allocated yet; it will be when it generates.+ free -= sum(+ int(need[gpu]) if name in loading else getattr(need, "headroom_mib", 0)+ for name, need in records.items()+ if name != owner and gpu in need+ )+ if own is not None and gpu in own:+ # Reloading the same owner frees what it holds now.+ free += max(own[gpu] - getattr(own, "headroom_mib", 0), 0)+ info["need_mib"][gpu], info["free_mib"][gpu] = int(mib), max(free, 0)+ if free < mib + FIT_MARGIN_MIB:+ info["short_mib"][gpu] = int(mib + FIT_MARGIN_MIB - free)+ info["reason"] = "short"+ if info["short_mib"]:+ return False, info+ if getattr(need_mib, "kind", None) == "image":+ for gpu in need_mib:+ if not _keeps_transformer_resident(+ need_mib, gpu, info["free_mib"][gpu], rows[gpu]["total_mib"]+ ):+ info["reason"] = "would_stream"+ return False, info+ return True, info+++def chat_load_in_flight() -> bool:+ """Whether a chat load (or swap) is running. Its GPUs aren't known before it launches, so callers treat it as+ overlapping every GPU."""+ try:+ from core.inference import llama_keepwarm, model_slots+ from core.inference.llama_cpp import chat_load_active+ return bool(+ llama_keepwarm._load_lock.locked()+ or chat_load_active()+ or model_slots.loading is not None+ )+ except Exception: # noqa: BLE001+ return False+++_CHAT_LOADING_MESSAGE = (+ "Unsloth Studio is loading a chat model on this GPU. Retry once it has finished loading."+)+++class ChatLoadInFlightRefusal(Exception):+ """A media load under Always met a chat load whose size isn't known yet. Retryable, unlike the Always refusal."""++ def __init__(self) -> None:+ super().__init__(_CHAT_LOADING_MESSAGE)++ def as_http_exception(+ self,+ path: Optional[str] = None,+ *,+ openai_errors: bool = False,+ ):+ return chat_loading_http_exception(path, openai_errors = openai_errors)+++def chat_loading_http_exception(path: Optional[str] = None, *, openai_errors: bool = False):+ """The transient 409, shaped like GpuBusyForAnotherAccountError.as_http_exception."""+ from fastapi import HTTPException+ from utils.api_errors import error_body_for_path++ retry_after = K3_RETRY_AFTER_S+ if openai_errors and not (path and path.startswith("/v1/")):+ path = "/v1/"+ detail = (+ error_body_for_path(path, _CHAT_LOADING_MESSAGE, status = 409, code = "gpu_busy", param = "model")+ if path and path.startswith("/v1/")+ else {"error": "gpu_busy", "message": _CHAT_LOADING_MESSAGE, "retry_after": retry_after}+ )+ return HTTPException(status_code = 409, detail = detail, headers = {"Retry-After": str(retry_after)})+++def _gb(mib: int) -> str:+ return f"{mib / 1024:.1f}"+++def need_free_refusal(info: dict, slots: list[dict]) -> Exception:+ """The Always refusal for a load that doesn't fit next to the models already loaded."""+ from utils.account_context import is_owner_context++ short = info.get("short_mib") or {}+ gpu = max(short, key = short.get) if short else None+ if gpu is None or not is_owner_context():+ # No numbers (masked host, unified memory, probe error) or a managed account: the plain refusal.+ return arb.refusal_for(slots)+ message = (+ f"This model needs about {_gb(info['need_mib'][gpu])} GB of GPU memory, and {_gb(info['free_mib'][gpu])} GB "+ "is free next to the models already loaded. Keep multiple models loaded is set to Always, so nothing is "+ "unloaded to make room. Unload one of them, or choose When they fit."+ )+ first = slots[0] if slots else {}+ return arb.MultiModelAlwaysRefusal(+ message, kind = first.get("kind"), model = first.get("model") or None+ )+++def foreign_on(owner: str, account_id: Optional[str]) -> int:+ """Generations another account runs on ``owner``'s model; only these block evicting it."""+ try:+ if owner == arb.CHAT:+ from state import active_generations+ return active_generations.foreign_count(account_id)+ if owner == arb.DIFFUSION:+ from hub.services.models import account_access+ if account_access.generation_is_foreign("diffusion"):+ return 1+ return 0+ if owner == arb.VIDEO:+ import sys++ video = sys.modules.get("core.inference.video")+ running = video.generation_account_in_flight() if video is not None else None+ return int(running is not None and running != account_id)+ except Exception as exc: # noqa: BLE001 - unknown reads as busy, so nothing is torn down under a stream+ logger.warning("gpu_residency: could not count generations on %s: %s", owner, exc)+ return 1+ return 0+++def _overlaps(gpus: Optional[Iterable[int]], target: Optional[Iterable[int]]) -> bool:+ return gpus is None or target is None or bool(set(gpus) & set(target))+++def _record_evicted(slots: list[dict], evicted: Optional[list], freed: list[str]) -> None:+ """Report evictions the way acquire_for does (to ``evicted`` or the route's sink) and log them."""+ try:+ from core.inference.api_monitor import api_monitor+ except Exception: # noqa: BLE001+ api_monitor = None+ arb.note_evictions(slots, evicted)+ for slot in slots:+ model = slot.get("model") or slot.get("kind") or ""+ freed.append(model)+ if api_monitor is not None:+ try:+ api_monitor.record_lifecycle(event = "unload", model = model, reason = "evicted")+ except Exception: # noqa: BLE001+ pass+++def _evict_resident(+ name: str, acting: Optional[str], evicted: Optional[list], freed: list[str]+) -> None:+ """Evict one whole resident. Caller holds the arbiter lock."""+ busy = foreign_on(name, acting)+ if busy:+ raise arb.GpuBusyForAnotherAccountError(name, busy)+ slots = arb.resident_slots(name) or [{"kind": _MEDIA_KIND.get(name, "chat"), "model": ""}]+ logger.info("gpu_arbiter: evicting %s to make room", name)+ arb._EVICTORS[name]()+ arb._drop_resident(name)+ _record_evicted(slots, evicted, freed)+++def _free_steps(+ owner: str,+ gpu_ids: Optional[Iterable[int]],+ reason: str,+ evicted: Optional[list],+ *,+ need_mib: Optional[dict] = None,+ short_mib: int = 0,+ acting: Optional[str] = None,+ media_only: bool = False,+) -> list[str]:+ """Free the cheapest residents on ``gpu_ids`` first, re-checking after each step. Caller holds the arbiter lock.++ Order: anything still loading, idle video servers, the idle image or video model, idle chat models kept+ alongside, then the active chat model. Residents on other GPUs are never touched. With ``need_mib`` it stops once+ the load fits; without one (a chat launch prices itself) it stops after the first step that freed something.+ ``media_only`` stops before any chat model. "Keep model in GPU memory" does not protect a model here, as with Off.+ """+ target = set(gpu_ids) if gpu_ids is not None else None+ freed: list[str] = []++ def done() -> bool:+ if need_mib:+ return can_load_alongside(owner, need_mib)[0]+ return bool(freed)++ def overlapping() -> list[str]:+ return [n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)]++ # 1. Residents still loading can't be sized.+ for name in overlapping():+ if media_only and name not in MEDIA_OWNERS:+ continue+ loading = (+ chat_load_in_flight()+ if name == arb.CHAT+ else name in MEDIA_OWNERS and _media_busy(name)[0]+ )+ if loading:+ _evict_resident(name, acting, evicted, freed)+ if freed and done():+ return freed+ # 2. Idle H3 video servers sit outside every owner's teardown; one beside a video model on other GPUs stays.+ video_gpus = arb._view().get(arb.VIDEO, ...)+ if owner != arb.VIDEO and (video_gpus is ... or _overlaps(video_gpus, target)):+ try:+ from core.inference.video_minimax_h3 import release_h3_native_servers+ if release_h3_native_servers(reason):+ freed.append("video-server")+ except Exception as exc: # noqa: BLE001+ logger.warning("gpu_residency: could not release the idle video server: %s", exc)+ if freed and done():+ return freed+ # 3. The idle image or video model on these GPUs.+ for name in overlapping():+ if name in MEDIA_OWNERS and not any(_media_busy(name)) and not foreign_on(name, acting):+ _evict_resident(name, acting, evicted, freed)+ if done():+ return freed+ if media_only:+ return freed+ # 4. Idle chat models kept alongside, least recently used first.+ if arb.CHAT in overlapping() or owner == arb.CHAT:+ from core.inference import model_slots++ short = short_mib or (+ max((can_load_alongside(owner, need_mib)[1].get("short_mib") or {0: 0}).values())+ if need_mib+ else 0+ )+ victims = model_slots.eviction_victims(None, short, gpu_ids)+ if victims:+ reports = {id(victim): _kept_slot_report(victim) for victim in victims}+ # A slot a request reached since it was picked stays loaded, so it isn't reported.+ dropped = model_slots.evict_these(victims) or []+ _record_evicted([reports[id(victim)] for victim in dropped], evicted, freed)+ if dropped and done():+ return freed+ # 5. The active chat model.+ if owner != arb.CHAT and arb.CHAT in overlapping():+ _evict_resident(arb.CHAT, acting, evicted, freed)+ return freed+++def _kept_slot_report(slot: Any) -> dict:+ try:+ from routes.inference import _kept_slot_reports+ reports = _kept_slot_reports(slot)+ if reports:+ return reports[0]+ except Exception: # noqa: BLE001+ pass+ from core.inference.model_ids import public_model_id++ request = getattr(slot, "request", None)+ raw = str(getattr(request, "model_path", "") or "")+ model = public_model_id(raw) or raw+ return {"kind": "chat", "model": model, "account": getattr(slot, "account", None)}+++def media_fit_kwargs(+ owner: str,+ request: Any,+ gpu_ordinal: Optional[int],+ policy: Optional[str] = None,+) -> dict:+ """The acquire_for keywords for an image or video load: the policy and, when known, what it will hold."""+ policy = policy or current_policy()+ if policy == "off":+ return {}+ need = media_load_need_mib(owner, request, gpu_ordinal)+ kwargs: dict = {"policy": policy}+ if gpu_ordinal is not None:+ # Known even when the load can't be priced, so residents on other GPUs stay out of it.+ kwargs["gpu_ids"] = [gpu_ordinal]+ if need:+ kwargs["need_mib"] = need+ return kwargs+++def free_for(+ owner: str,+ gpu_ids: Optional[Iterable[int]],+ reason: str,+ evicted: Optional[list] = None,+ *,+ need_mib: Optional[dict] = None,+ short_mib: int = 0,+ media_only: bool = False,+) -> list[str]:+ """Free residents on ``gpu_ids`` so ``owner`` fits, cheapest first. Appends {kind, model} to ``evicted``."""+ from utils.account_context import current_account_id++ acting = current_account_id()+ with arb._lock:+ arb._sync_from_view()+ return _free_steps(+ owner,+ gpu_ids,+ reason,+ evicted,+ need_mib = need_mib,+ short_mib = short_mib,+ acting = acting,+ media_only = media_only,+ )+++def _claim_holds_anything(name: str) -> bool:+ """Whether a kept claim holds or is loading a model; unreadable counts as yes. Caller holds the lock."""+ if arb.resident_slots(name):+ return True+ return arb._chat_occupied() if name == arb.CHAT else arb._media_occupied(name)+++def coreside_locked(+ owner: str,+ *,+ policy: str,+ need_mib: Optional[dict],+ gpu_ids: Optional[Iterable[int]],+ allow_evict: bool,+ evict_first: bool,+ acting: Optional[str],+ evicted: Optional[list],+) -> None:+ """Make room for ``owner`` under When they fit / Always, evicting only what has to go. Caller holds the lock.++ Residents on other GPUs stay. A chat load joins priced image and video models and lets its launch price them. An+ image or video load joins when it fits, else When they fit frees the cheapest residents and Always refuses.+ Anything that can't be priced is evicted first under When they fit, as with Off.+ """+ target = (+ arb.physical_gpu_ids(gpu_ids)+ if gpu_ids is not None+ else (frozenset(need_mib) if need_mib else None)+ )+ overlap = [+ name for name, gpus in arb._view().items() if name != owner and _overlaps(gpus, target)+ ]+ for name in [n for n in overlap if n not in arb._KEPT_UNDER_ALWAYS]:+ # A Decision API server is evicted as before.+ if not allow_evict:+ raise arb.GpuOwnerBusyError(name)+ _evict_resident(name, acting, evicted, [])+ kept = [n for n in overlap if n in arb._KEPT_UNDER_ALWAYS]+ if policy == "always":+ # A claim whose load failed holds nothing, so Always evicts it rather than refusing (gpu_arbiter.acquire_for).+ for name in [n for n in kept if not _claim_holds_anything(n)]:+ if not allow_evict:+ raise arb.GpuOwnerBusyError(name)+ _evict_resident(name, acting, evicted, [])+ kept.remove(name)++ def evict_or_refuse(names: list[str]) -> None:+ if not names:+ return+ if policy == "always":+ raise arb.refusal_for(+ [slot for n in names for slot in arb.resident_slots(n)], arb.KIND_LABELS[names[0]]+ )+ if not allow_evict:+ raise arb.GpuOwnerBusyError(names[-1])+ for name in names:+ _evict_resident(name, acting, evicted, [])+ kept.remove(name)++ if not kept:+ return+ if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_load_in_flight():+ # Its size isn't known until it launches.+ if policy == "always":+ raise ChatLoadInFlightRefusal()+ evict_or_refuse([arb.CHAT])+ if owner == arb.CHAT and not need_mib:+ if evict_first:+ # No estimate before launch (a non-GGUF load, or a load with no retry to make room).+ return evict_or_refuse(list(kept))+ records = _media_records()+ evict_or_refuse([n for n in kept if n not in records or _media_busy(n)[0]])+ if kept and _free_mib_by_gpu()[0] is None:+ evict_or_refuse(list(kept))+ return+ if need_mib is None or evict_first:+ return evict_or_refuse(list(kept))+ fits, info = can_load_alongside(owner, need_mib)+ if fits:+ return+ if policy == "always":+ raise need_free_refusal(info, [slot for n in kept for slot in arb.resident_slots(n)])+ if not allow_evict:+ raise arb.GpuOwnerBusyError(kept[-1])+ _free_steps(owner, target, f"GPU memory for {owner}", evicted, need_mib = need_mib, acting = acting)+ if not can_load_alongside(owner, need_mib)[0]:+ # Still short: unload the rest on these GPUs, as Off does, and let the planner offload.+ for name in [+ n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)+ ]:+ _evict_resident(name, acting, evicted, [])diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex e3c2dfcd47e..218236da468 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -639,6 +639,23 @@ def _net_of_held_vram(gpu_mem, held: dict[int, int]):]+def _net_of_unallocated_vram(gpu_mem, unallocated: dict[int, int]):+ """Take from each GPU's live free MiB what a resident image or video model has not allocated yet (its generation+ headroom): the total cap misses it once another process uses the card. Totals of 0 are left as is."""+ if not unallocated:+ return gpu_mem+ return [+ (idx, max(0, free - unallocated.get(idx, 0)) if total > 0 else free, total)+ for idx, free, total in gpu_mem+ ]+++def _net_of_other_models_vram(gpu_mem, held: dict[int, int], unallocated: dict[int, int]):+ """Live free less unallocated media VRAM, then capped at total less ``held``. The other order charges the+ headroom twice whenever the total cap binds (nothing else on the card)."""+ return _net_of_held_vram(_net_of_unallocated_vram(gpu_mem, unallocated), held)++_serving_backends: "weakref.WeakSet[LlamaCppBackend]" = weakref.WeakSet()# A WeakSet is not safe to iterate while another thread adds or drops a backend._serving_backends_lock = threading.Lock()@@ -26717,7 +26734,9 @@ class LlamaCppBackend:_gpu_mem = self._get_gpu_memory(binary, for_llama_server = not gpu_ids)_held_by_others = self._other_planned_vram_mib()_shared_gpus = frozenset(_held_by_others)- _gpu_mem = _net_of_held_vram(_gpu_mem, _held_by_others)+ _gpu_mem = _net_of_other_models_vram(+ _gpu_mem, _held_by_others, self._media_unallocated_vram_mib()+ )# Every present device gated out (#7624). Left alone the launch# takes the `--fit on` arm with `gpu_indices` still None, so no# mask is written, the child enumerates every unsupported card and@@ -36245,7 +36264,7 @@ class LlamaCppBackend:)def _other_planned_vram_mib(self) -> dict[int, int]:- """MiB per GPU the other serving backends' running servers hold."""+ """MiB per GPU the other serving backends' running servers and the resident image/video models hold."""held: dict[int, int] = {}serving = _serving_backends_snapshot()if self not in serving:@@ -36255,8 +36274,21 @@ class LlamaCppBackend:continuefor idx, mib in getattr(backend, "_planned_vram_mib", {}).items():held[idx] = held.get(idx, 0) + mib+ # Weights plus generation headroom, the full figure _net_of_held_vram needs (empty under Off).+ from core.inference.gpu_residency import media_held_mib++ for idx, mib in media_held_mib().items():+ held[idx] = held.get(idx, 0) + mibreturn held+ def _media_unallocated_vram_mib(self) -> dict[int, int]:+ """MiB per GPU of the resident image/video models' price not yet allocated (empty under Off)."""+ if self not in _serving_backends_snapshot():+ return {}+ from core.inference.gpu_residency import media_unallocated_mib++ return media_unallocated_mib()+def _note_server_pid(self, pid: int) -> None:if getattr(self, "_owns_pidfile", True):self._record_server_pid(pid)diff --git a/studio/backend/core/inference/media_auto_switch.py b/studio/backend/core/inference/media_auto_switch.pyindex 28c50593361..ffc41e75a14 100644--- a/studio/backend/core/inference/media_auto_switch.py+++ b/studio/backend/core/inference/media_auto_switch.py@@ -324,15 +324,26 @@ async def _gated_start_load(re-plan and the load registration take."""from fastapi import HTTPException+ from core.inference import gpu_arbiter, gpu_residencyfrom core.inference.media_keepwarm import admission_gatefrom core.inference.llama_keepwarm import inference_lifecycle_gate- needed = (- (inference_lifecycle_gate(), admission_gate(DIFFUSION), admission_gate(VIDEO))- if takes_the_gpu- else (admission_gate(owner),)- )try:+ # Under Always a chat load's GPUs aren't known yet, so it overlaps every card: refuse now+ # (retryable) instead of queueing on chat's gate behind it. Off and fit keep the wait, and+ # so does a CPU chat load (no GPU claim) or one on known GPUs (the arbiter checks overlap).+ if (+ takes_the_gpu+ and gpu_residency.chat_load_in_flight()+ and gpu_arbiter.residents().get(gpu_arbiter.CHAT, ...) is None+ and await asyncio.to_thread(gpu_residency.current_policy) == "always"+ ):+ raise gpu_residency.chat_loading_http_exception(openai_errors = openai_errors)+ needed = (+ (inference_lifecycle_gate(), admission_gate(DIFFUSION), admission_gate(VIDEO))+ if takes_the_gpu+ else (admission_gate(owner),)+ )async with contextlib.AsyncExitStack() as gates:for gate in needed:await bounded(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 1d7f67b04ea..5c07e1c5530 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -12482,6 +12482,194 @@ def release_chat_after_kept_models() -> bool:return model_slots.in_slot(None, lambda: release_if(CHAT, zero_vram_primary_only))+def release_chat_after_cpu_fallback(backend, identifier) -> bool:+ """A primary whose GPU start fell back to the CPU drops CHAT, unless something else still holds VRAM."""+ from core.inference.gpu_arbiter import CHAT, release_if++ def only_cpu_left() -> bool:+ if getattr(backend, "active_model_name", None) != identifier:+ return False+ entry = (getattr(backend, "models", None) or {}).get(identifier)+ if not isinstance(entry, dict) or entry.get("audio_device_fallback") != "cpu":+ return False+ llama = get_llama_cpp_backend()+ return not model_slots.holds_vram() and (not llama.is_active or llama.holds_no_vram)++ return model_slots.in_slot(None, lambda: release_if(CHAT, only_cpu_left))+++# True while load_model_gated can free memory and retry, so only there does a primary refuse a partial GPU fit.+_primary_can_retry_fit: contextvars.ContextVar[bool] = contextvars.ContextVar(+ "primary_can_retry_fit", default = False+)+++def _placement_gpu_ids(placement) -> Optional[list[int]]:+ # Only GGUF and native audio resolve their GPUs; any other chat load keeps the ids it asked for.+ if placement.resolved_gpu_ids is not None:+ return placement.resolved_gpu_ids+ return placement.requested_gpu_ids+++def _media_on_target_gpus(placement) -> bool:+ """Whether an image or video model kept loaded beside chat holds memory on the GPUs this load targets."""+ from core.inference.gpu_residency import media_held_mib++ held = media_held_mib()+ gpus = None if placement.gpu_ids_are_vulkan_ordinals else _placement_gpu_ids(placement)+ return bool(held) and (not gpus or bool(set(held) & set(gpus)))+++def _primary_refuses_partial_fit(serving: bool, policy: str, placement) -> bool:+ """Whether a primary GGUF load refuses a partial GPU fit beside an image or video model, to retry once room is+ made. Not under Always while a primary serves: the refusal lands after its teardown, leaving nothing loaded."""+ return (+ _primary_can_retry_fit.get()+ and not (serving and policy == "always")+ and _media_on_target_gpus(placement)+ )+++def _visible_ordinals(physical_ids) -> Optional[list[int]]:+ """Visible device ordinals for physical GPU ids, or None when one isn't visible (then every GPU counts)."""+ try:+ from utils.hardware.hardware import get_parent_visible_gpu_ids+ parent = list(get_parent_visible_gpu_ids() or ())+ except Exception: # noqa: BLE001+ return None+ if not parent:+ return [int(g) for g in physical_ids]+ if any(int(g) not in parent for g in physical_ids):+ return None+ return [parent.index(int(g)) for g in physical_ids]+++def _chat_fit_kwargs(+ config,+ placement,+ policy: Optional[str] = None,+ credit_gb: Optional[dict] = None,+) -> dict:+ """How the CHAT acquire shares the GPU under When they fit / Always.++ audio.cpp and native audio are priced from the placement estimate. A GGUF launch prices itself (it refuses a+ partial fit and load_model_gated makes room). Anything else has no estimate before launch: When they fit+ evicts first, Always tries the load and turns an out-of-memory failure into its refusal. ``credit_gb`` is what+ the outgoing primary frees per GPU when this load replaces it.+ """+ if policy is None:+ from core.inference.gpu_residency import current_policy+ policy = current_policy()+ if policy == "off":+ return {}+ kwargs: dict = {"policy": policy}+ gpus = _placement_gpu_ids(placement)+ ordinals = (+ _visible_ordinals(gpus) if gpus and not placement.gpu_ids_are_vulkan_ordinals else None+ )+ if ordinals is not None:+ # Only residents on these GPUs are weighed or unloaded; the arbiter maps them back to physical ids.+ kwargs["gpu_ids"] = ordinals+ if placement.native_required_gb and gpus and not placement.gpu_ids_are_vulkan_ordinals:+ credit = credit_gb or {}+ # Net of the outgoing primary, so it may be negative: a smaller replacement needs no extra room.+ kwargs["need_mib"] = {+ int(g): int((placement.native_required_gb / len(gpus) - credit.get(int(g), 0.0)) * 1024)+ for g in gpus+ }+ elif policy == "fit" and not (config.is_gguf and _primary_can_retry_fit.get()):+ kwargs["evict_first"] = True+ return kwargs+++def _outgoing_chat_credit_gb(+ placement, policy: str, replacing: bool, availability: "Optional[_NativeAudioAvailability]"+) -> tuple[Optional[dict], Optional[dict]]:+ """VRAM the primary chat model frees per GPU when a priced native load replaces it beside an image or video model.++ Returns (credit, wait): the CHAT acquire charges the new load less the credit, and the load waits for ``wait`` GB+ to be free after the teardown. (None, None) when nothing is credited.+ """+ if not (+ replacing+ and policy != "off"+ and placement.native_required_gb+ and placement.resolved_gpu_ids+ and _media_on_target_gpus(placement)+ ):+ return None, None+ from core.inference.gpu_arbiter import CHAT++ if availability is not None and availability.credited == CHAT:+ # The automatic placement already measured the outgoing chat model and waits for it.+ return availability.reclaimable_gpu_gb, None+ chat = _native_audio_post_handoff_free_gb(CHAT)+ if chat is None:+ return None, None+ target = list(placement.resolved_gpu_ids)+ share = float(placement.native_required_gb) / len(target)+ return chat.reclaimable_gpu_gb, {int(g): share for g in target}+++async def _make_room_for_primary(request: LoadRequest, exc, evicted: list) -> LoadRequest:+ """A primary chat load refused a partial GPU fit beside an image or video model: return the request to retry.++ When they fit frees the image or video model (cheapest first), else lets llama-server fit what it can. Always+ only accepts a smaller context; otherwise it refuses, since nothing may be unloaded.+ """+ from core.inference import gpu_arbiter, gpu_residency++ if exc.capped:+ return request.model_copy(update = {"force_alongside": True})+ if await asyncio.to_thread(gpu_residency.current_policy) == "always":+ rows = (await asyncio.to_thread(gpu_residency._free_mib_by_gpu))[0] or {}+ gpus = [g for g in exc.gpu_indices or () if "free_mib" in rows.get(g, {})]+ free = sum(rows[g]["free_mib"] for g in gpus)+ info = {+ "need_mib": {0: free + exc.short_mib},+ "free_mib": {0: free},+ "short_mib": {0: exc.short_mib},+ }+ held, short = gpu_arbiter.residents(), set(exc.gpu_indices or ())+ # Name only models that could be in the way: skip one known to sit on other GPUs.+ slots = [+ s+ for o in gpu_residency.MEDIA_OWNERS+ if not (short and held.get(o) and not held[o] & short)+ for s in gpu_arbiter.resident_slots(o)+ ]+ raise gpu_residency.need_free_refusal(info if gpus else {}, slots)+ freed = await _free_for_chat(exc, evicted)+ # Anything freed (an idle video server counts) earns a retry at full size; else llama-server fits what it can.+ return request if freed else request.model_copy(update = {"force_alongside": True})+++async def _free_for_chat(+ exc,+ evicted: list,+ media_only: bool = False,+) -> list[str]:+ """Free the cheapest residents on the GPUs a chat load came up short on; ``media_only`` keeps every chat model."""+ from core.inference import gpu_arbiter, gpu_residency++ # The route's eviction sink when it keeps one (it reports the slots itself), else the plain list.+ sink = gpu_arbiter.eviction_sink.get()+ slots: list = sink if sink is not None else []+ before = len(slots)+ freed = await asyncio.to_thread(+ gpu_residency.free_for,+ gpu_arbiter.CHAT,+ exc.gpu_indices,+ "a chat model",+ slots,+ short_mib = exc.short_mib,+ media_only = media_only,+ )+ if sink is None:+ evicted += [s["model"] for s in slots[before:] if s.get("model")]+ return freed++def reap_dead_managed_engine(backend) -> None:"""A crashed engine leaves nothing resident, so its claim and sharers go with it."""from core.inference.gpu_arbiter import CHAT, holds@@ -16207,20 +16395,38 @@ class _NativeAudioAvailability(NamedTuple):effective_free_gb: dict[int, float]reclaimable_gpu_gb: dict[int, float]owner_snapshot: tuple[Optional[str], int] = (None, 0)+ # The owner whose memory is credited, when not the last claimant (owner_snapshot[0]).+ credited_owner: Optional[str] = None++ @property+ def credited(self) -> Optional[str]:+ return self.credited_owner or self.owner_snapshot[0]-def _native_audio_post_handoff_free_gb() -> Optional[_NativeAudioAvailability]:- """Free VRAM after the outgoing Studio owner is torn down, or ``None``.+def _native_audio_post_handoff_free_gb(+ outgoing: Optional[str] = None,+) -> Optional[_NativeAudioAvailability]:+ """Free VRAM after the outgoing Studio owner (``outgoing``, default the last claimant) is torn down, or ``None``.Only memory attributable to a backend this CHAT handoff provably evicts iscredited. Training, STT, export and unknown/external processes remain inthe live-used figure. Missing attribution fails closed."""import utils.hardware as hardware- from core.inference.gpu_arbiter import CHAT, DIFFUSION, VIDEO, owner_snapshot+ from core.inference.gpu_arbiter import CHAT, DIFFUSION, VIDEO, holds, owner_snapshotoutgoing_owner = owner_snapshot()- owner = outgoing_owner[0]+ if (+ outgoing is None+ and outgoing_owner[0] in (DIFFUSION, VIDEO)+ and routed_slot.get() is None+ and holds(CHAT)+ ):+ # A primary load replaces the chat model; an image or video model claimed after it may stay beside.+ chat = _native_audio_post_handoff_free_gb(CHAT)+ if chat is not None:+ return chat+ owner = outgoing or outgoing_owner[0]live: dict[int, tuple[float, float]] = {}reclaimable: dict[int, float] = {}@@ -16378,6 +16584,7 @@ def _native_audio_post_handoff_free_gb() -> Optional[_NativeAudioAvailability]:},reclaimable_gpu_gb = reclaimable,owner_snapshot = outgoing_owner,+ credited_owner = owner,)@@ -18575,6 +18782,7 @@ async def load_model_gated(action = "Loading a model",)+ _primary_can_retry_fit.set(True)while True:try:response = await _run_tracked_load_model_impl(@@ -18589,6 +18797,8 @@ async def load_model_gated(except (GpuMemoryShortError, HTTPException) as exc:measured = isinstance(exc, GpuMemoryShortError)if not measured:+ if extra is None and _ran_out_of_memory(exc):+ await asyncio.to_thread(_refuse_oom_beside_media_under_always)# A non-GGUF load reports no fit up front: its out-of-memory failure# makes room the same way, one idle kept model at a time.if extra is None or not _ran_out_of_memory(exc):@@ -18599,6 +18809,10 @@ async def load_model_gated(if isinstance(detail, dict)else str(detail))+ if extra is None:+ # Only a primary beside an image or video model refuses a partial fit.+ request = await _make_room_for_primary(request, exc, evicted)+ continuefrom utils.multi_model_settings import get_multi_model_modeif await asyncio.to_thread(get_multi_model_mode) == "always":@@ -18612,6 +18826,9 @@ async def load_model_gated(raise refusal_for(await asyncio.to_thread(chat_eviction_reports)) from exc+ # An idle image or video model goes before any chat model, as gpu_residency orders it.+ if await _free_for_chat(exc, evicted, media_only = True):+ continue# No chat may start on a victim between its pick and teardown.async with inference_lifecycle_gate() if new_slot else nullcontext():victims = await asyncio.to_thread(@@ -18652,6 +18869,8 @@ async def load_model_gated(# the new one is placed on a GPU it no longer shares.await asyncio.to_thread(_unload_primary)note_keyed(replaced_reports)+ # No retry follows this load, so it must not refuse a partial fit beside media.+ _primary_can_retry_fit.set(False)response = await _run_tracked_load_model_impl(request,fastapi_request,@@ -18715,12 +18934,30 @@ async def load_model_gated(await asyncio.to_thread(model_slots.drop, extra)await asyncio.to_thread(release_chat_gpu_claim)finally:+ _primary_can_retry_fit.set(False)# A slot that would not stop stays stuck, but this attempt is over either way.with _scoped_load_attempts_lock:_pending_load_attempts.pop(attempt.token, None)_finish_load_attempt(attempt)+def _refuse_oom_beside_media_under_always() -> None:+ """Under Always a chat load that ran out of memory beside an image or video model is refused, not retried."""+ from core.inference import gpu_arbiter, gpu_residency++ held = gpu_arbiter.residents()+ chat_gpus = held.get(gpu_arbiter.CHAT)+ # Only a model sharing a GPU with chat could have caused it; None (unknown) shares every GPU.+ slots = [+ s+ for o in gpu_residency.MEDIA_OWNERS+ if o in held and (chat_gpus is None or held[o] is None or held[o] & chat_gpus)+ for s in gpu_arbiter.resident_slots(o)+ ]+ if slots and gpu_residency.current_policy() == "always":+ raise gpu_arbiter.refusal_for(slots)++def _ran_out_of_memory(exc: HTTPException) -> bool:if isinstance(exc.detail, dict): # a classified audio.cpp failure# Out of CPU RAM: evicting GPU models frees none of it.@@ -19807,7 +20044,9 @@ async def _load_model_impl()if speech_codec_path is not None:gguf_intent = replace(gguf_intent, audio_codec_path = speech_codec_path)- if not replacing and not request.force_alongside:+ if not request.force_alongside and (+ not replacing or _primary_refuses_partial_fit(serving, multi_model_mode, placement)+ ):gguf_intent = replace(gguf_intent, refuse_partial_gpu_fit = True)same_loaded_model = llama_backend.matches_load_source(gguf_intent)_loaded_companion_roots = tuple(@@ -20041,10 +20280,10 @@ async def _load_model_impl(post_handoff_needed_gb = float((final_training_info or initial_training_info or {}).get("needed_gb") or required)- if expected_native_owner[0] in (DIFFUSION, VIDEO):+ if final_availability.credited in (DIFFUSION, VIDEO):post_media_handoff_gpu = selected_gpupost_media_handoff_needed_gb = post_handoff_needed_gb- elif expected_native_owner[0] == CHAT:+ elif final_availability.credited == CHAT:post_chat_handoff_expected_free_gb = {selected_gpu: post_handoff_needed_gb}if chat_load_needs_gpu:@@ -20057,6 +20296,18 @@ async def _load_model_impl(if not replacing:handoff_kwargs["alongside"] = Truehandoff_kwargs.update(policy_kwargs(multi_model_mode))+ outgoing_chat_gb, chat_wait_gb = await asyncio.to_thread(+ _outgoing_chat_credit_gb,+ placement,+ multi_model_mode,+ replacing,+ final_availability if expected_native_owner is not None else None,+ )+ if post_chat_handoff_expected_free_gb is None:+ post_chat_handoff_expected_free_gb = chat_wait_gb+ handoff_kwargs.update(+ _chat_fit_kwargs(config, placement, multi_model_mode, outgoing_chat_gb)+ )await asyncio.to_thread(acquire_for_request,CHAT,@@ -20068,13 +20319,9 @@ async def _load_model_impl(if isinstance(exc, OwnerChangedError):raise HTTPException(status_code = 409, detail = str(exc)) from excraise- # Wait for the freed VRAM only when the media model was actually evicted.- if (- post_media_handoff_gpu is not None- and post_media_handoff_needed_gb is not None- and not holds(expected_native_owner[0])- ):- settled = await asyncio.to_thread(+ if post_media_handoff_gpu is not None and post_media_handoff_needed_gb is not None:+ # Wait for the freed VRAM only when the media model was actually evicted.+ settled = holds(expected_native_owner[0]) or await asyncio.to_thread(_wait_for_native_audio_gpu_free_gb,post_media_handoff_gpu,post_media_handoff_needed_gb,@@ -20087,9 +20334,10 @@ async def _load_model_impl("was stopped. Retry the audio-model load."),)- # The outgoing media allocation is gone now. Re-run the training- # policy against live free VRAM only, so a concurrent training- # change cannot hide behind the earlier reclaimable snapshot.+ # Re-run the training policy against live free VRAM only: the+ # media allocation is gone, or was kept beside and never freed,+ # and a concurrent training change cannot hide behind the+ # earlier reclaimable snapshot.live_placement = placement._replace(# Automatic selection is already fixed at this point. Bind# the live-only training recheck to that exact card, or a@@ -20535,6 +20783,10 @@ async def _load_model_impl(except Exception:pass+ if replacing and chat_load_needs_gpu:+ # A GPU start that fell back to the CPU holds no VRAM, so the claim it took would evict media for nothing.+ await asyncio.to_thread(release_chat_after_cpu_fallback, backend, config.identifier)+return LoadResponse(engine = request.engine,engine_parallelism = request.engine_parallelism,@@ -50601,7 +50853,6 @@ async def load_diffusion_model_gated(begin_media_load,guarded_media_begin,note_evictions,- policy_kwargs,refuse_replacing_media,require_no_foreign_generations,release,@@ -50945,6 +51196,12 @@ async def load_diffusion_model_gated()if needs_gpu:+ from core.inference.gpu_residency import media_fit_kwargs++ fit_kwargs = await asyncio.to_thread(+ media_fit_kwargs, DIFFUSION, request, gpu_ordinal, multi_model_mode+ )+# Register the in-flight load UNDER the arbiter lock: otherwise a competing acquire in that gap evicts DIFFUSION before# the load is marked, finds nothing to cancel, and both allocate at once. The training admission wraps the same span.def _acquire_and_begin():@@ -50956,7 +51213,7 @@ async def load_diffusion_model_gated(guarded_media_begin(DIFFUSION, multi_model_mode, media_names, _begin_load),- **policy_kwargs(multi_model_mode),+ **fit_kwargs,)finally:eviction_sink.reset(sink_token)diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.pyindex 7b9ad7347f3..b4bcd90c4ef 100644--- a/studio/backend/routes/video.py+++ b/studio/backend/routes/video.py@@ -343,7 +343,6 @@ async def load_video_model_gated(acquire_for_request,eviction_sink,note_evictions,- policy_kwargs,begin_media_load,guarded_media_begin,refusal_for,@@ -509,7 +508,13 @@ async def load_video_model_gated(# Register the in-flight load UNDER the arbiter lock: otherwise a competing acquire in that gap evicts VIDEO# before the load is marked, finds nothing to cancel, and both allocate at once. The training admission wraps# the same span.+ from core.inference.gpu_residency import media_fit_kwargsfrom routes.inference import _diffusion_training_admission++ fit_kwargs = await asyncio.to_thread(+ media_fit_kwargs, VIDEO, request, gpu_ordinal, multi_model_mode+ )+def _acquire_and_begin():token = eviction_sink.set(evicted)try:@@ -517,7 +522,7 @@ async def load_video_model_gated(return acquire_for_request(VIDEO,guarded_media_begin(VIDEO, multi_model_mode, media_names, _begin_load),- **policy_kwargs(multi_model_mode),+ **fit_kwargs,)finally:eviction_sink.reset(token)diff --git a/studio/backend/tests/test_audio_cpu_fallback_claim.py b/studio/backend/tests/test_audio_cpu_fallback_claim.pynew file mode 100644index 00000000000..7d93106d809--- /dev/null+++ b/studio/backend/tests/test_audio_cpu_fallback_claim.py@@ -0,0 +1,136 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A chat load that took the GPU but fell back to the CPU drops CHAT once nothing holds VRAM.++Otherwise the next image or video load evicts a model that never used the GPU.+"""++import inspect++import pytest++import core.inference.model_slots as model_slots+import routes.inference as inf+from core.inference import gpu_arbiter+++class FakeLlama:+ holds_no_vram = False++ def __init__(self, active = False):+ self.is_loaded = self.is_active = active+ self.model_identifier = "org/chat-GGUF" if active else None+++class FakeOrchestrator:+ def __init__(+ self,+ active = None,+ entry = None,+ ):+ self.active_model_name = active+ self.models = {active: dict(entry or {})} if active else {}+ self.loading_models = ()+++PRIMARY = "org/tts-audio-cpp"+++def evicted(monkeypatch):+ calls = []+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)+ monkeypatch.setitem(gpu_arbiter._EVICTORS, gpu_arbiter.CHAT, lambda: calls.append("chat"))+ monkeypatch.setitem(+ gpu_arbiter._EVICTORS, gpu_arbiter.DIFFUSION, lambda: calls.append("diffusion")+ )+ monkeypatch.setattr(gpu_arbiter, "_release_idle_video_servers", lambda reason: None)+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama())+ monkeypatch.setattr(model_slots, "slots", [])+ monkeypatch.setattr(model_slots, "stuck", [])+ monkeypatch.setattr(model_slots, "loading", None)+ return calls+++def _primary(entry):+ return FakeOrchestrator(PRIMARY, entry)+++def test_a_cpu_fallback_primary_drops_chat_so_media_evicts_nothing(evicted):+ assert inf.release_chat_after_cpu_fallback(_primary({"audio_device_fallback": "cpu"}), PRIMARY)+ assert gpu_arbiter.current_owner() is None+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)+ assert evicted == [] and gpu_arbiter.current_owner() == gpu_arbiter.DIFFUSION+++ "entry", [{}, {"audio_device_fallback": None}, {"audio_device_fallback": "gpu"}]+)+def test_no_recorded_fallback_keeps_chat(evicted, entry):+ assert not inf.release_chat_after_cpu_fallback(_primary(entry), PRIMARY)+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT+++def test_a_replaced_primary_keeps_chat(evicted):+ backend = _primary({"audio_device_fallback": "cpu"})+ backend.active_model_name = "org/other"+ assert not inf.release_chat_after_cpu_fallback(backend, PRIMARY)+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT+++def test_a_kept_model_loading_keeps_chat(evicted, monkeypatch):+ slot = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")+ monkeypatch.setattr(model_slots, "loading", (slot, "org/kept-GGUF"))+ assert not inf.release_chat_after_cpu_fallback(+ _primary({"audio_device_fallback": "cpu"}), PRIMARY+ )+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT+++def test_a_kept_model_on_the_gpu_keeps_chat(evicted, monkeypatch):+ kept = model_slots.ExtraSlot(FakeLlama(active = True), FakeOrchestrator(), "owner")+ monkeypatch.setattr(model_slots, "slots", [kept])+ assert model_slots.holds_vram()+ assert not inf.release_chat_after_cpu_fallback(+ _primary({"audio_device_fallback": "cpu"}), PRIMARY+ )+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT+ # A CPU-only kept model holds no VRAM, so the claim goes.+ kept.llama._gpu_offload_active = False+ assert inf.release_chat_after_cpu_fallback(_primary({"audio_device_fallback": "cpu"}), PRIMARY)+++def test_a_primary_llama_server_on_the_gpu_keeps_chat(evicted, monkeypatch):+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama(active = True))+ assert not inf.release_chat_after_cpu_fallback(+ _primary({"audio_device_fallback": "cpu"}), PRIMARY+ )+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT+++def test_a_kept_audio_cpp_fallback_beside_a_gpu_primary_keeps_chat(evicted, monkeypatch):+ # The kept model's own orchestrator is not the primary: the call site is gated on `replacing`,+ # and even if it ran, the primary llama-server's VRAM keeps the claim.+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama(active = True))+ kept = FakeOrchestrator("org/kept-audio-cpp", {"audio_device_fallback": "cpu"})+ monkeypatch.setattr(model_slots, "slots", [model_slots.ExtraSlot(FakeLlama(), kept, "owner")])+ assert not inf.release_chat_after_cpu_fallback(kept, "org/kept-audio-cpp")+ assert gpu_arbiter.current_owner() == gpu_arbiter.CHAT+ gpu_arbiter.acquire_for(gpu_arbiter.DIFFUSION, lambda: None)+ assert evicted == ["chat"]+++def test_the_release_runs_for_a_primary_gpu_load_before_the_response():+ source = inspect.getsource(inf._load_model_impl)+ call = "await asyncio.to_thread(release_chat_after_cpu_fallback, backend, config.identifier)"+ assert source.count(call) == 1+ gate = "if replacing and chat_load_needs_gpu:\n"+ assert source.count(gate) == 1+ at = source.index(call)+ # The gate opens the block, then only the comment line sits between it and the call.+ assert source[source.index(gate) + len(gate) : at].strip().startswith("#")+ assert source[source.index(gate) + len(gate) : at].count("\n") == 1+ # The standard path's response statement follows directly, so the call is never inside its arguments.+ after = source[at + len(call) :]+ assert after.lstrip().startswith("return LoadResponse(")diff --git a/studio/backend/tests/test_gpu_arbiter.py b/studio/backend/tests/test_gpu_arbiter.pyindex 84b51914744..35035e5dc87 100644--- a/studio/backend/tests/test_gpu_arbiter.py+++ b/studio/backend/tests/test_gpu_arbiter.py@@ -61,6 +61,347 @@ def test_reacquiring_same_owner_does_not_evict(calls):assert arb.current_owner() == arb.CHAT+# Keep multiple models loaded set to When they fit / Always (gpu_residency.coreside_locked) next to Off.+++def cores(calls, monkeypatch):+ import sys+ import types++ import core.inference.gpu_residency as res+ import utils.hardware.hardware # noqa: F401+ from core.inference import model_slots, video_minimax_h3+ from core.inference.api_monitor import api_monitor+ from state import active_generations++ env = types.SimpleNamespace(calls = calls, lifecycle = [], h3 = [], probes = [])+ for name, value in {+ "_residents": {},+ "_media_needs": {},+ "_accounts": {},+ "_owner_account": None,+ "_prior_account": None,+ }.items():+ monkeypatch.setattr(arb, name, value)+ monkeypatch.setitem(arb._EVICTORS, arb.VIDEO, lambda: calls.append("evict-video"))+ for owner, kind in ((arb.CHAT, "chat"), (arb.DIFFUSION, "image"), (arb.VIDEO, "video")):+ monkeypatch.setitem(+ arb._SLOT_REPORTERS, owner, lambda kind = kind: [{"kind": kind, "model": f"org/{kind}"}]+ )+ monkeypatch.setattr(arb, "_release_idle_video_servers", env.h3.append)+ monkeypatch.setattr(+ sys.modules["utils.hardware.hardware"], "get_parent_visible_gpu_ids", lambda: []+ )+ monkeypatch.setattr(res, "chat_load_in_flight", lambda: False)+ monkeypatch.setattr(res, "_media_busy", lambda owner: (False, False))+ monkeypatch.setattr(res, "_loaded_or_loading", lambda owner: True)+ rows = {gpu: {"free_mib": 8192, "total_mib": 24576} for gpu in (0, 1)}+ monkeypatch.setattr(res, "_free_mib_by_gpu", lambda: (rows, None))+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *args, **kwargs: [])+ monkeypatch.setattr(video_minimax_h3, "release_h3_native_servers", lambda reason: 0)+ monkeypatch.setattr(+ api_monitor, "record_lifecycle", lambda **event: env.lifecycle.append(event)+ )++ def fits(*blockers):+ # Short while any of ``blockers`` is still resident, so a re-check after an eviction passes.+ def probe(owner, need_mib):+ env.probes.append(owner)+ short = any(name in arb._view() for name in blockers)+ return not short, {+ "need_mib": {0: 6144},+ "free_mib": {0: 2048},+ "short_mib": {0: 5120} if short else {},+ "reason": "short" if short else "fits",+ }++ monkeypatch.setattr(res, "can_load_alongside", probe)++ def no_probe(owner, need_mib):+ raise AssertionError("can_load_alongside must not run here")++ env.fits = fits+ env.no_probe = lambda: monkeypatch.setattr(res, "can_load_alongside", no_probe)+ active_generations.reset_for_tests()+ yield env+ active_generations.reset_for_tests()+++CHAT_SLOT, IMAGE_SLOT = (+ {"kind": "chat", "model": "org/chat"},+ {"kind": "image", "model": "org/image"},+)+++ "policy, need, short, evicts",+ [+ ("off", {0: 4096}, False, True), # today's swap+ ("fit", {0: 4096}, False, False), # fits: both stay+ ("fit", {0: 4096}, True, True), # short: chat freed, cheapest first+ ("fit", None, False, True), # unpriced: evict first, as Off does+ ("always", {0: 4096}, False, False), # fits: joins+ ],+)+def test_image_load_beside_a_resident_chat_model_policy_matrix(cores, policy, need, short, evicts):+ cores.fits(*([arb.CHAT] if short else []))+ arb.acquire_for(arb.CHAT, gpu_ids = [0])+ evicted: list = []+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, need_mib = need, evicted = evicted)+ assert cores.calls == (["evict-chat"] if evicts else [])+ assert set(arb.residents()) == ({arb.DIFFUSION} if evicts else {arb.CHAT, arb.DIFFUSION})+ assert evicted == ([CHAT_SLOT] if evicts else [])+ # The fit path records each eviction in the API monitor; Off keeps today's silent swap.+ assert [event["reason"] for event in cores.lifecycle] == (+ ["evicted"] if evicts and policy != "off" else []+ )+ assert arb._media_needs.get(arb.DIFFUSION) == (need if policy != "off" else None)++[email protected]("managed", [False, True])+def test_always_refuses_an_image_load_that_does_not_fit_without_evicting(cores, managed):+ from utils.account_context import AccountContext, run_as++ cores.fits(arb.CHAT)+ arb.acquire_for(arb.CHAT, gpu_ids = [0])++ def load():+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "always", need_mib = {0: 6144})++ with pytest.raises(arb.MultiModelAlwaysRefusal) as refused:+ run_as(AccountContext("b" * 32, "bob"), load) if managed else load()+ message = str(refused.value)+ if managed:+ assert "is set to keep loaded" in message and "GB" not in message+ else:+ assert message.startswith("This model needs about 6.0 GB of GPU memory, and 2.0 GB is free")+ assert "is set to Always" in message+ assert cores.calls == [] and cores.lifecycle == []+ assert set(arb.residents()) == {arb.CHAT}++[email protected]("policy", ["off", "fit", "always"])+def test_image_load_on_another_gpu_never_evicts_chat_unless_off(cores, policy):+ cores.no_probe()+ arb.acquire_for(arb.CHAT, gpu_ids = [0])+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, need_mib = {1: 4096})+ if policy == "off":+ assert cores.calls == ["evict-chat"]+ assert arb.residents() == {arb.DIFFUSION: frozenset({1})}+ else:+ assert cores.calls == []+ assert arb.residents() == {arb.CHAT: frozenset({0}), arb.DIFFUSION: frozenset({1})}++[email protected]("policy", ["fit", "always"])+def test_chat_load_on_another_gpu_keeps_an_unpriced_image_model(cores, policy):+ # Unpriced media on GPU 1 can't be counted, but a chat load pinned to GPU 0 never touches it.+ arb.acquire_for(arb.DIFFUSION, lambda: None, gpu_ids = [1])+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = policy, evict_first = True)+ assert cores.calls == []+ assert arb.residents() == {arb.DIFFUSION: frozenset({1}), arb.CHAT: frozenset({0})}++[email protected]("policy", ["fit", "always"])+def test_unpriced_image_load_on_another_gpu_keeps_chat(cores, policy):+ # sd.cpp or H3 can't be priced, but a load pinned to GPU 1 still leaves chat on GPU 0 alone.+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = policy)+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, gpu_ids = [1])+ assert cores.calls == []+ assert arb.residents() == {arb.CHAT: frozenset({0}), arb.DIFFUSION: frozenset({1})}+++def test_media_fit_kwargs_keep_the_gpu_without_a_price(monkeypatch):+ import core.inference.gpu_residency as res++ priced = res.MediaNeed({1: 4096}, weights_mib = 3000, headroom_mib = 1096, kind = "image")+ need = {"value": None}+ monkeypatch.setattr(res, "media_load_need_mib", lambda *a: need["value"])+ assert res.media_fit_kwargs(arb.DIFFUSION, None, 1, "off") == {}+ assert res.media_fit_kwargs(arb.DIFFUSION, None, 1, "fit") == {"policy": "fit", "gpu_ids": [1]}+ assert res.media_fit_kwargs(arb.VIDEO, None, None, "always") == {"policy": "always"}+ need["value"] = priced+ assert res.media_fit_kwargs(arb.DIFFUSION, None, 1, "fit") == {+ "policy": "fit",+ "gpu_ids": [1],+ "need_mib": priced,+ }+++def test_chat_gpus_add_up_across_loads_and_survive_a_reassert(cores):+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = "fit")+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [1], policy = "fit", alongside = True)+ assert arb.residents() == {arb.CHAT: frozenset({0, 1})}+ arb.acquire_for(arb.CHAT, policy = "fit") # a re-assert places nothing new+ assert arb.residents() == {arb.CHAT: frozenset({0, 1})}+ arb.acquire_for(+ arb.CHAT, lambda: None, policy = "fit"+ ) # a load on unknown GPUs counts as every GPU+ assert arb.residents() == {arb.CHAT: None}+++def test_primary_chat_replacement_drops_the_gpus_it_left(cores, monkeypatch):+ from core.inference import model_slots++ # Replacing the primary with nothing kept alongside leaves no chat model on the old GPU.+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = "fit")+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [1], policy = "fit")+ assert arb.residents() == {arb.CHAT: frozenset({1})}+ arb.acquire_for(arb.CHAT, lambda: None, policy = "fit")+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [1], policy = "fit")+ assert arb.residents() == {arb.CHAT: frozenset({1})}+ # A model kept alongside still holds GPU 1, so a primary moved to GPU 0 adds up.+ monkeypatch.setattr(model_slots, "holds_vram", lambda: True)+ arb.acquire_for(arb.CHAT, lambda: None, gpu_ids = [0], policy = "fit")+ assert arb.residents() == {arb.CHAT: frozenset({0, 1})}+++ "policy, priced, kwargs, short, outcome",+ [+ ("fit", True, {}, False, "join"), # the launch prices the idle image model+ ("always", True, {}, False, "join"),+ ("fit", False, {}, False, "evict"), # unpriced media can't be counted+ ("always", False, {}, False, "refuse"),+ ("fit", True, {"evict_first": True}, False, "evict"), # no estimate before launch+ (+ "fit",+ True,+ {"need_mib": {0: 6144}},+ False,+ "join",+ ), # audio.cpp, priced from native_required_gb+ ("fit", True, {"need_mib": {0: 6144}}, True, "evict"),+ ],+)+def test_chat_load_beside_an_idle_image_model(cores, policy, priced, kwargs, short, outcome):+ cores.fits(*([arb.DIFFUSION] if short else []))+ arb.acquire_for(+ arb.DIFFUSION,+ lambda: None,+ policy = "fit" if priced else "off",+ need_mib = {0: 4096} if priced else None,+ gpu_ids = [0],+ )+ assert (arb.DIFFUSION in arb._media_needs) is priced+ evicted: list = []++ def load():+ arb.acquire_for(arb.CHAT, gpu_ids = [0], policy = policy, evicted = evicted, **kwargs)++ if outcome == "refuse":+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always"):+ load()+ else:+ load()+ assert cores.calls == (["evict-diffusion"] if outcome == "evict" else [])+ assert evicted == ([IMAGE_SLOT] if outcome == "evict" else [])+ expected = {"join": {arb.CHAT, arb.DIFFUSION}, "evict": {arb.CHAT}, "refuse": {arb.DIFFUSION}}[+ outcome+ ]+ assert set(arb.residents()) == expected+++ "policy, need, refused",+ [+ ("fit", {1: 4096}, False), # Bob's chat is on GPU 0: nothing of his is evicted+ ("fit", {0: 4096}, True), # short on Bob's GPU: his stream blocks the eviction+ ("off", {1: 4096}, True), # Off keeps the global guard+ ],+)+def test_another_accounts_stream_only_blocks_evicting_its_own_resident(+ cores, policy, need, refused+):+ import threading++ from state.active_generations import ActiveGeneration+ from utils.account_context import AccountContext, run_as++ alice, bob = AccountContext("a" * 32, "alice"), AccountContext("b" * 32, "bob")+ cores.fits(arb.CHAT)+ run_as(bob, arb.acquire_for, arb.CHAT, lambda: None, gpu_ids = [0])+ with ActiveGeneration(threading.Event(), run_id = "bob-chat", account_id = bob.account_id):++ def load():+ run_as(+ alice, arb.acquire_for, arb.DIFFUSION, lambda: None, policy = policy, need_mib = need+ )++ if refused:+ with pytest.raises(arb.GpuBusyForAnotherAccountError):+ load()+ else:+ load()+ assert cores.calls == []+ assert arb.holds(arb.CHAT) and arb.holds(arb.DIFFUSION) is not refused+ assert arb.owner_account(arb.CHAT) == bob.account_id+++def test_free_for_collects_what_it_evicts_and_spares_other_gpus(cores):+ import core.inference.gpu_residency as res++ cores.fits(arb.CHAT)+ arb.acquire_for(arb.CHAT, gpu_ids = [0])+ arb.acquire_for(arb.VIDEO, lambda: None, policy = "fit", need_mib = {1: 4096})+ evicted: list = []+ freed = res.free_for(arb.DIFFUSION, [0], "test", evicted, need_mib = {0: 6144})+ assert freed == ["org/chat"] and evicted == [CHAT_SLOT]+ assert cores.calls == ["evict-chat"]+ assert set(arb.residents()) == {arb.VIDEO}+ assert cores.lifecycle == [{"event": "unload", "model": "org/chat", "reason": "evicted"}]++[email protected]("video_resident", [True, False])+def test_a_fit_join_keeps_the_video_server_while_video_is_resident(cores, video_resident):+ cores.fits()+ arb.acquire_for(arb.CHAT, gpu_ids = [0])+ if video_resident:+ arb.acquire_for(arb.VIDEO, lambda: None, policy = "fit", need_mib = {1: 4096})+ cores.h3.clear()+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})+ assert cores.calls == []+ assert cores.h3 == ([] if video_resident else [f"GPU acquired for {arb.DIFFUSION}"])++[email protected]("policy", ["always", "fit"])+def test_image_load_behind_an_in_flight_chat_load(cores, monkeypatch, policy):+ import core.inference.gpu_residency as res+ from fastapi import HTTPException++ cores.fits(arb.CHAT)+ monkeypatch.setattr(res, "chat_load_in_flight", lambda: True)+ arb.acquire_for(arb.CHAT, gpu_ids = [0])++ def load():+ arb.acquire_for_request(arb.DIFFUSION, lambda: None, policy = policy, need_mib = {0: 4096})++ if policy == "always":+ # Its size isn't known yet, so Always answers the retryable K3 409 instead of the Always refusal.+ with pytest.raises(HTTPException) as refused:+ load()+ assert refused.value.status_code == 409+ detail = refused.value.detail+ assert detail["error"] == "gpu_busy" and detail["retry_after"] == 15+ assert detail["message"].startswith("Unsloth Studio is loading a chat model")+ assert refused.value.headers == {"Retry-After": "15"}+ assert cores.calls == [] and set(arb.residents()) == {arb.CHAT}+ else:+ load() # today's handoff+ assert cores.calls == ["evict-chat"] and set(arb.residents()) == {arb.DIFFUSION}++[email protected]("owner", [arb.DIFFUSION, arb.VIDEO])+def test_a_lone_media_load_under_fit_never_probes(cores, owner):+ cores.no_probe()+ for _ in range(2): # a reload of the same owner has no other resident either+ arb.acquire_for(owner, lambda: None, policy = "fit", need_mib = {0: 4096})+ assert cores.calls == []+ assert arb.residents() == {owner: frozenset({0})}+ assert arb._media_needs[owner] == {0: 4096}++def test_release_clears_owner(calls):arb.acquire_for(arb.DIFFUSION)arb.release(arb.DIFFUSION)diff --git a/studio/backend/tests/test_gpu_arbitration_sim.py b/studio/backend/tests/test_gpu_arbitration_sim.pyindex 13ea2e482ae..7bd7ac96354 100644--- a/studio/backend/tests/test_gpu_arbitration_sim.py+++ b/studio/backend/tests/test_gpu_arbitration_sim.py@@ -10,6 +10,7 @@ import itertoolsimport sqlite3import threadingimport time+from pathlib import Pathfrom types import SimpleNamespaceimport pytest@@ -716,6 +717,411 @@ def test_cpu_media_drain_does_not_refuse_unrelated_gpu_work(sim, media_switch, msim.finish("a")+# Keep multiple models loaded (When they fit / Always) against a fake VRAM ledger: the arbiter, gpu_residency and the+# chat pricing helpers are real; only the GPU probe, the evictors and the slot reports are fakes.+GIB = 1024+++class VramSim:+ def __init__(self, monkeypatch, totals_gb):+ import utils.hardware as hardware+ from core.inference import gpu_residency, model_slots, video_minimax_h3+ from utils.hardware import hardware as hardware_impl++ self.residency = gpu_residency+ self.totals = {gpu: gb * GIB for gpu, gb in enumerate(totals_gb)}+ self.footprint: dict[str, dict[int, int]] = {}+ self.evictions: list[str] = []+ self.probes = 0+ self.index_kind = "physical"+ self.parent = sorted(self.totals)+ self.policy = "off"+ for owner in BACKENDS:+ monkeypatch.setitem(arb._EVICTORS, owner, lambda o = owner: self._evict(o))+ monkeypatch.setitem(+ arb._SLOT_REPORTERS,+ owner,+ lambda o = owner: [+ {"kind": arb.KIND_LABELS[o], "model": f"{arb.KIND_LABELS[o]}-model"}+ ],+ )+ monkeypatch.setattr(arb, "_residents", {})+ monkeypatch.setattr(arb, "_accounts", {})+ monkeypatch.setattr(arb, "_media_needs", {})+ monkeypatch.setattr(arb, "_release_idle_video_servers", lambda reason: None)+ monkeypatch.setattr(video_minimax_h3, "release_h3_native_servers", lambda reason: 0)+ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)+ monkeypatch.setattr(hardware, "get_visible_gpu_utilization", self._utilization)+ monkeypatch.setattr(hardware_impl, "get_parent_visible_gpu_ids", lambda: list(self.parent))+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: self.policy)+ monkeypatch.setattr(gpu_residency, "_media_busy", lambda owner: (False, False))+ monkeypatch.setattr(gpu_residency, "_loaded_or_loading", lambda owner: True)+ monkeypatch.setattr(gpu_residency, "chat_load_in_flight", lambda: False)+ # The diffusion planner's own verdict is covered in test_gpu_residency_coexistence.py.+ monkeypatch.setattr(gpu_residency, "_keeps_transformer_resident", lambda *a: True)+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *a, **k: [])++ def _utilization(self):+ self.probes += 1+ return {+ "index_kind": self.index_kind,+ "devices": [+ {"index": gpu, "vram_total_gb": total / GIB, "vram_used_gb": self.used(gpu) / GIB}+ for gpu, total in self.totals.items()+ ],+ }++ def used(self, gpu):+ return sum(held.get(gpu, 0) for held in self.footprint.values())++ def _evict(self, owner):+ self.evictions.append(owner)+ self.footprint.pop(owner, None)++ def load(self, owner, held, **kwargs):+ evicted: list = []++ def register():+ self.footprint[owner] = dict(held)++ arb.acquire_for(owner, register, evicted = evicted, **kwargs)+ return evicted++ def load_image(+ self,+ need_gb,+ gpu = 0,+ headroom_gb = 4,+ **kwargs,+ ):+ """An image load priced at ``need_gb``; idle, only its weights are allocated (the headroom is not)."""+ need, headroom = need_gb * GIB, headroom_gb * GIB+ priced = self.residency.MediaNeed(+ {gpu: need}, weights_mib = need - headroom, headroom_mib = headroom, kind = "image"+ )+ return self.load(arb.DIFFUSION, {gpu: need - headroom}, need_mib = priced, **kwargs)++ def load_chat(+ self,+ gb = 10,+ gpu_ids = (0,),+ **kwargs,+ ):+ held = {gpu: gb * GIB // max(len(gpu_ids or (0,)), 1) for gpu in (gpu_ids or (0,))}+ return self.load(+ arb.CHAT, held, gpu_ids = list(gpu_ids) if gpu_ids is not None else None, **kwargs+ )+++def vram(monkeypatch):+ return lambda *totals_gb: VramSim(monkeypatch, totals_gb)+++CHAT_SLOT = {"kind": "chat", "model": "chat-model"}+IMAGE_SLOT = {"kind": "image", "model": "image-model"}++[email protected]("policy", ["fit", "always"])+def test_image_joins_chat_on_one_gpu_when_it_fits(vram, policy):+ v = vram(80)+ v.load_chat(10)+ assert v.load_image(20, policy = policy) == []+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}+ assert v.residency.media_held_mib() == {0: 20 * GIB}+ assert v.used(0) == 26 * GIB # 10 GB chat + 16 GB idle image weights+++def test_image_evicts_chat_on_one_gpu_when_short_under_fit(vram):+ v = vram(80)+ v.load_chat(10)+ assert v.load_image(75, policy = "fit") == [CHAT_SLOT]+ assert v.evictions == [arb.CHAT] and set(arb.residents()) == {arb.DIFFUSION}+ assert arb.CHAT not in v.footprint+++def test_image_short_on_one_gpu_under_always_refuses_and_keeps_chat(vram):+ v = vram(80)+ v.load_chat(10)+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always"):+ v.load_image(75, policy = "always")+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT}+ assert v.footprint == {arb.CHAT: {0: 10 * GIB}} and v.residency.media_held_mib() == {}++[email protected]("policy", ["fit", "always"])+def test_image_on_another_gpu_never_evicts_or_probes(vram, policy):+ v = vram(80, 80)+ v.load_chat(10, gpu_ids = (0,))+ assert v.load_image(75, gpu = 1, policy = policy) == []+ assert v.evictions == [] and v.probes == 0+ assert arb.residents() == {arb.CHAT: frozenset({0}), arb.DIFFUSION: frozenset({1})}+++def test_a_refused_image_registration_keeps_the_loaded_image_on_its_gpu(vram):+ v = vram(80, 80)+ v.load_image(20, gpu = 0, policy = "always")++ def refuse():+ raise arb.MultiModelAlwaysRefusal("another image model is loaded")++ with pytest.raises(arb.MultiModelAlwaysRefusal):+ arb.acquire_for(arb.DIFFUSION, refuse, policy = "always", gpu_ids = [1])+ assert arb.residents()[arb.DIFFUSION] == frozenset({0})+ # A chat load on GPU 0 still sees the image there.+ with pytest.raises(arb.MultiModelAlwaysRefusal):+ v.load_chat(70, gpu_ids = (0,), policy = "always", evict_first = True)+ assert v.evictions == [] and set(arb.residents()) == {arb.DIFFUSION}+++def test_image_on_another_gpu_under_off_still_swaps(vram):+ v = vram(80, 80)+ v.load_chat(10, gpu_ids = (0,))+ assert v.load_image(20, gpu = 1) == [CHAT_SLOT]+ assert v.evictions == [arb.CHAT] and set(arb.residents()) == {arb.DIFFUSION}+++def _chat_kwargs(+ config,+ placement,+ *,+ primary_can_retry = False,+):+ from routes import inference as inf+ token = inf._primary_can_retry_fit.set(primary_can_retry)+ try:+ return inf._chat_fit_kwargs(config, placement)+ finally:+ inf._primary_can_retry_fit.reset(token)+++def _placement(+ gpus = (0,),+ *,+ vulkan = False,+ native_gb = None,+):+ from routes.inference import _LoadPlacement+ resolved = list(gpus) if gpus is not None else None+ return _LoadPlacement(resolved, resolved, vulkan, None, native_required_gb = native_gb)+++def test_gguf_chat_joins_an_idle_image_and_its_launch_prices_the_image(vram):+ from core.inference import llama_cpp++ v = vram(80, 80)+ v.policy = "fit"+ v.load_image(20, policy = "fit")+ placement = _placement((0,))+ kwargs = _chat_kwargs(SimpleNamespace(is_gguf = True), placement, primary_can_retry = True)+ assert kwargs == {"policy": "fit", "gpu_ids": [0]}+ assert v.load_chat(10, **kwargs) == []+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}++ backend = llama_cpp.LlamaCppBackend(manages_processes = False)+ llama_cpp.register_serving_backend(backend)+ try:+ held = backend._other_planned_vram_mib()+ finally:+ llama_cpp.unregister_serving_backend(backend)+ assert held == {+ 0: 20 * GIB+ } # weights plus the headroom the idle image will allocate when it generates+ # A stale free reading on GPU 0 is capped at total less what the image holds; GPU 1 is left alone.+ probed = [(0, 80 * GIB, 80 * GIB), (1, 70 * GIB, 80 * GIB)]+ assert llama_cpp._net_of_held_vram(probed, held) == [+ (0, 60 * GIB, 80 * GIB),+ (1, 70 * GIB, 80 * GIB),+ ]+++def test_gguf_chat_sizing_keeps_an_idle_image_headroom_when_another_process_uses_the_gpu(vram):+ from core.inference import llama_cpp++ v = vram(80)+ v.policy = "fit"+ v.load_image(24, headroom_gb = 8, policy = "fit") # 16 GB weights allocated, 8 GB headroom not yet+ v.footprint["other process"] = {0: 20 * GIB}+ probed = [(0, v.totals[0] - v.used(0), v.totals[0])]+ assert probed == [(0, 44 * GIB, 80 * GIB)]+ backend = llama_cpp.LlamaCppBackend(manages_processes = False)+ llama_cpp.register_serving_backend(backend)+ try:+ held = backend._other_planned_vram_mib()+ unallocated = backend._media_unallocated_vram_mib()+ finally:+ llama_cpp.unregister_serving_backend(backend)+ sized = llama_cpp._net_of_other_models_vram(probed, held, unallocated)+ # 44 GB is free, but 8 GB of it is the image's generation budget.+ assert sized == [(0, 36 * GIB, 80 * GIB)]+ launch = " ".join(Path(llama_cpp.__file__).read_text(encoding = "utf-8").split())+ assert (+ "_gpu_mem = _net_of_other_models_vram( _gpu_mem, _held_by_others, self._media_unallocated_vram_mib() )"+ in launch+ )++[email protected]("loading", [False, True])+def test_gguf_chat_sizing_charges_the_image_once_on_an_otherwise_empty_gpu(+ vram, monkeypatch, loading+):+ """Nothing else on the card: the total cap alone charges the image's full price, never twice."""+ from core.inference import llama_cpp++ v = vram(80)+ v.policy = "fit"+ v.load_image(24, headroom_gb = 8, policy = "fit")+ if loading:+ monkeypatch.setattr(v.residency, "_media_busy", lambda owner: (True, False))+ v.footprint.pop(arb.DIFFUSION) # nothing allocated yet+ probed = [(0, v.totals[0] - v.used(0), v.totals[0])]+ backend = llama_cpp.LlamaCppBackend(manages_processes = False)+ llama_cpp.register_serving_backend(backend)+ try:+ held = backend._other_planned_vram_mib()+ unallocated = backend._media_unallocated_vram_mib()+ finally:+ llama_cpp.unregister_serving_backend(backend)+ sized = llama_cpp._net_of_other_models_vram(probed, held, unallocated)+ assert sized == [(0, 56 * GIB, 80 * GIB)]+++def test_a_loading_image_is_held_in_full_by_chat_sizing(vram, monkeypatch):+ from core.inference import llama_cpp++ v = vram(80)+ v.load_image(24, headroom_gb = 8, policy = "fit")+ monkeypatch.setattr(v.residency, "_media_busy", lambda owner: (True, False))+ assert v.residency.media_unallocated_mib() == {0: 24 * GIB}+ assert llama_cpp._net_of_unallocated_vram([(0, 60 * GIB, 0)], {0: 24 * GIB}) == [+ (0, 60 * GIB, 0)+ ] # an integrated GPU's reading already counts shared memory++[email protected]("alongside", [False, True])+def test_chat_reasserted_behind_an_image_keeps_the_account_that_loaded_it(vram, alongside):+ """Another account reusing the chat model (or adding a slot) while an image sits beside it does not take it."""+ v = vram(80)+ v.policy = "fit"+ run_as(ALICE, v.load_chat, 10)+ assert run_as(ALICE, v.load_image, 20, policy = "fit") == []+ assert arb.owner_account(arb.CHAT) == ALICE.account_id+ if alongside:+ run_as(BOB, v.load_chat, 5, policy = "fit", alongside = True)+ else:+ run_as(BOB, arb.acquire_for_request, arb.CHAT, policy = "fit")+ assert set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}+ assert arb.owner_account(arb.CHAT) == ALICE.account_id+ assert arb.owner_account(arb.DIFFUSION) == ALICE.account_id+++def test_gguf_chat_with_no_fit_retry_evicts_first_under_fit(vram):+ v = vram(80)+ v.policy = "fit"+ assert _chat_kwargs(SimpleNamespace(is_gguf = True), _placement()) == {+ "policy": "fit",+ "gpu_ids": [0],+ "evict_first": True,+ }+++ ("policy", "expected", "evicted"),+ [+ ("fit", {"policy": "fit", "gpu_ids": [0], "evict_first": True}, [IMAGE_SLOT]),+ ("always", {"policy": "always", "gpu_ids": [0]}, []),+ ],+)+def test_non_gguf_chat_beside_an_image(vram, policy, expected, evicted):+ v = vram(80)+ v.policy = policy+ v.load_image(20, policy = policy)+ placement = _placement((0,))+ kwargs = _chat_kwargs(SimpleNamespace(is_gguf = False), placement, primary_can_retry = True)+ assert kwargs == expected+ assert v.load_chat(10, **kwargs) == evicted+ assert v.evictions == [arb.DIFFUSION for _ in evicted]+ assert arb.CHAT in arb.residents() and (arb.DIFFUSION in arb.residents()) == (not evicted)+++def _audio_cpp_chat(v, native_gb, **placement_kw):+ placement = _placement(native_gb = native_gb, **placement_kw)+ kwargs = _chat_kwargs(SimpleNamespace(is_gguf = False), placement)+ gpus = None if placement.gpu_ids_are_vulkan_ordinals else placement.resolved_gpu_ids+ return kwargs, v.load(arb.CHAT, {0: int(native_gb * GIB)}, **{"gpu_ids": gpus, **kwargs})++[email protected]("policy", ["fit", "always"])+def test_audio_cpp_chat_joins_a_resident_image_when_native_estimate_fits(vram, policy):+ v = vram(80)+ v.policy = policy+ v.load_image(20, policy = policy)+ kwargs, evicted = _audio_cpp_chat(v, 6.0)+ assert kwargs == {"policy": policy, "gpu_ids": [0], "need_mib": {0: 6144}}+ assert evicted == [] and v.evictions == [] and set(arb.residents()) == {arb.CHAT, arb.DIFFUSION}+++def test_audio_cpp_chat_short_beside_an_image_evicts_it_under_fit(vram):+ v = vram(80)+ v.policy = "fit"+ v.load_image(20, policy = "fit")+ kwargs, evicted = _audio_cpp_chat(v, 70.0)+ assert kwargs["need_mib"] == {0: 70 * GIB}+ assert evicted == [IMAGE_SLOT] and v.evictions == [arb.DIFFUSION]+ assert set(arb.residents()) == {arb.CHAT} and v.residency.media_held_mib() == {}+++def test_audio_cpp_chat_short_beside_an_image_refuses_under_always(vram):+ v = vram(80)+ v.policy = "always"+ v.load_image(20, policy = "always")+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always"):+ _audio_cpp_chat(v, 70.0)+ assert v.evictions == [] and set(arb.residents()) == {arb.DIFFUSION}++[email protected]("placement_kw", [{"vulkan": True}, {"gpus": None}])+def test_audio_cpp_chat_without_physical_gpus_evicts_first_under_fit(vram, placement_kw):+ v = vram(80)+ v.policy = "fit"+ v.load_image(20, policy = "fit")+ kwargs, evicted = _audio_cpp_chat(v, 6.0, **placement_kw)+ assert kwargs == {"policy": "fit", "evict_first": True}+ assert evicted == [IMAGE_SLOT] and v.evictions == [arb.DIFFUSION]+++def _masked(v):+ # A UUID/MIG mask: rows carry relative indices and there are no numeric parent ids.+ v.index_kind = "relative"+ v.parent = []+ v.load_chat(10, gpu_ids = None)++[email protected]("priced", [False, True])+def test_masked_host_image_beside_chat_evicts_first_under_fit(vram, priced):+ v = vram(80)+ _masked(v)+ assert v.residency.media_load_need_mib(arb.DIFFUSION, SimpleNamespace(), None) is None+ if priced:+ assert v.load_image(20, policy = "fit") == [CHAT_SLOT]+ else:+ assert v.load(arb.DIFFUSION, {0: 16 * GIB}, policy = "fit") == [CHAT_SLOT]+ assert v.evictions == [arb.CHAT] and set(arb.residents()) == {arb.DIFFUSION}++[email protected]("priced", [False, True])+def test_masked_host_image_beside_chat_refuses_under_always(vram, priced):+ v = vram(80)+ _masked(v)+ with pytest.raises(arb.MultiModelAlwaysRefusal, match = "is set to Always") as refused:+ if priced:+ v.load_image(20, policy = "always")+ else:+ v.load(arb.DIFFUSION, {0: 16 * GIB}, policy = "always")+ assert refused.value.kind == "chat" and refused.value.model == "chat-model"+ assert v.evictions == [] and set(arb.residents()) == {arb.CHAT}++@pytest.mark.parametrize("owner", [arb.DIFFUSION, arb.VIDEO])@pytest.mark.parametrize("resident_owner", [arb.CHAT, arb.DIFFUSION, arb.VIDEO])def test_actual_media_load_routes_refuse_before_engine_activation_or_load(@@ -816,3 +1222,96 @@ def test_chat_resident_fast_path_adds_no_account_policy_work(route, chat_switch,asyncio.run(arun_as(OWNER, route._maybe_auto_switch_model("org/one-GGUF", request, OWNER.username)))++[email protected]("policy", ["fit", "always"])+def test_automatic_native_placement_credits_the_chat_model_an_image_sits_beside(+ vram, monkeypatch, policy+):+ """Chat (10 GB) then an image (2 GB) on 16 GB: a primary native load replaces the chat model, so its 10 GB+ count, not the image's 2 GB (the image stays loaded beside it)."""+ from routes import inference as inf++ v = vram(16)+ v.policy = policy+ v.load_chat(10)+ assert v.load_image(4, headroom_gb = 2, policy = policy) == []+ assert arb.owner_snapshot()[0] == arb.DIFFUSION+ monkeypatch.setattr(+ inf, "get_inference_backend", lambda: SimpleNamespace(active_model_name = None)+ )+ llama = SimpleNamespace(is_loaded = True, reclaimable_gpu_memory_gb = lambda: {0: 10.0})+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: llama)+ availability = inf._native_audio_post_handoff_free_gb()+ assert availability is not None+ assert availability.credited == arb.CHAT+ assert availability.effective_free_gb == {0: 14.0}+ assert availability.owner_snapshot == arb.owner_snapshot()+ placement = _placement((0,), native_gb = 8.0)+ # The CHAT acquire reuses this measurement and keeps the image beside the new model.+ credit, wait = inf._outgoing_chat_credit_gb(placement, policy, True, availability)+ assert credit == {0: 10.0} and wait is None++[email protected]("policy", ["fit", "always"])+def test_native_load_replacing_the_primary_is_credited_with_its_memory(vram, monkeypatch, policy):+ """A priced native load that replaces a 40 GB primary beside an idle image fits once the primary goes."""+ from routes import inference as inf++ v = vram(80)+ v.policy = policy+ v.load_chat(40)+ assert v.load_image(20, policy = policy) == []+ outgoing = inf._NativeAudioAvailability({0: 64.0}, {0: 40.0}, arb.owner_snapshot())+ monkeypatch.setattr(+ inf,+ "_native_audio_post_handoff_free_gb",+ lambda outgoing_owner = None: outgoing if outgoing_owner == arb.CHAT else None,+ )+ placement = _placement((0,), native_gb = 30.0)+ credit, wait = inf._outgoing_chat_credit_gb(placement, policy, True, None)+ assert credit == {0: 40.0} and wait == {0: 30.0}+ kwargs = inf._chat_fit_kwargs(SimpleNamespace(is_gguf = False), placement, policy, credit)+ assert v.load(arb.CHAT, {0: 30 * GIB}, replacing = True, **kwargs) == []+ assert v.evictions == [] and arb.DIFFUSION in arb.residents()+ # Loaded alongside, nothing is torn down, so nothing is credited.+ assert inf._outgoing_chat_credit_gb(placement, policy, False, None) == (None, None)+++def test_extra_chat_load_that_evicts_the_image_keeps_the_chat_loader(vram):+ """Alice's chat model sits behind her image; Bob's extra (unpriced) chat model evicts the image under When they+ fit, but the primary chat model is still Alice's."""+ v = vram(24)+ run_as(ALICE, v.load_chat, 10, policy = "fit")+ assert run_as(ALICE, v.load_image, 8, policy = "fit") == []+ assert arb.owner_snapshot()[0] == arb.DIFFUSION+ run_as(+ BOB,+ v.load,+ arb.CHAT,+ {0: 20 * GIB},+ alongside = True,+ policy = "fit",+ gpu_ids = [0],+ evict_first = True,+ )+ assert v.evictions == [arb.DIFFUSION]+ assert arb.owner_account(arb.CHAT) == ALICE.account_id++[email protected]("policy", ["fit", "always"])+def test_smaller_native_load_replacing_the_primary_fits_beside_a_tightly_packed_image(+ vram, monkeypatch, policy+):+ """24 GB: a 14 GB chat model and an image admitted with no slack (6.5 GB weights + 2.5 GB headroom). A 2 GB+ native model replacing the chat model frees 12 GB, so the image stays loaded."""+ from routes import inference as inf++ v = vram(24)+ v.policy = policy+ v.load_chat(14)+ assert v.load_image(9, headroom_gb = 2.5, policy = policy) == []+ placement = _placement((0,), native_gb = 2.0)+ kwargs = inf._chat_fit_kwargs(SimpleNamespace(is_gguf = False), placement, policy, {0: 14.0})+ assert v.load(arb.CHAT, {0: 2 * GIB}, replacing = True, **kwargs) == []+ assert v.evictions == [] and arb.DIFFUSION in arb.residents()diff --git a/studio/backend/tests/test_gpu_residency_coexistence.py b/studio/backend/tests/test_gpu_residency_coexistence.pynew file mode 100644index 00000000000..dde3fef1e42--- /dev/null+++ b/studio/backend/tests/test_gpu_residency_coexistence.py@@ -0,0 +1,688 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for core/inference/gpu_residency.py: pricing image and video loads that stay on a GPU beside chat."""++import tempfile+import unittest+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import MagicMock, patch++import utils.hardware.hardware as _hw_module+from core.inference import gpu_arbiter as arb+from core.inference import gpu_residency as gr+from core.inference.diffusion import DiffusionBackend+from core.inference.diffusion_memory import estimate_image_runtime_mib, estimate_video_runtime_mib+from utils.hardware import DeviceType++MIB = 1024 * 1024+++class _GpuCacheResetMixin:+ """Reset module-level GPU caches between tests to prevent state leaks."""++ def tearDown(self):+ _hw_module._physical_gpu_count = None+ _hw_module._visible_gpu_count = None+++def _parent_ids(ids):+ return patch("utils.hardware.hardware.get_parent_visible_gpu_ids", return_value = list(ids))+++def _image_request(+ model_path,+ gguf_filename = "z-image-turbo-Q4_K_M.gguf",+ **kw,+):+ fields = dict(+ model_kind = None, base_repo = None, family_override = None, text_encoder_file = None, vae_file = None+ )+ fields.update(kw)+ return SimpleNamespace(model_path = str(model_path), gguf_filename = gguf_filename, **fields)+++def _video_request(model_path, gguf_filename, **kw):+ fields = dict(model_kind = None, base_repo = None, family_override = None, display_repo_id = None)+ fields.update(kw)+ return SimpleNamespace(model_path = str(model_path), gguf_filename = gguf_filename, **fields)+++def _sparse(path: Path, mib: int) -> Path:+ path.parent.mkdir(parents = True, exist_ok = True)+ with open(path, "wb") as f:+ f.truncate(mib * MIB)+ return path+++class TestMediaLoadNeed(_GpuCacheResetMixin, unittest.TestCase):+ def setUp(self):+ self._tmp = tempfile.TemporaryDirectory()+ self.root = Path(self._tmp.name) / "Z-Image-Turbo-GGUF"+ _sparse(self.root / "z-image-turbo-Q4_K_M.gguf", 4000)+ self._patches = [+ # Native engines are host-dependent; the diffusers path is what these tests price.+ patch(+ "core.inference.diffusion_engine_router.predict_engine", return_value = "diffusers"+ ),+ patch.object(+ DiffusionBackend,+ "_companion_cache_bytes",+ staticmethod(lambda base, *a, **k: 6000 * MIB),+ ),+ ]+ for p in self._patches:+ p.start()++ def tearDown(self):+ for p in reversed(self._patches):+ p.stop()+ self._tmp.cleanup()+ super().tearDown()++ def _image(+ self,+ ordinal,+ parent = (2, 3),+ request = None,+ ):+ with _parent_ids(parent):+ return gr.media_load_need_mib(+ arb.DIFFUSION, request or _image_request(self.root), ordinal+ )++ def test_gguf_image_prices_weights_plus_headroom_on_the_physical_gpu(self):+ need = self._image(1)+ self.assertIsNotNone(need)+ weights = int(4000 * 1.05) + 6000+ self.assertEqual(need.weights_mib, weights)+ self.assertEqual(need.kind, "image")+ self.assertEqual(list(need), [3]) # ordinal 1 of parent [2, 3]+ self.assertEqual(need[3], weights + need.headroom_mib)+ self.assertGreaterEqual(need.headroom_mib, 1024)+ self.assertLessEqual(need.headroom_mib, estimate_image_runtime_mib(width = None, height = None))++ def test_uncached_checkpoint_is_unknown(self):+ self.assertIsNone(+ self._image(0, request = _image_request(self.root, "z-image-turbo-Q8_0.gguf"))+ )++ def test_uncached_companions_are_unknown(self):+ with patch.object(+ DiffusionBackend, "_companion_cache_bytes", staticmethod(lambda *a, **k: 0)+ ):+ self.assertIsNone(self._image(0))++ def test_unknown_family_is_unknown(self):+ with patch("core.inference.diffusion_families.detect_family_for_pick", return_value = None):+ self.assertIsNone(self._image(0))++ def test_native_engine_is_unknown(self):+ with patch("core.inference.diffusion_engine_router.predict_engine", return_value = "sd_cpp"):+ self.assertIsNone(self._image(0))++ def test_automatic_gpu_with_several_visible_is_unknown(self):+ self.assertIsNone(self._image(None, parent = (0, 1)))++ def test_automatic_gpu_with_one_visible_uses_it(self):+ need = self._image(None, parent = (5,))+ self.assertEqual(list(need), [5])++ def test_masked_gpus_are_unknown(self):+ self.assertIsNone(self._image(0, parent = ()))+ self.assertIsNone(self._image(4, parent = (0, 1)))++ def test_internal_error_is_unknown(self):+ with patch("core.inference.diffusion.resolve_model_kind", side_effect = RuntimeError("boom")):+ self.assertIsNone(self._image(0))+ with patch(+ "utils.hardware.hardware.get_parent_visible_gpu_ids", side_effect = OSError("smi")+ ):+ self.assertIsNone(gr.media_load_need_mib(arb.DIFFUSION, _image_request(self.root), 0))++ def test_unknown_kind_is_unknown(self):+ with _parent_ids((0,)):+ self.assertIsNone(gr.media_load_need_mib("chat", _image_request(self.root), 0))++ def _video_family(self, name = "wan-2.1"):+ return SimpleNamespace(+ name = name,+ base_repo = "Wan-AI/Wan2.1-T2V-14B-Diffusers",+ is_moe = False,+ resolution_presets = ((832, 480),),+ default_num_frames = 81,+ )++ def test_video_gguf_prices_checkpoint_plus_companion_table(self):+ root = Path(self._tmp.name) / "Wan-GGUF"+ _sparse(root / "wan-Q4_K_M.gguf", 3000)+ fam = self._video_family()+ with (+ patch("core.inference.video._detect_load_family", return_value = fam),+ patch(+ "core.inference.diffusion_component_sizes.video_components_gb",+ return_value = (28.0, 10.0, 0.5),+ ),+ _parent_ids((2, 3)),+ ):+ need = gr.media_load_need_mib(arb.VIDEO, _video_request(root, "wan-Q4_K_M.gguf"), 0)+ companions = int(10.5 * 1000.0**3 / MIB)+ self.assertEqual(need.weights_mib, int(3000 * 1.05) + companions)+ self.assertEqual(+ need.headroom_mib, estimate_video_runtime_mib(width = 832, height = 480, num_frames = 81)+ )+ self.assertEqual(need.kind, "video")+ self.assertEqual(dict(need), {2: need.weights_mib + need.headroom_mib})++ def test_video_h3_native_is_unknown(self):+ root = Path(self._tmp.name) / "H3-GGUF"+ _sparse(root / "h3-Q4_K_M.gguf", 10)+ with (+ patch(+ "core.inference.video._detect_load_family",+ return_value = self._video_family("minimax-h3"),+ ),+ patch(+ "core.inference.diffusion_component_sizes.video_components_gb",+ return_value = (1.0, 1.0, 1.0),+ ),+ _parent_ids((0,)),+ ):+ self.assertIsNone(+ gr.media_load_need_mib(arb.VIDEO, _video_request(root, "h3-Q4_K_M.gguf"), 0)+ )+++def _util(*devices, index_kind = "physical"):+ rows = [dict(index = i, vram_total_gb = t, vram_used_gb = u) for i, t, u in devices]+ return patch(+ "utils.hardware.get_visible_gpu_utilization",+ return_value = {"index_kind": index_kind, "devices": rows},+ )+++def _cuda(device = DeviceType.CUDA):+ return patch("utils.hardware.get_device", return_value = device)+++def _need(+ per_gpu,+ weights = 3000,+ headroom = 2000,+ kind = "video",+):+ return gr.MediaNeed(per_gpu, weights_mib = weights, headroom_mib = headroom, kind = kind)+++class _ArbiterResetMixin:+ """Empty arbiter state for each test, restored afterwards."""++ _FIELDS = ("_owner", "_residents", "_media_needs", "_accounts")++ def setUp(self):+ super().setUp()+ self._saved = {name: getattr(arb, name) for name in self._FIELDS}+ arb._owner, arb._residents, arb._media_needs, arb._accounts = None, {}, {}, {}+ # Residents set by hand have no engine behind them; TestFailedLoadPrice covers the check.+ self._engine_patch = patch.object(gr, "_loaded_or_loading", return_value = True)+ self._engine_patch.start()+ self.addCleanup(self._engine_patch.stop)++ def tearDown(self):+ for name, value in self._saved.items():+ setattr(arb, name, value)+ super().tearDown()++ def _resident(+ self,+ owner,+ gpus,+ need = None,+ ):+ arb._residents[owner] = frozenset(gpus)+ arb._owner = owner+ if need is not None:+ arb._media_needs[owner] = need+++FREE_80_20 = 80 * 1024 - 20 * 1024 # free MiB on an 80 GB card with 20 GB used+++class TestCanLoadAlongside(_ArbiterResetMixin, unittest.TestCase):+ def _check(+ self,+ need,+ owner = arb.VIDEO,+ device = DeviceType.CUDA,+ util = None,+ ):+ with _cuda(device), util or _util((0, 80.0, 20.0)):+ return gr.can_load_alongside(owner, need)++ def test_default_deny_reasons(self):+ need = _need({0: 1000})+ self.assertEqual(+ gr.can_load_alongside(arb.VIDEO, None), (False, {"reason": "estimate_unavailable"})+ )+ self.assertEqual(gr.can_load_alongside(arb.VIDEO, {})[1]["reason"], "estimate_unavailable")+ for device in (DeviceType.MLX, DeviceType.CPU, DeviceType.XPU):+ self.assertEqual(+ self._check(need, device = device), (False, {"reason": "non_accelerator"})+ )+ with (+ _cuda(),+ patch("utils.hardware.get_visible_gpu_utilization", side_effect = RuntimeError("smi")),+ ):+ self.assertEqual(+ gr.can_load_alongside(arb.VIDEO, need), (False, {"reason": "probe_error"})+ )+ for kind in ("relative", "unresolved"):+ self.assertEqual(+ self._check(need, util = _util((0, 80.0, 20.0), index_kind = kind))[1]["reason"],+ "unmapped_gpus",+ )+ self.assertEqual(self._check(_need({5: 1000}))[1], {"reason": "gpu_unknown", "gpu": 5})++ def test_unified_memory_is_denied(self):+ need = _need({0: 1000})+ for flag in ("unified_memory", "shared_memory"):+ util = {+ "index_kind": "physical",+ "devices": [{"index": 0, "vram_total_gb": 80.0, "vram_used_gb": 1.0, flag: True}],+ }+ with _cuda(), patch("utils.hardware.get_visible_gpu_utilization", return_value = util):+ self.assertEqual(+ gr.can_load_alongside(arb.VIDEO, need),+ (False, {"reason": "unified_memory", "gpu": 0}),+ )+ self.assertEqual(self._check(need, util = _util((0, 0, 0.0)))[1]["reason"], "unified_memory")++ def test_margin_boundary(self):+ fits, info = self._check(_need({0: FREE_80_20 - gr.FIT_MARGIN_MIB}))+ self.assertTrue(fits)+ self.assertEqual(info["reason"], "fits")+ fits, info = self._check(_need({0: FREE_80_20 - gr.FIT_MARGIN_MIB + 1}))+ self.assertFalse(fits)+ self.assertEqual((info["reason"], info["short_mib"]), ("short", {0: 1}))++ def test_one_short_gpu_fails_the_whole_load(self):+ util = _util((0, 80.0, 20.0), (1, 24.0, 20.0))+ fits, info = self._check(_need({0: 1000, 1: 4000}), util = util)+ self.assertFalse(fits)+ self.assertEqual(list(info["short_mib"]), [1])+ self.assertEqual(info["free_mib"], {0: FREE_80_20, 1: 4 * 1024})++ def test_other_resident_headroom_is_subtracted(self):+ self._resident(arb.VIDEO, {0}, _need({0: 5000}, weights = 3000, headroom = 2000))+ _, info = self._check(_need({0: 1000}, kind = "other"), owner = arb.DIFFUSION)+ self.assertEqual(info["free_mib"], {0: FREE_80_20 - 2000})++ def test_a_resident_still_loading_counts_in_full(self):+ # Its weights aren't in live free VRAM yet, so a second media load can't be admitted into that gap.+ self._resident(arb.VIDEO, {0}, _need({0: 5000}, weights = 3000, headroom = 2000))+ with patch.object(gr, "_media_busy", lambda owner: (owner == arb.VIDEO, False)):+ _, info = self._check(_need({0: 1000}, kind = "other"), owner = arb.DIFFUSION)+ self.assertEqual(info["free_mib"], {0: FREE_80_20 - 5000})++ def test_own_record_adds_back_its_weights(self):+ self._resident(arb.VIDEO, {0}, _need({0: 5000}, weights = 3000, headroom = 2000))+ _, info = self._check(_need({0: 1000}), owner = arb.VIDEO)+ self.assertEqual(info["free_mib"], {0: FREE_80_20 + 3000})++ def test_image_that_would_stream_is_denied(self):+ plan = MagicMock(return_value = "plan")+ with (+ _parent_ids((0,)),+ patch("core.inference.diffusion_memory.plan_diffusion_memory", plan),+ patch(+ "core.inference.diffusion_memory.plan_keeps_transformer_resident",+ return_value = False,+ ),+ ):+ fits, info = self._check(_need({0: 1000}, kind = "image"), owner = arb.DIFFUSION)+ self.assertFalse(fits)+ self.assertEqual(info["reason"], "would_stream")+ kw = plan.call_args.kwargs+ self.assertIsNone(kw["requested_mode"])+ self.assertEqual((kw["model_dense_mib"], kw["runtime_headroom_mib"]), (3000, 2000))+ self.assertEqual(+ (kw["device_memory"].device, kw["device_memory"].free_mib), ("cuda:0", FREE_80_20)+ )++ def test_image_kept_resident_fits_and_video_skips_the_planner(self):+ with (+ _parent_ids((0,)),+ patch(+ "core.inference.diffusion_memory.plan_keeps_transformer_resident", return_value = True+ ),+ ):+ self.assertTrue(self._check(_need({0: 1000}, kind = "image"), owner = arb.DIFFUSION)[0])+ with patch(+ "core.inference.diffusion_memory.plan_diffusion_memory",+ side_effect = AssertionError("planner"),+ ):+ self.assertTrue(self._check(_need({0: 1000}, kind = "video"))[0])+++class TestPipelineNeed(_GpuCacheResetMixin, unittest.TestCase):+ def setUp(self):+ import json++ self._tmp = tempfile.TemporaryDirectory()+ self.root = Path(self._tmp.name) / "Z-Image-Turbo"+ self.root.mkdir()+ (self.root / "model_index.json").write_text(+ json.dumps(+ {+ "_class_name": "ZImagePipeline",+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],+ "text_encoder": ["transformers", "Qwen3Model"],+ "tokenizer": ["transformers", "Qwen2Tokenizer"],+ "transformer": ["diffusers", "ZImageTransformer2DModel"],+ "vae": ["diffusers", "AutoencoderKL"],+ }+ )+ )+ (self.root / "vae").mkdir()+ (self.root / "vae" / "config.json").write_text("{}")+ _sparse(self.root / "vae" / "diffusion_pytorch_model.safetensors", 100)+ self._patches = [+ patch(+ "core.inference.diffusion_engine_router.predict_engine", return_value = "diffusers"+ ),+ patch(+ "core.inference.diffusion_families.detect_family_for_pick",+ return_value = SimpleNamespace(name = "z-image"),+ ),+ ]+ for p in self._patches:+ p.start()++ def tearDown(self):+ for p in reversed(self._patches):+ p.stop()+ self._tmp.cleanup()+ super().tearDown()++ def _need(self):+ request = _image_request(self.root, None, model_kind = "pipeline")+ with _parent_ids((0,)):+ return gr.media_load_need_mib(arb.DIFFUSION, request, 0)++ def test_a_partly_downloaded_pipeline_is_unknown(self):+ self.assertIsNone(self._need())++ def test_a_complete_pipeline_is_priced_from_every_component(self):+ (self.root / "scheduler").mkdir()+ (self.root / "scheduler" / "scheduler_config.json").write_text("{}")+ (self.root / "tokenizer").mkdir()+ (self.root / "tokenizer" / "tokenizer_config.json").write_text("{}")+ (self.root / "tokenizer" / "tokenizer.json").write_text("{}")+ for name, stem, mib in (+ ("text_encoder", "model", 8000),+ ("transformer", "diffusion_pytorch_model", 12000),+ ):+ (self.root / name).mkdir()+ (self.root / name / "config.json").write_text("{}")+ _sparse(self.root / name / f"{stem}.safetensors", mib)+ need = self._need()+ self.assertIsNotNone(need)+ self.assertGreaterEqual(need.weights_mib, 20000)+++class TestFailedLoadPrice(_ArbiterResetMixin, unittest.TestCase):+ def test_a_price_counts_only_while_its_model_is_loaded_or_loading(self):+ self._engine_patch.stop()+ self._resident(arb.VIDEO, {0}, _need({0: 5000}))+ for state, loading, counted in (+ (None, (), False),+ ("s", (), True),+ (None, ("org/v",), True),+ ):+ engine = SimpleNamespace(_state = state, loading_repo_ids = lambda loading = loading: loading)+ with patch.object(gr, "_media_engine", return_value = engine):+ self.assertEqual(gr.media_held_mib(), {0: 5000} if counted else {})+ with patch.object(gr, "_media_engine", return_value = None):+ self.assertEqual(gr.media_held_mib(), {0: 5000})+++class TestMediaHeldMib(_ArbiterResetMixin, unittest.TestCase):+ def test_nothing_resident_is_empty(self):+ self.assertEqual(gr.media_held_mib(), {})+ arb._media_needs[arb.VIDEO] = _need({0: 5000}) # a record with no resident+ self.assertEqual(gr.media_held_mib(), {})++ def test_weights_plus_headroom_summed_per_gpu(self):+ self._resident(arb.CHAT, {0, 1})+ self._resident(arb.VIDEO, {0}, _need({0: 5000}))+ self._resident(arb.DIFFUSION, {0, 1}, _need({0: 7000, 1: 1000}, kind = "image"))+ self.assertEqual(gr.media_held_mib(), {0: 12000, 1: 1000})++ def test_non_resident_record_is_ignored(self):+ self._resident(arb.DIFFUSION, {0}, _need({0: 7000}, kind = "image"))+ arb._media_needs[arb.VIDEO] = _need({0: 5000})+ self.assertEqual(gr.media_held_mib(), {0: 7000})++ def test_corrupt_record_never_raises(self):+ self._resident(arb.VIDEO, {0})+ arb._media_needs[arb.VIDEO] = {0: "lots"}+ self.assertEqual(gr.media_held_mib(), {})+++class TestFreeForOrder(_ArbiterResetMixin, unittest.TestCase):+ def setUp(self):+ super().setUp()+ self.log, self.evicted = [], []+ self.loading, self.generating, self.foreign, self.fits = (+ set(),+ set(),+ {},+ MagicMock(return_value = (False, {})),+ )+ recorder = lambda name: (lambda: self.log.append(name))+ self.dropped = list+ self.monitor = MagicMock()+ self._patches = [+ patch.dict(+ arb._EVICTORS,+ {+ arb.CHAT: recorder("chat"),+ arb.DIFFUSION: recorder("image"),+ arb.VIDEO: recorder("video"),+ },+ ),+ patch.object(+ arb,+ "resident_slots",+ side_effect = lambda n: [+ {"kind": gr._MEDIA_KIND.get(n, "chat"), "model": f"{n}-model"}+ ],+ ),+ patch.object(+ gr, "_media_busy", side_effect = lambda n: (n in self.loading, n in self.generating)+ ),+ patch.object(gr, "chat_load_in_flight", side_effect = lambda: arb.CHAT in self.loading),+ patch.object(gr, "foreign_on", side_effect = lambda n, a: self.foreign.get(n, 0)),+ patch.object(gr, "can_load_alongside", self.fits),+ patch.object(+ gr, "_kept_slot_report", return_value = {"kind": "chat", "model": "kept-model"}+ ),+ patch(+ "core.inference.video_minimax_h3.release_h3_native_servers",+ side_effect = lambda reason: self.log.append("h3") or 1,+ ),+ patch(+ "core.inference.model_slots.eviction_victims",+ side_effect = lambda *a: self.log.append("kept") or ["slot"],+ ),+ patch(+ "core.inference.model_slots.evict_these",+ side_effect = lambda v: self.log.append("evict_these") or self.dropped(v),+ ),+ patch("core.inference.api_monitor.api_monitor", self.monitor),+ patch("utils.account_context.current_account_id", return_value = "me"),+ ]+ for p in self._patches:+ p.start()++ def tearDown(self):+ for p in reversed(self._patches):+ p.stop()+ super().tearDown()++ def _free(+ self,+ owner = arb.DIFFUSION,+ gpus = (0,),+ ):+ return gr.free_for(+ owner, gpus, "test", self.evicted, need_mib = _need({0: 1000}, kind = "image")+ )++ def test_an_h3_server_beside_video_on_another_gpu_stays(self):+ self._resident(arb.VIDEO, {1})+ self._resident(arb.CHAT, {0})+ self._free()+ self.assertNotIn("h3", self.log)++ def test_a_kept_slot_a_request_reached_is_not_reported(self):+ self._resident(arb.CHAT, {0})+ self.dropped = lambda victims: []+ self._free()+ self.assertNotIn({"kind": "chat", "model": "kept-model"}, self.evicted)++ def test_idle_media_before_kept_then_active_chat(self):+ self._resident(arb.VIDEO, {0})+ self._resident(arb.CHAT, {0})+ freed = self._free()+ self.assertEqual(self.log, ["h3", "video", "kept", "evict_these", "chat"])+ self.assertEqual(freed, ["video-server", "video-model", "kept-model", "chat-model"])+ self.assertEqual(arb._view(), {})+ self.assertEqual(+ self.evicted,+ [+ {"kind": "video", "model": "video-model"},+ {"kind": "chat", "model": "kept-model"},+ {"kind": "chat", "model": "chat-model"},+ ],+ )+ self.monitor.record_lifecycle.assert_any_call(+ event = "unload", model = "video-model", reason = "evicted"+ )+ self.assertEqual(self.monitor.record_lifecycle.call_count, 3)++ def test_loading_resident_goes_first(self):+ self._resident(arb.VIDEO, {0})+ self._resident(arb.CHAT, {0})+ self.loading = {arb.VIDEO, arb.CHAT}+ self._free()+ self.assertEqual(self.log[:3], ["video", "chat", "h3"])+ self.assertNotIn("kept", self.log) # CHAT is gone, so no kept slots overlap++ def test_stops_once_it_fits(self):+ self._resident(arb.VIDEO, {0})+ self._resident(arb.CHAT, {0})+ self.fits.side_effect = [(False, {}), (True, {})]+ self.assertEqual(self._free(), ["video-server", "video-model"])+ self.assertEqual(self.log, ["h3", "video"])+ self.assertIn(arb.CHAT, arb._view())++ def test_disjoint_gpus_and_busy_media_are_untouched(self):+ self._resident(arb.VIDEO, {1})+ self._resident(arb.CHAT, {0})+ self._free()+ self.assertNotIn("video", self.log)+ self._resident(arb.VIDEO, {0})+ self.generating = {arb.VIDEO}+ self.log.clear()+ self._resident(arb.CHAT, {0})+ self._free()+ self.assertEqual(self.log, ["h3", "kept", "evict_these", "chat"])+ self.assertIn(arb.VIDEO, arb._view())++ def test_foreign_generation_blocks_the_eviction(self):+ self._resident(arb.CHAT, {0})+ self.foreign = {arb.CHAT: 2}+ with self.assertRaises(arb.GpuBusyForAnotherAccountError) as ctx:+ self._free()+ self.assertEqual((ctx.exception.owner, ctx.exception.active), (arb.CHAT, 2))+ self.assertNotIn("chat", self.log)+ self.assertIn(arb.CHAT, arb._view())+++class TestOtherPlannedIncludesMedia(_ArbiterResetMixin, unittest.TestCase):+ def test_media_held_is_added_for_a_registered_backend(self):+ from core.inference import llama_cpp+ from core.inference.llama_cpp import LlamaCppBackend++ mine, other, stray = (LlamaCppBackend(manages_processes = False) for _ in range(3))+ other._process = object()+ other._planned_vram_mib = {0: 4000}+ self._resident(arb.DIFFUSION, {0, 1}, _need({0: 7000, 1: 500}, kind = "image"))+ for backend in (mine, other):+ llama_cpp.register_serving_backend(backend)+ try:+ self.assertEqual(mine._other_planned_vram_mib(), {0: 11000, 1: 500})+ self.assertEqual(stray._other_planned_vram_mib(), {})+ finally:+ for backend in (mine, other):+ llama_cpp.unregister_serving_backend(backend)+++class TestChatLoadingRefusal(unittest.TestCase):+ def test_api_shape(self):+ exc = gr.chat_loading_http_exception("/api/inference/images/load")+ self.assertEqual(exc.status_code, 409)+ self.assertEqual(exc.headers, {"Retry-After": "15"})+ self.assertEqual(set(exc.detail), {"error", "message", "retry_after"})+ self.assertEqual((exc.detail["error"], exc.detail["retry_after"]), ("gpu_busy", 15))+ self.assertEqual(gr.ChatLoadInFlightRefusal().as_http_exception().detail, exc.detail)++ def test_openai_shape(self):+ for exc in (+ gr.chat_loading_http_exception("/v1/images/generations"),+ gr.chat_loading_http_exception("/api/x", openai_errors = True),+ ):+ self.assertEqual(exc.headers, {"Retry-After": "15"})+ self.assertEqual(exc.detail["error"]["code"], "gpu_busy")+ message = exc.detail["error"]["message"]+ self.assertNotIn("is set to Always", message)+ self.assertNotIn("is set to keep loaded", message)+++class TestKeptSlotReport(unittest.TestCase):+ """An evicted kept slot reports the public id the status code shows, never a host path."""++ def test_loaded_gguf_slot_reports_public_id(self):+ llama = SimpleNamespace(+ is_loaded = True,+ model_identifier = "/srv/models/Qwen3-4B-Q4_K_M.gguf",+ _openai_advertised_id = None,+ )+ orchestrator = SimpleNamespace(active_model_name = None)+ slot = SimpleNamespace(+ llama = llama,+ orchestrator = orchestrator,+ account = "me",+ request = SimpleNamespace(model_path = "/srv/models/Qwen3-4B-Q4_K_M.gguf"),+ )+ report = gr._kept_slot_report(slot)+ self.assertEqual(report["model"], "Qwen3-4B-Q4_K_M")+ self.assertEqual(report["kind"], "chat")+ self.assertEqual(report["account"], "me")++ def test_fallback_never_reports_a_path(self):+ slot = SimpleNamespace(+ llama = None,+ orchestrator = None,+ account = None,+ request = SimpleNamespace(model_path = "/srv/models/Qwen3-4B-Q4_K_M.gguf"),+ )+ with patch("routes.inference._kept_slot_reports", side_effect = RuntimeError("x")):+ report = gr._kept_slot_report(slot)+ self.assertEqual(report["model"], "Qwen3-4B-Q4_K_M")+++if __name__ == "__main__":+ unittest.main()diff --git a/studio/backend/tests/test_media_auto_switch.py b/studio/backend/tests/test_media_auto_switch.pyindex 87a4c796fb1..0fb48574066 100644--- a/studio/backend/tests/test_media_auto_switch.py+++ b/studio/backend/tests/test_media_auto_switch.py@@ -1744,6 +1744,117 @@ def test_a_cpu_switch_does_not_hold_the_gates_it_cannot_evict_behind(assert held == []+def _chat_loading():+ """A chat load in flight, as the real one shows it: chat's load lock held."""+ assert chat._load_lock.acquire(blocking = False)+ try:+ yield+ finally:+ chat._load_lock.release()++[email protected]("openai_errors", [False, True])+def test_always_refuses_behind_a_chat_load_without_waiting(+ flux, enabled, takes_the_gpu, backend, loads, monkeypatch, openai_errors+):+ # Under Always the chat load's GPUs aren't known yet, so it overlaps every card: the switch+ # answers the retryable 409 at once instead of queueing on chat's gate behind the load.+ import core.inference.gpu_residency as residency++ monkeypatch.setattr(residency, "current_policy", lambda: "always")+ # The chat load has claimed the GPU.+ monkeypatch.setattr(arb, "_owner", arb.CHAT)+ monkeypatch.setattr(arb, "_residents", {arb.CHAT: None})++ def _gate():+ raise AssertionError("an Always switch must not wait behind a chat load")++ monkeypatch.setattr(chat, "inference_lifecycle_gate", _gate)++ with _chat_loading():+ began = time.monotonic()+ with pytest.raises(HTTPException) as excinfo:+ _switch("black-forest-labs/FLUX.1-dev", openai_errors = openai_errors)+ elapsed = time.monotonic() - began++ assert excinfo.value.status_code == 409+ assert excinfo.value.headers["Retry-After"] == "15"+ detail = excinfo.value.detail+ if openai_errors:+ assert detail["error"]["code"] == "gpu_busy"+ else:+ assert detail["error"] == "gpu_busy"+ assert detail["retry_after"] == 15+ assert isinstance(detail["message"], str)+ assert set(detail) == {"error", "message", "retry_after"}+ assert "is set to Always" not in detail["message"]+ assert "is set to keep loaded" not in detail["message"]+ assert loads == []+ assert elapsed < 5.0+ # The switch's locks went back with the refusal.+ assert not asyncio.run(_lock_is_held(arb.DIFFUSION))+++def test_fit_keeps_todays_wait_behind_a_chat_load(+ flux, enabled, takes_the_gpu, backend, loads, monkeypatch+):+ # Only Always refuses: under fit the switch still queues on chat's lifecycle gate, and a+ # load that holds it past the budget ends in today's 503.+ import core.inference.gpu_residency as residency++ monkeypatch.setattr(residency, "current_policy", lambda: "fit")+ monkeypatch.setattr(mas, "_SWITCH_BUDGET_S", 0.4)+ assert chat._lifecycle_lock.acquire(blocking = False)+ try:+ with _chat_loading():+ with pytest.raises(HTTPException) as excinfo:+ _switch("black-forest-labs/FLUX.1-dev")+ finally:+ chat._lifecycle_lock.release()++ assert excinfo.value.status_code == 503+ assert loads == []++[email protected]("claim", [{}, {arb.CHAT: frozenset({0})}])+def test_always_waits_behind_a_cpu_or_placed_chat_load(+ flux, enabled, takes_the_gpu, backend, loads, monkeypatch, claim+):+ # A CPU chat load never claims the GPU, and one on known GPUs is checked for overlap by the+ # arbiter, so the switch queues on chat's gate as under fit instead of answering the 409.+ import core.inference.gpu_residency as residency++ monkeypatch.setattr(residency, "current_policy", lambda: "always")+ monkeypatch.setattr(arb, "_owner", arb.CHAT if claim else None)+ monkeypatch.setattr(arb, "_residents", dict(claim))+ monkeypatch.setattr(mas, "_SWITCH_BUDGET_S", 0.4)+ assert chat._lifecycle_lock.acquire(blocking = False)+ try:+ with _chat_loading():+ with pytest.raises(HTTPException) as excinfo:+ _switch("black-forest-labs/FLUX.1-dev")+ finally:+ chat._lifecycle_lock.release()++ assert excinfo.value.status_code == 503+ assert loads == []+++def test_cpu_switch_ignores_a_chat_load_under_always(flux, enabled, backend, loads, monkeypatch):+ # A CPU load evicts nothing on the GPU, so a chat load there is no reason to refuse it.+ import core.inference.gpu_residency as residency++ monkeypatch.setattr(backends, "load_takes_the_gpu", lambda: False)+ monkeypatch.setattr(mas, "load_takes_the_gpu", lambda: False)+ monkeypatch.setattr(residency, "current_policy", lambda: "always")++ with _chat_loading():+ _switch("black-forest-labs/FLUX.1-dev")++ assert [pick.model_id for _owner, pick in loads] == ["black-forest-labs/FLUX.1-dev"]++def test_the_video_route_refuses_a_flow_control_the_target_does_not_expose(monkeypatch):# LTX-2 exposes no audio_flow_shift, and _resolve_flow_shifts would only say so after the# switch had evicted the resident pipeline and loaded the target.diff --git a/studio/backend/tests/test_multi_model_primary_fit.py b/studio/backend/tests/test_multi_model_primary_fit.pynew file mode 100644index 00000000000..160b23da1aa--- /dev/null+++ b/studio/backend/tests/test_multi_model_primary_fit.py@@ -0,0 +1,415 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A primary chat load beside a kept image or video model: make room under When they fit, refuse under Always."""++import asyncio+from types import SimpleNamespace++import pytest+from fastapi import HTTPException++import core.inference.model_slots as model_slots+import core.inference.orchestrator as orchestrator+import routes.inference as inf+from core.inference import gpu_arbiter, gpu_residency+from core.inference.llama_cpp import GpuMemoryShortError+from models.inference import LoadRequest, LoadResponse+from utils import multi_model_settings+from utils.account_context import AccountContext, arun_as++BOB = AccountContext("b" * 32, "bob")+IMG = {"kind": "image", "model": "org/Img"}+++class FakeLlama:+ holds_no_vram, is_vision, context_length = False, False, 4096++ def __init__(+ self,+ identifier = None,+ variant = None,+ ):+ self.model_identifier, self.hf_variant = identifier, variant+ self.is_loaded = self.is_active = identifier is not None+++class FakeOrchestrator:+ active_model_name, models = None, {}+++def env(monkeypatch):+ """Primary-only Unsloth Studio with spies on everything that could unload a model."""+ monkeypatch.setattr(inf, "_llama_cpp_backend", FakeLlama("org/A-GGUF", "Q4_K_M"))+ monkeypatch.setattr(orchestrator, "_inference_backend", FakeOrchestrator())+ monkeypatch.setattr(model_slots, "slots", [])+ monkeypatch.setattr(model_slots, "stuck", [])+ monkeypatch.setattr(inf, "LlamaCppBackend", FakeLlama)+ monkeypatch.setattr(inf, "InferenceOrchestrator", FakeOrchestrator)+ monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", lambda: None)+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)+ monkeypatch.setattr(multi_model_settings, "get_multi_model_enabled", lambda: True)+ state = SimpleNamespace(+ policy = "fit",+ requests = [],+ drops = [],+ evicts = [],+ unloads = [],+ free_calls = [],+ frees = [IMG],+ errors = [],+ retry_flags = [],+ )+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: state.policy)+ monkeypatch.setattr(model_slots, "drop", lambda slot: state.drops.append(slot))+ monkeypatch.setattr(model_slots, "evict", lambda *a: state.evicts.append(a) or [])+ monkeypatch.setattr(inf, "_unload_primary", lambda: state.unloads.append(True))++ def free_for(+ owner,+ gpu_ids,+ reason,+ evicted = None,+ *,+ need_mib = None,+ short_mib = 0,+ media_only = False,+ ):+ call = {"owner": owner, "gpu_ids": tuple(gpu_ids or ()), "short_mib": short_mib}+ state.free_calls.append({**call, "media_only": True} if media_only else call)+ evicted.extend(dict(s) for s in state.frees)+ return [s["model"] for s in state.frees]++ monkeypatch.setattr(gpu_residency, "free_for", free_for)+ rows = {0: {"free_mib": 4096, "total_mib": 81920}}+ monkeypatch.setattr(gpu_residency, "_free_mib_by_gpu", lambda: (rows, None))+ monkeypatch.setattr(+ gpu_arbiter, "resident_slots", lambda o: [dict(IMG)] if o == gpu_arbiter.DIFFUSION else []+ )++ async def load(request, *args, **kwargs):+ state.requests.append(request)+ state.retry_flags.append(inf._primary_can_retry_fit.get())+ if state.errors and not request.force_alongside:+ raise state.errors.pop(0)+ return LoadResponse(+ status = "loaded", model = request.model_path, display_name = "A", inference = {}+ )++ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)+ return state+++def _short(capped = False):+ return GpuMemoryShortError(+ "needs 13 GB, 8 GB free", capped = capped, short_mib = 5000, gpu_indices = (0,)+ )+++def _gated(account = None):+ async def run():+ coro = inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF"), None, "s")+ response = await (arun_as(account, coro) if account else coro)+ return response, inf._primary_can_retry_fit.get()++ return asyncio.run(run())+++def _nothing_unloaded(state):+ assert state.drops == [] and state.evicts == [] and state.unloads == []+ assert model_slots.stuck == []+++def test_fit_frees_the_image_model_and_retries(env):+ assert inf._primary_can_retry_fit.get() is False+ env.errors = [_short()]+ response, flag_after = _gated()+ assert len(env.requests) == 2 and not env.requests[1].force_alongside+ assert response.evicted == ["org/Img"]+ assert env.free_calls == [{"owner": gpu_arbiter.CHAT, "gpu_ids": (0,), "short_mib": 5000}]+ assert env.drops == [] and model_slots.stuck == [] # model_slots.drop(None) never runs+ assert env.retry_flags == [True, True] and flag_after is False+++def test_freed_media_is_reported_once(env):+ env.errors = [_short()]+ response, _ = _gated()+ # The route's eviction sink carries it into both fields; the retry must not add it a second time.+ assert response.evicted == ["org/Img"] and response.evicted_slots == [IMG]+++def test_an_idle_video_server_freed_alone_still_earns_a_full_size_retry(env, monkeypatch):+ env.errors = [_short()]+ monkeypatch.setattr(+ gpu_residency, "free_for", lambda *a, **k: env.free_calls.append(a) or ["video-server"]+ )+ response, _ = _gated()+ assert [r.force_alongside for r in env.requests] == [False, False] and not response.evicted+++def test_fit_with_nothing_freed_lets_llama_server_fit_what_it_can(env):+ env.errors, env.frees = [_short()], []+ response, _ = _gated()+ assert [r.force_alongside for r in env.requests] == [False, True]+ assert response.status == "loaded" and not response.evicted+ _nothing_unloaded(env)++[email protected]("policy", ["fit", "always"])+def test_capped_fit_retries_at_a_smaller_context(env, policy):+ env.policy, env.errors = policy, [_short(capped = True)]+ response, _ = _gated()+ assert [r.force_alongside for r in env.requests] == [False, True]+ assert response.status == "loaded"+ assert env.free_calls == []+ _nothing_unloaded(env)+++def _refused(account = None) -> str:+ """The Always refusal reaches the caller as the K2 409, header included."""+ with pytest.raises(HTTPException) as info:+ _gated(account)+ assert info.value.status_code == 409+ assert info.value.headers.get("X-Unsloth-Refusal") == "multi-model-always"+ return str(info.value.detail)+++def test_always_refuses_with_the_numbers_for_the_owner(env):+ env.policy, env.errors = "always", [_short()]+ message = _refused()+ assert "is set to Always" in message+ # 5000 MiB short beside 4096 MiB free: needs 8.9 GB, 4.0 GB is free.+ assert "needs about 8.9 GB of GPU memory, and 4.0 GB is free" in message+ assert env.free_calls == [] and len(env.requests) == 1+ _nothing_unloaded(env)+++def test_always_refuses_a_managed_account_with_the_plain_copy(env):+ env.policy, env.errors = "always", [_short()]+ message = _refused(BOB)+ assert "GB" not in message+ assert "the image model org/Img, which Unsloth Studio is set to keep loaded" in message+ assert env.free_calls == []+ _nothing_unloaded(env)+++def test_always_names_only_media_on_the_short_gpu(env, monkeypatch):+ # The image on GPU 1 isn't in the way of a chat load short on GPU 0; the video model there is.+ vid = {"kind": "video", "model": "org/Vid"}+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.VIDEO)+ monkeypatch.setattr(+ gpu_arbiter,+ "_residents",+ {gpu_arbiter.DIFFUSION: frozenset({1}), gpu_arbiter.VIDEO: frozenset({0})},+ )+ monkeypatch.setattr(+ gpu_arbiter,+ "resident_slots",+ lambda o: {gpu_arbiter.DIFFUSION: [dict(IMG)], gpu_arbiter.VIDEO: [dict(vid)]}.get(o, []),+ )+ env.policy, env.errors = "always", [_short()]+ message = _refused(BOB)+ assert "org/Vid" in message and "org/Img" not in message+ _nothing_unloaded(env)+++def _image_resident(monkeypatch):+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)+ monkeypatch.setattr(gpu_arbiter, "_residents", {gpu_arbiter.DIFFUSION: frozenset({0})})+ assert gpu_arbiter.holds(gpu_arbiter.DIFFUSION)+++def _oom():+ return HTTPException(status_code = 500, detail = "CUDA out of memory")+++def test_non_gguf_oom_under_always_beside_an_image_model_is_refused(env, monkeypatch):+ _image_resident(monkeypatch)+ env.policy, env.errors = "always", [_oom()]+ message = _refused()+ assert "is set to Always" in message and "org/Img" in message+ assert env.free_calls == []+ _nothing_unloaded(env)+++def test_non_gguf_oom_under_always_beside_media_on_another_gpu_keeps_the_original_error(+ env, monkeypatch+):+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)+ monkeypatch.setattr(+ gpu_arbiter,+ "_residents",+ {gpu_arbiter.DIFFUSION: frozenset({0}), gpu_arbiter.CHAT: frozenset({1})},+ )+ env.policy, env.errors = "always", [_oom()]+ with pytest.raises(HTTPException) as info:+ _gated()+ assert info.value.status_code == 500 and info.value.detail == "CUDA out of memory"+ _nothing_unloaded(env)+++def test_non_gguf_oom_under_fit_keeps_the_original_error(env, monkeypatch):+ _image_resident(monkeypatch)+ env.errors = [_oom()]+ with pytest.raises(HTTPException) as info:+ _gated()+ assert info.value.status_code == 500 and info.value.detail == "CUDA out of memory"+ assert env.free_calls == [] and len(env.requests) == 1+ _nothing_unloaded(env)+++def test_non_gguf_oom_under_always_without_media_keeps_the_original_error(env, monkeypatch):+ monkeypatch.setattr(gpu_arbiter, "_owner", None)+ env.policy, env.errors = "always", [_oom()]+ with pytest.raises(HTTPException) as info:+ _gated()+ assert info.value.status_code == 500+ _nothing_unloaded(env)+++def _placement(+ resolved,+ vulkan = False,+ native_gb = None,+):+ return SimpleNamespace(+ requested_gpu_ids = None,+ resolved_gpu_ids = resolved,+ gpu_ids_are_vulkan_ordinals = vulkan,+ native_required_gb = native_gb,+ )+++ "held, placement, expected",+ [+ ({}, _placement([0]), False),+ ({1: 4096}, _placement([0]), False),+ ({1: 4096}, _placement([1]), True),+ ({1: 4096}, _placement(None), True),+ ({1: 4096}, _placement([0], vulkan = True), True),+ ({}, _placement([0], vulkan = True), False),+ ],+)+def test_media_on_target_gpus(monkeypatch, held, placement, expected):+ monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: held)+ assert inf._media_on_target_gpus(placement) is expected+++def test_chat_fit_kwargs(monkeypatch):+ policy = SimpleNamespace(value = "off")+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: policy.value)+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: [])+ gguf, native = SimpleNamespace(is_gguf = True), SimpleNamespace(is_gguf = False)+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {}+ policy.value = "fit"+ token = inf._primary_can_retry_fit.set(True)+ try:+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {"policy": "fit", "gpu_ids": [0]}+ # Anything but GGUF has no fit estimate before launch, even inside the retry.+ assert inf._chat_fit_kwargs(native, _placement([0])) == {+ "policy": "fit",+ "gpu_ids": [0],+ "evict_first": True,+ }+ finally:+ inf._primary_can_retry_fit.reset(token)+ # Auto-switch calls the tracked impl without load_model_gated, so nothing would make room after.+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {+ "policy": "fit",+ "gpu_ids": [0],+ "evict_first": True,+ }+ native_need = inf._chat_fit_kwargs(native, _placement([0, 1], native_gb = 8.0))+ assert native_need == {"policy": "fit", "gpu_ids": [0, 1], "need_mib": {0: 4096, 1: 4096}}+ policy.value = "always"+ assert inf._chat_fit_kwargs(gguf, _placement([0])) == {"policy": "always", "gpu_ids": [0]}+ # No target GPUs (or Vulkan ordinals) leaves every GPU in play.+ assert inf._chat_fit_kwargs(gguf, _placement(None)) == {"policy": "always"}+ assert inf._chat_fit_kwargs(gguf, _placement([0], vulkan = True)) == {"policy": "always"}+++ "parent, physical, expected",+ [+ ([], [3], [3]), # no mask: physical ids are the ordinals+ ([2, 3], [3], [1]), # CUDA_VISIBLE_DEVICES=2,3+ ([2, 3], [3, 2], [1, 0]),+ ([2, 3], [5], None), # not visible: every GPU counts+ ],+)+def test_visible_ordinals(monkeypatch, parent, physical, expected):+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: parent)+ assert inf._visible_ordinals(physical) == expected+++def test_the_replace_fallback_never_refuses_a_partial_fit():+ # Nothing retries the fallback load, so a refusal beside media there would escape as a raw GpuMemoryShortError.+ import inspect++ source = inspect.getsource(inf.load_model_gated)+ unload = source.index("await asyncio.to_thread(_unload_primary)")+ assert (+ unload+ < source.index("_primary_can_retry_fit.set(False)", unload)+ < source.index("_run_tracked_load_model_impl", unload)+ )+++def test_a_pinned_non_gguf_chat_load_keeps_its_requested_gpus(monkeypatch):+ # Only GGUF and native audio resolve their GPUs: a Transformers load pinned to GPU 1 still targets GPU 1.+ pinned = inf._LoadPlacement([1], None, False, None)+ monkeypatch.setattr(gpu_residency, "current_policy", lambda: "always")+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: [])+ monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: {0: 4096})+ assert inf._chat_fit_kwargs(SimpleNamespace(is_gguf = False), pinned) == {+ "policy": "always",+ "gpu_ids": [1],+ }+ assert inf._media_on_target_gpus(pinned) is False+++ ("serving", "policy", "expected"),+ [(False, "always", True), (True, "always", False), (True, "fit", True), (False, "fit", True)],+)+def test_a_serving_primary_reloaded_under_always_is_not_torn_down_for_a_refusal(+ monkeypatch, serving, policy, expected+):+ # Always only refuses; the partial-fit check runs after the old llama-server is killed.+ monkeypatch.setattr(gpu_residency, "media_held_mib", lambda: {0: 4096})+ monkeypatch.setattr("utils.hardware.hardware.get_parent_visible_gpu_ids", lambda: [])+ placement = inf._LoadPlacement([0], [0], False, None)+ token = inf._primary_can_retry_fit.set(True)+ try:+ assert inf._primary_refuses_partial_fit(serving, policy, placement) is expected+ finally:+ inf._primary_can_retry_fit.reset(token)+++def test_an_extra_chat_load_frees_an_idle_image_before_replacing_the_primary(env, monkeypatch):+ # Chat A + an idle image under When they fit: chat B loaded alongside comes up short. The image goes and B+ # loads beside A, instead of A being unloaded (and the image after it, by the replacement primary load).+ extra = model_slots.ExtraSlot(FakeLlama(), FakeOrchestrator(), "owner")++ async def select(_request):+ return extra++ monkeypatch.setattr(inf, "_select_load_slot", select)+ monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: "fit")+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *a, **k: [])+ monkeypatch.setattr(inf, "_raise_or_cancel_active_generations", lambda **k: None)+ env.errors = [_short()]++ async def run():+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)+ return await inf.load_model_gated(request, None, "s")++ response = asyncio.run(run())+ assert env.unloads == []+ assert env.free_calls == [+ {"owner": gpu_arbiter.CHAT, "gpu_ids": (0,), "short_mib": 5000, "media_only": True}+ ]+ assert len(env.requests) == 2 and env.requests[1].alongside+ assert response.evicted == ["org/Img"]