Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Prices image/video residents against live free VRAM; anything unpriceable counts as not fitting"""
from __future__ import annotations
from types import SimpleNamespace
from typing import Any, Iterable, Optional
from core.inference import gpu_arbiter as arb
from loggers import get_logger
logger = get_logger(__name__)
# fits iff free_mib >= need_mib + FIT_MARGIN_MIB on every GPU the load uses. Live free VRAM already excludes resident
# weights, so the margin only covers estimate error and allocator slack. An image load must also keep its
# transformer on the GPU (the diffusion planner adds its own reserve on top, so this is not doubled there).
FIT_MARGIN_MIB = 1024
# A chat load in flight finishes on its own; the media auto-switch uses the same wait.
K3_RETRY_AFTER_S = 15
MEDIA_OWNERS = (arb.DIFFUSION, arb.VIDEO)
_MEDIA_KIND = {arb.DIFFUSION: "image", arb.VIDEO: "video"}
class MediaNeed(dict):
"""{physical GPU id: weights + generation headroom MiB}, plus the split the planner rule needs."""
def __init__(self, per_gpu: dict, *, weights_mib: int, headroom_mib: int, kind: str):
super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})
self.weights_mib = int(weights_mib)
self.headroom_mib = int(headroom_mib)
self.kind = kind
class ChatNeed(dict):
"""Chat load {GPU: MiB}, priced against free memory plus ``credit_mib`` released by the model it replaces."""
def __init__(
self,
per_gpu: dict,
*,
credit_mib: int = 0,
):
super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})
self.credit_mib = int(credit_mib)
def current_policy() -> str:
"""Keep multiple models loaded, read as the owner. Never call this under the arbiter lock (it reads a database)."""
try:
from utils.multi_model_settings import get_multi_model_mode
return get_multi_model_mode()
except Exception as exc: # noqa: BLE001 - an unreadable setting keeps today's behaviour
logger.warning("gpu_residency: could not read Keep multiple models loaded: %s", exc)
return "off"
def _media_records() -> dict[str, dict]:
"""Admitted needs of the media residents still on the GPU. Lock free: acquire_for calls this holding the lock."""
records = dict(getattr(arb, "_media_needs", {}) or {})
if not records:
return {}
resident = set(arb._view()) if hasattr(arb, "_view") else {arb._owner}
return {
owner: need
for owner, need in records.items()
if owner in resident and _loaded_or_loading(owner)
}
def _loaded_or_loading(owner: str) -> bool:
"""False once a media load failed and left nothing loaded, so its price stops counting. Unknown reads as True."""
try:
engine = _media_engine(owner)
if engine is None:
return True
return getattr(engine, "_state", None) is not None or bool(engine.loading_repo_ids())
except Exception: # noqa: BLE001
return True
def media_held_mib() -> dict[int, int]:
"""{physical GPU id: MiB} held by resident image and video models (weights + headroom). Never raises."""
try:
held: dict[int, int] = {}
for need in _media_records().values():
for gpu, mib in need.items():
held[gpu] = held.get(gpu, 0) + int(mib)
return held
except Exception as exc: # noqa: BLE001
logger.warning("gpu_residency: could not price the resident media models: %s", exc)
return {}
def media_owner_held_mib(owner: str) -> Optional[int]:
"""MiB one resident media owner holds across its GPUs: its admitted need less the generation headroom
it is not using while idle (as media_unallocated_mib counts it). None when unpriced or loading. Never raises."""
try:
need = _media_records().get(owner)
if not need:
return None
loading, generating = _media_busy(owner)
if loading:
return None
headroom = 0 if generating else int(getattr(need, "headroom_mib", 0))
total = sum(max(0, int(mib) - headroom) for mib in need.values())
return total or None
except Exception: # noqa: BLE001
return None
def media_unallocated_mib() -> dict[int, int]:
"""{physical GPU id: MiB} of the media price live free VRAM still shows as free: an idle resident's generation
headroom, or all of a resident still loading. Never raises."""
try:
held: dict[int, int] = {}
for owner, need in _media_records().items():
loading = _media_busy(owner)[0]
for gpu, mib in need.items():
unallocated = int(mib) if loading else int(getattr(need, "headroom_mib", 0))
held[gpu] = held.get(gpu, 0) + unallocated
return held
except Exception as exc: # noqa: BLE001
logger.warning("gpu_residency: could not price the resident media models: %s", exc)
return {}
def _media_engine(owner: str) -> Any:
from core.inference.media_keepwarm import engine_if_imported
return engine_if_imported(owner)
def _media_busy(owner: str) -> tuple[bool, bool]:
"""(loading, generating) for a media owner; unknown reads as busy, so it is never freed as idle."""
try:
engine = _media_engine(owner)
if engine is None:
return False, False
loading = bool(engine.loading_repo_ids())
progress = engine.generate_progress() or {}
from core.inference.media_keepwarm import _TRACKERS
tracker = _TRACKERS.get(owner)
outstanding = bool(tracker is not None and tracker.outstanding())
return loading, bool(progress.get("active")) or outstanding
except Exception: # noqa: BLE001
return True, True
def summarize_resident_media() -> dict[str, Any]:
"""Report the resident image and video models. Never raises."""
summary: dict[str, Any] = {}
records = {}
try:
records = _media_records()
resident = arb.residents()
except Exception: # noqa: BLE001
resident = {}
for owner in MEDIA_OWNERS:
loading, generating = _media_busy(owner) if owner in resident else (False, False)
summary[owner] = {
"resident": owner in resident,
"gpus": sorted(resident.get(owner) or ()),
"loading": loading,
"generating": generating,
"priced": owner in records,
}
summary["loading"] = any(summary[owner]["loading"] for owner in MEDIA_OWNERS)
summary["held_mib"] = media_held_mib()
return summary
def media_load_need_mib(kind: str, request: Any, gpu_ordinal: Optional[int]) -> Optional[MediaNeed]:
"""MiB a new image or video load will hold, from files on disk; None if unknown, or automatic on
several GPUs."""
try:
from utils.hardware.hardware import get_parent_visible_gpu_ids
parent = list(get_parent_visible_gpu_ids() or ())
if gpu_ordinal is None:
physical = parent[0] if len(parent) == 1 else None
else:
physical = parent[gpu_ordinal] if 0 <= gpu_ordinal < len(parent) else None
if physical is None: # several GPUs and no pick, or a UUID/MIG mask with no numeric ids
return None
priced = {arb.DIFFUSION: _image_need, arb.VIDEO: _video_need}.get(kind, lambda *_: None)(
request, gpu_ordinal
)
if priced is None or not priced[0] or priced[0] <= 0:
return None
weights, headroom = int(priced[0]), int(priced[1])
return MediaNeed(
{physical: weights + headroom},
weights_mib = weights,
headroom_mib = headroom,
kind = _MEDIA_KIND[kind],
)
except Exception as exc: # noqa: BLE001 - an unpriced load is treated as not fitting
logger.warning("gpu_residency: could not price the %s load: %s", kind, exc)
return None
def _cached_pick(repo: str, filename: Optional[str]) -> Optional[str]:
"""The picked checkpoint file when it is already on disk (local path or either Hub cache root), else None."""
from pathlib import Path
local = Path(repo).expanduser()
if local.is_file():
return str(local)
if not filename or Path(filename).is_absolute() or ".." in Path(filename).parts:
return None
if local.is_dir():
return str(local / filename) if (local / filename).is_file() else None
from huggingface_hub import try_to_load_from_cache
from core.inference.diffusion import hub_cache_dir
for root in (hub_cache_dir(), None):
hit = try_to_load_from_cache(repo, filename, cache_dir = root)
if isinstance(hit, str) and Path(hit).is_file():
return hit
return None
def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:
"""(weights MiB, headroom MiB) for a diffusers image load, the way DiffusionBackend._plan_memory prices it."""
from core.inference.diffusion import (
DiffusionBackend as DB,
_image_variant_hint,
_single_file_fp8_upcast,
resolve_model_kind,
)
from core.inference.diffusion_auto_policy import resident_bytes_from_declared
from core.inference.diffusion_engine_router import (
has_active_loras,
mlx_load_blocker,
predict_engine,
)
from core.inference.diffusion_families import (
detect_family_for_pick,
local_pipeline_components_are_complete,
named_variant_base,
prefer_ungated_mirror,
resolve_base_repo,
)
from core.inference.diffusion_gguf_pipeline import whole_pipeline_gguf_resident_mib
from core.inference.diffusion_memory import (
estimate_gguf_resident_mib,
estimate_image_runtime_mib,
estimate_safetensors_dense_mib,
file_size_mib,
)
from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS, ENGINE_MLX
repo, name = request.model_path, request.gguf_filename
if getattr(request, "text_encoder_file", None) or getattr(request, "vae_file", None):
return None # supplied companions are priced from their headers at load time only
kind = resolve_model_kind(name, request.model_kind)
fam = detect_family_for_pick(repo, name, request.family_override)
if fam is None:
return None
engine = predict_engine(
fam,
model_kind = kind,
gpu_ordinal = gpu_ordinal,
gguf_filename = name,
engine = getattr(request, "engine", None),
with_loras = has_active_loras(getattr(request, "loras", None)),
mlx_blocker = mlx_load_blocker(
transformer_quant = getattr(request, "transformer_quant", None),
text_encoder_quant = getattr(request, "text_encoder_quant", None),
),
)
if engine == ENGINE_MLX:
return _mlx_image_need(request, fam)
if engine != ENGINE_DIFFUSERS:
return None
mib = 1024 * 1024
if kind == "pipeline":
base = repo
files: dict[str, int] = {}
dirs: list = []
def grab(d):
dirs.append(d)
sizes = DB._local_dir_weight_sizes(d, exclude_transformer = False)
for rel, size in sizes.items():
files[rel] = max(files.get(rel, 0), size)
return sizes
DB._union_over_cached_revs(prefer_ungated_mirror(repo), grab)
# A partial download would be priced as the whole pipeline.
if not any(local_pipeline_components_are_complete(d, "model_index.json") for d in dirs):
return None
weights = (resident_bytes_from_declared(repo, files.items()) or 0) // mib
else:
path = _cached_pick(repo, name)
if path is None:
return None
whole = getattr(fam, "single_file_is_pipeline", False)
if kind == "single_file":
weights = estimate_safetensors_dense_mib(
file_size_mib(path), fp8_upcast = _single_file_fp8_upcast(path, fam)
)
elif whole:
weights = whole_pipeline_gguf_resident_mib(path) or estimate_gguf_resident_mib(
file_size_mib(path)
)
else:
weights = estimate_gguf_resident_mib(file_size_mib(path))
base = (request.base_repo or "").strip()
if (
not base
): # offline only: the link a past load recorded, else the curated variant the name spells
from hub.utils.companion_assets import read_companion_links
linked = read_companion_links().get(str(repo).strip().lower()) or []
base = linked[0] if linked else (named_variant_base(fam, repo, name) or "")
base = resolve_base_repo(fam, base)
if not whole:
companions = max(
DB._companion_cache_bytes(prefer_ungated_mirror(base)),
DB._companion_cache_bytes(base),
)
if not companions:
return None # the text encoder and VAE are not downloaded yet
weights = (weights or 0) + companions // mib
hint = _image_variant_hint(fam.name, name, repo, base)
return weights, estimate_image_runtime_mib(width = None, height = None, family = hint)
def _mlx_image_need(request: Any, fam: Any) -> Optional[tuple[int, int]]:
"""(weights MiB, decode headroom MiB) of an MLX image load, from headers."""
from core.inference.diffusion_families import resolve_base_repo
from core.inference.mlx_diffusion_backend import (
decode_headroom_bytes,
fetch_base_snapshot,
get_mlx_diffusion_backend,
plan_for_load,
resolve_variant_base,
)
gguf = _cached_pick(request.model_path, request.gguf_filename)
if gguf is None:
return None
base = resolve_base_repo(fam, request.base_repo)
if not request.base_repo:
base = resolve_variant_base(
request.model_path,
fam,
None,
request.gguf_filename,
fallback = base,
local_files_only = True,
)
base_dir = fetch_base_snapshot(base, None, local_files_only = True)
headroom = decode_headroom_bytes(get_mlx_diffusion_backend()._rt())
plan = plan_for_load(gguf, base_dir, headroom_bytes = headroom)
if plan.source != "plan":
return None
mib = 1024 * 1024
# Tiled peak suffices: the renderer tiles once the pool cannot take an untiled decode.
return (plan.planned_bytes - headroom) // mib, -(-headroom // mib)
def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:
"""(weights MiB, headroom MiB) for a video load: checkpoint file sizes plus the family's companion table."""
from core.inference.diffusion import DiffusionBackend as DB
from core.inference.diffusion_component_sizes import video_components_gb
from core.inference.diffusion_memory import (
estimate_gguf_resident_mib,
estimate_safetensors_dense_mib,
estimate_video_runtime_mib,
file_size_mib,
)
from core.inference.video import _detect_load_family, resolve_video_model_kind
from core.inference.video_families import resolve_video_base_repo
from core.inference.video_minimax_h3 import is_h3_native
from core.inference.video_moe_pair import moe_partner_filename
repo, name = request.model_path, request.gguf_filename
kind = resolve_video_model_kind(name, request.model_kind)
fam = _detect_load_family(
repo, name, request.family_override, request.display_repo_id, request.model_kind
)
if fam is None or is_h3_native(fam, kind):
return None # H3 native runs its own server processes, priced elsewhere
gb = 1000.0**3 / (1024 * 1024) # MiB per GB of the family tables
if kind == "pipeline":
components = video_components_gb(fam, repo, allow_hub = False)
if components is None or not DB._cache_bytes(repo):
return None
weights = int(sum(components) * gb)
else:
names = [name] + ([moe_partner_filename(name)] if getattr(fam, "is_moe", False) else [])
paths = [_cached_pick(repo, n) if n else None for n in names]
components = video_components_gb(
fam, resolve_video_base_repo(fam, request.base_repo, repo), allow_hub = False
)
if None in paths or components is None:
return None
sizes = [file_size_mib(p) for p in paths]
price = estimate_gguf_resident_mib if kind == "gguf" else estimate_safetensors_dense_mib
if None in sizes:
return None
weights = sum(price(s) for s in sizes) + int((components[1] + components[2]) * gb)
preset = fam.resolution_presets[0]
return weights, estimate_video_runtime_mib(
width = preset[0], height = preset[1], num_frames = fam.default_num_frames
)
def _on_metal() -> bool:
try:
from utils.hardware import DeviceType, get_device
return get_device() == DeviceType.MLX
except Exception: # noqa: BLE001
return False
def shared_metal_headroom_bytes() -> Optional[int]:
"""Headroom on Apple's GPU less resident MLX chat growth; None unless priced media is on Metal."""
if not metal_media_priced():
return None
import psutil
from core.inference.llama_cpp import _apple_wired_headroom_bytes
wired = _apple_wired_headroom_bytes()
if wired is None:
return None
pool = min(wired, int(psutil.virtual_memory().available))
return max(0, pool - (_mlx_chat_unallocated_mib() or 0) * 1024 * 1024)
def metal_media_priced() -> bool:
"""Whether a media model is kept on Metal at a price: one that reserved the tiled decode, so it must tile."""
return _on_metal() and bool(media_held_mib())
def _metal_pool() -> tuple[Optional[dict[int, dict]], Optional[str]]:
"""Unified memory as GPU 0: wired headroom capped at available RAM (both exclude loaded models)."""
try:
import psutil
from core.inference.llama_cpp import _apple_wired_headroom_bytes
wired = _apple_wired_headroom_bytes()
available = int(psutil.virtual_memory().available)
except Exception as exc: # noqa: BLE001
logger.warning("gpu_residency: Metal memory probe failed: %s", exc)
return None, "probe_error"
if wired is None or available <= 0:
return None, "probe_error"
return {0: {"free_mib": min(wired, available) // (1024 * 1024), "metal": True}}, None
def _free_mib_by_gpu() -> tuple[Optional[dict[int, dict]], Optional[str]]:
"""{physical id: {free_mib, total_mib}} or (None, reason) when the host can't be priced."""
try:
from utils.hardware import DeviceType, get_device, get_visible_gpu_utilization
if get_device() == DeviceType.MLX:
return _metal_pool()
if get_device() != DeviceType.CUDA:
return None, "non_accelerator"
util = get_visible_gpu_utilization() or {}
except Exception as exc: # noqa: BLE001
logger.warning("gpu_residency: GPU probe failed: %s", exc)
return None, "probe_error"
if util.get("index_kind") != "physical":
return None, "unmapped_gpus"
rows: dict[int, dict] = {}
for device in util.get("devices") or ():
total_gb, used_gb = device.get("vram_total_gb"), device.get("vram_used_gb")
if device.get("unified_memory") or device.get("shared_memory") or not total_gb:
rows[device["index"]] = {"unified": True}
continue
if used_gb is None:
continue
total = int(total_gb * 1024)
rows[device["index"]] = {
"free_mib": max(int(total - used_gb * 1024), 0),
"total_mib": total,
}
return rows, None
def _keeps_transformer_resident(need: MediaNeed, gpu: int, free_mib: int, total_mib: int) -> bool:
"""The diffusion planner's verdict for this load at the free memory left: "fits" never means streaming weights."""
from core.inference.diffusion_memory import (
DeviceMemory,
plan_diffusion_memory,
plan_keeps_transformer_resident,
)
ordinal = _ordinal_for_physical(gpu)
plan = plan_diffusion_memory(
target = SimpleNamespace(supports_model_cpu_offload = True),
device_memory = DeviceMemory("cuda", f"cuda:{ordinal}", "discrete_vram", free_mib, total_mib),
model_dense_mib = need.weights_mib,
runtime_headroom_mib = need.headroom_mib,
# Planned as auto whatever the load asks: Balanced and Low VRAM hold less than an auto plan that stays resident.
requested_mode = None,
)
return plan_keeps_transformer_resident(plan)
def _ordinal_for_physical(gpu: int) -> int:
try:
from utils.hardware.hardware import get_parent_visible_gpu_ids
parent = list(get_parent_visible_gpu_ids() or ())
return parent.index(gpu) if gpu in parent else gpu
except Exception: # noqa: BLE001
return gpu
def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dict]:
"""Whether ``owner`` fits next to the residents already loaded. Side-effect free; default-deny with a reason."""
if not need_mib:
return False, {"reason": "estimate_unavailable"}
rows, reason = _free_mib_by_gpu()
if rows is None:
return False, {"reason": reason}
records = _media_records()
own = records.get(owner)
# A resident still loading has not allocated its weights yet either, so live free VRAM misses all of it.
loading = {name for name in records if name != owner and _media_busy(name)[0]}
info: dict[str, Any] = {"need_mib": {}, "free_mib": {}, "short_mib": {}, "reason": "fits"}
for gpu, mib in need_mib.items():
row = rows.get(gpu)
if row is None:
return False, {"reason": "gpu_unknown", "gpu": gpu}
if row.get("unified"):
return False, {"reason": "unified_memory", "gpu": gpu}
free = row["free_mib"]
# An idle resident's generation headroom is not allocated yet; it will be when it generates.
free -= sum(
int(need[gpu]) if name in loading else getattr(need, "headroom_mib", 0)
for name, need in records.items()
if name != owner and gpu in need
)
if row.get("metal"):
free -= _mlx_chat_unallocated_mib() or 0
free += getattr(need_mib, "credit_mib", 0)
if own is not None and gpu in own:
# Reloading the same owner frees what it holds now.
free += max(own[gpu] - getattr(own, "headroom_mib", 0), 0)
info["need_mib"][gpu], info["free_mib"][gpu] = int(mib), max(free, 0)
if free < mib + FIT_MARGIN_MIB:
info["short_mib"][gpu] = int(mib + FIT_MARGIN_MIB - free)
info["reason"] = "short"
if info["short_mib"]:
return False, info
if getattr(need_mib, "kind", None) == "image":
for gpu in [g for g in need_mib if not rows[g].get("metal")]:
if not _keeps_transformer_resident(
need_mib, gpu, info["free_mib"][gpu], rows[gpu]["total_mib"]
):
info["reason"] = "would_stream"
return False, info
return True, info
def chat_load_in_flight() -> bool:
"""Chat load or swap running; its GPUs are unknown before launch, so it overlaps all GPUs."""
try:
from core.inference import llama_keepwarm, model_slots
from core.inference.llama_cpp import chat_load_active
return bool(
llama_keepwarm._load_lock.locked()
or chat_load_active()
or model_slots.loading is not None
)
except Exception: # noqa: BLE001
return False
def chat_growth_unpriced() -> bool:
"""On Metal, whether a resident MLX chat model may still grow its KV / prompt cache by an unpriced amount."""
return _on_metal() and _mlx_chat_unallocated_mib() is None
def _mlx_chat_unallocated_mib() -> Optional[int]:
"""MiB resident MLX chat models priced but have not allocated yet; None when any is unpriced."""
total = 0
try:
from core.inference import model_slots, orchestrator
for orch in [
orchestrator._inference_backend,
*(s.orchestrator for s in model_slots.resident()),
]:
mib = _orch_unallocated_mib(orch)
if mib is None:
return None
total += mib
return total
except Exception: # noqa: BLE001
return None
def _orch_unallocated_mib(orch) -> Optional[int]:
total = 0
for entry in list((getattr(orch, "models", None) or {}).values()):
if isinstance(entry, dict) and entry.get("is_mlx"):
growth = entry.get("mlx_growth")
if growth is None:
return None
rows = max(1, int(orch.effective_parallel_slots), int(growth.get("slots", 1)))
priced = growth["mib"] + (rows - 1) * growth["row_mib"]
at_load, now = growth.get("footprint_mib"), mlx_worker_footprint_mib(orch)
grown = now - at_load if at_load is not None and now is not None else 0
total += max(0, priced - max(0, grown))
return total
def mlx_replaced_credit_mib(orch) -> int:
"""MiB a load into ``orch`` gets back from the MLX model it replaces (each load spawns a fresh worker)."""
try:
if not any(
isinstance(e, dict) and e.get("is_mlx")
for e in list((getattr(orch, "models", None) or {}).values())
):
return 0
return (mlx_worker_footprint_mib(orch) or 0) + (_orch_unallocated_mib(orch) or 0)
except Exception: # noqa: BLE001
return 0
def mlx_cache_bounded(entry: dict, slots: int = 1) -> bool:
"""Whether the MLX cache stops at its priced context; batched vision rows drop the window."""
if slots > 1 and entry.get("context_unbounded_when_batched"):
return False
return entry.get("context_length_enforced") is True or bool(entry.get("mlx_context_budget"))
def mlx_slots_outgrow_price(orch, n_parallel) -> bool:
"""Whether serving ``n_parallel`` rows grows a resident MLX model beside media past what it was admitted at."""
try:
from core.inference.llama_server_args import clamp_parallel_slots
entry = (getattr(orch, "models", None) or {}).get(orch.active_model_name or "") or {}
return bool(
_on_metal()
and entry.get("is_mlx")
and media_held_mib()
and clamp_parallel_slots(n_parallel) > int(orch.effective_parallel_slots)
)
except Exception: # noqa: BLE001
return False
def mlx_worker_footprint_mib(orch) -> Optional[int]:
"""MiB the MLX worker process holds now, Metal buffers included, or None when unreadable."""
proc = getattr(orch, "_proc", None)
if proc is None or proc.pid is None:
return None
from core.inference.llama_cpp import LlamaCppBackend
usage = LlamaCppBackend._darwin_rusage(proc.pid)
return None if usage is None else usage[1] // (1024 * 1024)
_CHAT_LOADING_MESSAGE = (
"Unsloth Studio is loading a chat model on this GPU. Retry once it has finished loading."
)
class ChatLoadInFlightRefusal(Exception):
"""A media load under Always met a chat load whose size isn't known yet. Retryable, unlike the Always refusal."""
def __init__(self) -> None:
super().__init__(_CHAT_LOADING_MESSAGE)
def as_http_exception(
self,
path: Optional[str] = None,
*,
openai_errors: bool = False,
):
return chat_loading_http_exception(path, openai_errors = openai_errors)
def chat_loading_http_exception(path: Optional[str] = None, *, openai_errors: bool = False):
"""The transient 409, shaped like GpuBusyForAnotherAccountError.as_http_exception."""
from fastapi import HTTPException
from utils.api_errors import error_body_for_path
retry_after = K3_RETRY_AFTER_S
if openai_errors and not (path and path.startswith("/v1/")):
path = "/v1/"
detail = (
error_body_for_path(path, _CHAT_LOADING_MESSAGE, status = 409, code = "gpu_busy", param = "model")
if path and path.startswith("/v1/")
else {"error": "gpu_busy", "message": _CHAT_LOADING_MESSAGE, "retry_after": retry_after}
)
return HTTPException(status_code = 409, detail = detail, headers = {"Retry-After": str(retry_after)})
def _gb(mib: int) -> str:
return f"{mib / 1024:.1f}"
def need_free_refusal(info: dict, slots: list[dict]) -> Exception:
"""The Always refusal for a load that doesn't fit next to the models already loaded."""
from utils.account_context import is_owner_context
short = info.get("short_mib") or {}
gpu = max(short, key = short.get) if short else None
if gpu is None or not is_owner_context():
# No numbers (masked host, unified memory, probe error) or a managed account: the plain refusal.
return arb.refusal_for(slots)
message = (
f"This model needs about {_gb(info['need_mib'][gpu])} GB of GPU memory, and {_gb(info['free_mib'][gpu])} GB "
"is free next to the models already loaded. Keep multiple models loaded is set to Always, so nothing is "
"unloaded to make room. Unload one of them, or choose When they fit."
)
first = slots[0] if slots else {}
return arb.MultiModelAlwaysRefusal(
message, kind = first.get("kind"), model = first.get("model") or None
)
def foreign_on(owner: str, account_id: Optional[str]) -> int:
"""Generations another account runs on ``owner``'s model; only these block evicting it."""
try:
if owner == arb.CHAT:
from state import active_generations
return active_generations.foreign_count(account_id)
if owner == arb.DIFFUSION:
from hub.services.models import account_access
if account_access.generation_is_foreign("diffusion"):
return 1
return 0
if owner == arb.VIDEO:
import sys
video = sys.modules.get("core.inference.video")
running = video.generation_account_in_flight() if video is not None else None
return int(running is not None and running != account_id)
except Exception as exc: # noqa: BLE001 - unknown reads as busy, so nothing is torn down under a stream
logger.warning("gpu_residency: could not count generations on %s: %s", owner, exc)
return 1
return 0
def _overlaps(gpus: Optional[Iterable[int]], target: Optional[Iterable[int]]) -> bool:
return gpus is None or target is None or bool(set(gpus) & set(target))
def _record_evicted(slots: list[dict], evicted: Optional[list], freed: list[str]) -> None:
"""Report evictions the way acquire_for does (to ``evicted`` or the route's sink) and log them."""
try:
from core.inference.api_monitor import api_monitor
except Exception: # noqa: BLE001
api_monitor = None
arb.note_evictions(slots, evicted)
for slot in slots:
model = slot.get("model") or slot.get("kind") or ""
freed.append(model)
if api_monitor is not None:
try:
api_monitor.record_lifecycle(event = "unload", model = model, reason = "evicted")
except Exception: # noqa: BLE001
pass
def _evict_resident(
name: str, acting: Optional[str], evicted: Optional[list], freed: list[str]
) -> None:
"""Evict one whole resident. Caller holds the arbiter lock."""
busy = foreign_on(name, acting)
if busy:
raise arb.GpuBusyForAnotherAccountError(name, busy)
slots = arb.resident_slots(name) or [{"kind": _MEDIA_KIND.get(name, "chat"), "model": ""}]
logger.info("gpu_arbiter: evicting %s to make room", name)
arb._EVICTORS[name]()
arb._drop_resident(name)
_record_evicted(slots, evicted, freed)
def _free_steps(
owner: str,
gpu_ids: Optional[Iterable[int]],
reason: str,
evicted: Optional[list],
*,
need_mib: Optional[dict] = None,
short_mib: int = 0,
acting: Optional[str] = None,
media_only: bool = False,
) -> list[str]:
"""Caller holds the arbiter lock; frees cheapest residents on gpu_ids only (never other GPUs),
stopping once the load fits (or after one step when unpriced); active chat model last;
media_only spares chat models; keep-in-memory does not protect."""
target = set(gpu_ids) if gpu_ids is not None else None
freed: list[str] = []
def done() -> bool:
if need_mib:
return can_load_alongside(owner, need_mib)[0]
return bool(freed)
def overlapping() -> list[str]:
return [n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)]
# 1. Residents still loading can't be sized.
for name in overlapping():
if media_only and name not in MEDIA_OWNERS:
continue
loading = (
chat_load_in_flight()
if name == arb.CHAT
else name in MEDIA_OWNERS and _media_busy(name)[0]
)
if loading:
_evict_resident(name, acting, evicted, freed)
if freed and done():
return freed
# 2. Idle H3 video servers sit outside every owner's teardown; one beside a video model on other GPUs stays.
video_gpus = arb._view().get(arb.VIDEO, ...)
if owner != arb.VIDEO and (video_gpus is ... or _overlaps(video_gpus, target)):
try:
from core.inference.video_minimax_h3 import release_h3_native_servers
if release_h3_native_servers(reason):
freed.append("video-server")
except Exception as exc: # noqa: BLE001
logger.warning("gpu_residency: could not release the idle video server: %s", exc)
if freed and done():
return freed
# 3. The idle image or video model on these GPUs.
for name in overlapping():
if name in MEDIA_OWNERS and not any(_media_busy(name)) and not foreign_on(name, acting):
_evict_resident(name, acting, evicted, freed)
if done():
return freed
if media_only:
return freed
# 4. Idle chat models kept alongside, least recently used first.
if arb.CHAT in overlapping() or owner == arb.CHAT:
from core.inference import model_slots
short = short_mib or (
max((can_load_alongside(owner, need_mib)[1].get("short_mib") or {0: 0}).values())
if need_mib
else 0
)
victims = model_slots.eviction_victims(None, short, gpu_ids)
if victims:
reports = {id(victim): _kept_slot_report(victim) for victim in victims}
# A slot a request reached since it was picked stays loaded, so it isn't reported.
dropped = model_slots.evict_these(victims) or []
_record_evicted([reports[id(victim)] for victim in dropped], evicted, freed)
if dropped and done():
return freed
# 5. The active chat model.
if owner != arb.CHAT and arb.CHAT in overlapping():
_evict_resident(arb.CHAT, acting, evicted, freed)
return freed
def _kept_slot_report(slot: Any) -> dict:
try:
from routes.inference import _kept_slot_reports
reports = _kept_slot_reports(slot)
if reports:
return reports[0]
except Exception: # noqa: BLE001
pass
from core.inference.model_ids import public_model_id
request = getattr(slot, "request", None)
raw = str(getattr(request, "model_path", "") or "")
model = public_model_id(raw) or raw
return {"kind": "chat", "model": model, "account": getattr(slot, "account", None)}
def media_fit_kwargs(
owner: str,
request: Any,
gpu_ordinal: Optional[int],
policy: Optional[str] = None,
) -> dict:
"""The acquire_for keywords for an image or video load: the policy and, when known, what it will hold."""
policy = policy or current_policy()
if policy == "off":
return {}
need = media_load_need_mib(owner, request, gpu_ordinal)
kwargs: dict = {"policy": policy}
if gpu_ordinal is not None:
# Known even when the load can't be priced, so residents on other GPUs stay out of it.
kwargs["gpu_ids"] = [gpu_ordinal]
if need:
kwargs["need_mib"] = need
return kwargs
def settle_unpriced_chat_beside_media(policy: str) -> None:
"""Media beside an unpriced chat model: When they fit evicts it, Always refuses."""
if policy not in ("fit", "always") or not _on_metal():
return
from utils.account_context import current_account_id
with arb._lock:
arb._sync_from_view()
media = [name for name in MEDIA_OWNERS if name in arb._residents]
if not media:
return
if policy == "always":
raise arb.refusal_for(
[slot for name in media for slot in arb.resident_slots(name)],
arb.KIND_LABELS[media[0]],
)
for name in media:
_evict_resident(name, current_account_id(), None, [])
def forget_media_price(owner: str, cancel: Any = None) -> None:
"""Drop ``owner``'s price; gives up on ``cancel`` since an evictor holding the lock awaits our drain."""
while not arb._lock.acquire(timeout = 0.05):
if cancel is not None and cancel.is_set():
from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
try:
arb._media_needs.pop(owner, None)
finally:
arb._lock.release()
def refuse_unless_fits(
owner: str,
*,
policy: str = "off",
gpu_ids: Optional[Iterable[int]] = None,
need_mib: Optional[dict] = None,
cancel: Any = None,
) -> None:
"""acquire_for(allow_evict=False) check for loads inside a generation; mutates nothing; gives up
on cancel because a lock-holding evictor awaits our drain."""
from utils.account_context import current_account_id
while not arb._lock.acquire(timeout = 0.05):
if cancel is not None and cancel.is_set():
from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
try:
arb._sync_from_view()
others = [name for name in arb._residents if name != owner]
if not others:
return
if policy != "off" and owner in arb._KEPT_UNDER_ALWAYS:
coreside_locked(
owner,
policy = policy,
need_mib = need_mib,
gpu_ids = gpu_ids,
allow_evict = False,
evict_first = False,
acting = current_account_id(),
evicted = None,
)
return
raise arb.GpuOwnerBusyError(others[-1])
finally:
arb._lock.release()
def free_for(
owner: str,
gpu_ids: Optional[Iterable[int]],
reason: str,
evicted: Optional[list] = None,
*,
need_mib: Optional[dict] = None,
short_mib: int = 0,
media_only: bool = False,
) -> list[str]:
"""Free residents on ``gpu_ids`` so ``owner`` fits, cheapest first. Appends {kind, model} to ``evicted``."""
from utils.account_context import current_account_id
acting = current_account_id()
with arb._lock:
arb._sync_from_view()
return _free_steps(
owner,
gpu_ids,
reason,
evicted,
need_mib = need_mib,
short_mib = short_mib,
acting = acting,
media_only = media_only,
)
def _claim_holds_anything(name: str) -> bool:
"""Whether a kept claim holds or is loading a model; unreadable counts as yes. Caller holds the lock."""
if arb.resident_slots(name):
return True
return arb._chat_occupied() if name == arb.CHAT else arb._media_occupied(name)
def coreside_locked(
owner: str,
*,
policy: str,
need_mib: Optional[dict],
gpu_ids: Optional[Iterable[int]],
allow_evict: bool,
evict_first: bool,
acting: Optional[str],
evicted: Optional[list],
) -> None:
"""Makes room for owner under When they fit / Always; under When they fit, unpriceable residents
are evicted first."""
target = (
arb.physical_gpu_ids(gpu_ids)
if gpu_ids is not None
else (frozenset(need_mib) if need_mib else None)
)
overlap = [
name for name, gpus in arb._view().items() if name != owner and _overlaps(gpus, target)
]
for name in [n for n in overlap if n not in arb._KEPT_UNDER_ALWAYS]:
# A Decision API server is evicted as before.
if not allow_evict:
raise arb.GpuOwnerBusyError(name)
_evict_resident(name, acting, evicted, [])
kept = [n for n in overlap if n in arb._KEPT_UNDER_ALWAYS]
if policy == "always":
# A claim whose load failed holds nothing, so Always evicts it rather than refusing (gpu_arbiter.acquire_for).
for name in [n for n in kept if not _claim_holds_anything(n)]:
if not allow_evict:
raise arb.GpuOwnerBusyError(name)
_evict_resident(name, acting, evicted, [])
kept.remove(name)
def evict_or_refuse(names: list[str]) -> None:
if not names:
return
if policy == "always":
raise arb.refusal_for(
[slot for n in names for slot in arb.resident_slots(n)], arb.KIND_LABELS[names[0]]
)
if not allow_evict:
raise arb.GpuOwnerBusyError(names[-1])
for name in names:
_evict_resident(name, acting, evicted, [])
kept.remove(name)
if not kept:
return
if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_load_in_flight():
# Its size isn't known until it launches.
if policy == "always":
raise ChatLoadInFlightRefusal()
evict_or_refuse([arb.CHAT])
if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_growth_unpriced():
evict_or_refuse([arb.CHAT])
if owner == arb.CHAT and not need_mib:
if evict_first:
# No estimate before launch (a non-GGUF load, or a load with no retry to make room).
return evict_or_refuse(list(kept))
records = _media_records()
evict_or_refuse([n for n in kept if n not in records or _media_busy(n)[0]])
if kept and _free_mib_by_gpu()[0] is None:
evict_or_refuse(list(kept))
return
if need_mib is None or evict_first:
return evict_or_refuse(list(kept))
if isinstance(need_mib, ChatNeed):
evict_or_refuse([n for n in kept if n in MEDIA_OWNERS and n not in _media_records()])
if not kept:
return
fits, info = can_load_alongside(owner, need_mib)
if fits:
return
if policy == "always":
raise need_free_refusal(info, [slot for n in kept for slot in arb.resident_slots(n)])
if not allow_evict:
raise arb.GpuOwnerBusyError(kept[-1])
_free_steps(
owner,
target,
f"GPU memory for {owner}",
evicted,
need_mib = need_mib,
acting = acting,
media_only = owner == arb.CHAT and _on_metal(),
)
if not can_load_alongside(owner, need_mib)[0]:
# Still short: unload the rest on these GPUs, as Off does, and let the planner offload.
for name in [
n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)
]:
_evict_resident(name, acting, evicted, [])