Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

42.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Prices image/video residents against live free VRAM; anything unpriceable counts as not fitting"""

from __future__ import annotations

from types import SimpleNamespace
from typing import Any, Iterable, Optional

from core.inference import gpu_arbiter as arb
from loggers import get_logger

logger = get_logger(__name__)

# fits iff free_mib >= need_mib + FIT_MARGIN_MIB on every GPU the load uses. Live free VRAM already excludes resident
# weights, so the margin only covers estimate error and allocator slack. An image load must also keep its
# transformer on the GPU (the diffusion planner adds its own reserve on top, so this is not doubled there).
FIT_MARGIN_MIB = 1024
# A chat load in flight finishes on its own; the media auto-switch uses the same wait.
K3_RETRY_AFTER_S = 15

MEDIA_OWNERS = (arb.DIFFUSION, arb.VIDEO)
_MEDIA_KIND = {arb.DIFFUSION: "image", arb.VIDEO: "video"}


class MediaNeed(dict):
    """{physical GPU id: weights + generation headroom MiB}, plus the split the planner rule needs."""

    def __init__(self, per_gpu: dict, *, weights_mib: int, headroom_mib: int, kind: str):
        super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})
        self.weights_mib = int(weights_mib)
        self.headroom_mib = int(headroom_mib)
        self.kind = kind


class ChatNeed(dict):
    """Chat load {GPU: MiB}, priced against free memory plus ``credit_mib`` released by the model it replaces."""

    def __init__(
        self,
        per_gpu: dict,
        *,
        credit_mib: int = 0,
    ):
        super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})
        self.credit_mib = int(credit_mib)


def current_policy() -> str:
    """Keep multiple models loaded, read as the owner. Never call this under the arbiter lock (it reads a database)."""
    try:
        from utils.multi_model_settings import get_multi_model_mode
        return get_multi_model_mode()
    except Exception as exc:  # noqa: BLE001 - an unreadable setting keeps today's behaviour
        logger.warning("gpu_residency: could not read Keep multiple models loaded: %s", exc)
        return "off"


def _media_records() -> dict[str, dict]:
    """Admitted needs of the media residents still on the GPU. Lock free: acquire_for calls this holding the lock."""
    records = dict(getattr(arb, "_media_needs", {}) or {})
    if not records:
        return {}
    resident = set(arb._view()) if hasattr(arb, "_view") else {arb._owner}
    return {
        owner: need
        for owner, need in records.items()
        if owner in resident and _loaded_or_loading(owner)
    }


def _loaded_or_loading(owner: str) -> bool:
    """False once a media load failed and left nothing loaded, so its price stops counting. Unknown reads as True."""
    try:
        engine = _media_engine(owner)
        if engine is None:
            return True
        return getattr(engine, "_state", None) is not None or bool(engine.loading_repo_ids())
    except Exception:  # noqa: BLE001
        return True


def media_held_mib() -> dict[int, int]:
    """{physical GPU id: MiB} held by resident image and video models (weights + headroom). Never raises."""
    try:
        held: dict[int, int] = {}
        for need in _media_records().values():
            for gpu, mib in need.items():
                held[gpu] = held.get(gpu, 0) + int(mib)
        return held
    except Exception as exc:  # noqa: BLE001
        logger.warning("gpu_residency: could not price the resident media models: %s", exc)
        return {}


def media_owner_held_mib(owner: str) -> Optional[int]:
    """MiB one resident media owner holds across its GPUs: its admitted need less the generation headroom
    it is not using while idle (as media_unallocated_mib counts it). None when unpriced or loading. Never raises."""
    try:
        need = _media_records().get(owner)
        if not need:
            return None
        loading, generating = _media_busy(owner)
        if loading:
            return None
        headroom = 0 if generating else int(getattr(need, "headroom_mib", 0))
        total = sum(max(0, int(mib) - headroom) for mib in need.values())
        return total or None
    except Exception:  # noqa: BLE001
        return None


def media_unallocated_mib() -> dict[int, int]:
    """{physical GPU id: MiB} of the media price live free VRAM still shows as free: an idle resident's generation
    headroom, or all of a resident still loading. Never raises."""
    try:
        held: dict[int, int] = {}
        for owner, need in _media_records().items():
            loading = _media_busy(owner)[0]
            for gpu, mib in need.items():
                unallocated = int(mib) if loading else int(getattr(need, "headroom_mib", 0))
                held[gpu] = held.get(gpu, 0) + unallocated
        return held
    except Exception as exc:  # noqa: BLE001
        logger.warning("gpu_residency: could not price the resident media models: %s", exc)
        return {}


def _media_engine(owner: str) -> Any:
    from core.inference.media_keepwarm import engine_if_imported
    return engine_if_imported(owner)


def _media_busy(owner: str) -> tuple[bool, bool]:
    """(loading, generating) for a media owner; unknown reads as busy, so it is never freed as idle."""
    try:
        engine = _media_engine(owner)
        if engine is None:
            return False, False
        loading = bool(engine.loading_repo_ids())
        progress = engine.generate_progress() or {}
        from core.inference.media_keepwarm import _TRACKERS

        tracker = _TRACKERS.get(owner)
        outstanding = bool(tracker is not None and tracker.outstanding())
        return loading, bool(progress.get("active")) or outstanding
    except Exception:  # noqa: BLE001
        return True, True


def summarize_resident_media() -> dict[str, Any]:
    """Report the resident image and video models. Never raises."""
    summary: dict[str, Any] = {}
    records = {}
    try:
        records = _media_records()
        resident = arb.residents()
    except Exception:  # noqa: BLE001
        resident = {}
    for owner in MEDIA_OWNERS:
        loading, generating = _media_busy(owner) if owner in resident else (False, False)
        summary[owner] = {
            "resident": owner in resident,
            "gpus": sorted(resident.get(owner) or ()),
            "loading": loading,
            "generating": generating,
            "priced": owner in records,
        }
    summary["loading"] = any(summary[owner]["loading"] for owner in MEDIA_OWNERS)
    summary["held_mib"] = media_held_mib()
    return summary


def media_load_need_mib(kind: str, request: Any, gpu_ordinal: Optional[int]) -> Optional[MediaNeed]:
    """MiB a new image or video load will hold, from files on disk; None if unknown, or automatic on
    several GPUs."""
    try:
        from utils.hardware.hardware import get_parent_visible_gpu_ids

        parent = list(get_parent_visible_gpu_ids() or ())
        if gpu_ordinal is None:
            physical = parent[0] if len(parent) == 1 else None
        else:
            physical = parent[gpu_ordinal] if 0 <= gpu_ordinal < len(parent) else None
        if physical is None:  # several GPUs and no pick, or a UUID/MIG mask with no numeric ids
            return None
        priced = {arb.DIFFUSION: _image_need, arb.VIDEO: _video_need}.get(kind, lambda *_: None)(
            request, gpu_ordinal
        )
        if priced is None or not priced[0] or priced[0] <= 0:
            return None
        weights, headroom = int(priced[0]), int(priced[1])
        return MediaNeed(
            {physical: weights + headroom},
            weights_mib = weights,
            headroom_mib = headroom,
            kind = _MEDIA_KIND[kind],
        )
    except Exception as exc:  # noqa: BLE001 - an unpriced load is treated as not fitting
        logger.warning("gpu_residency: could not price the %s load: %s", kind, exc)
        return None


def _cached_pick(repo: str, filename: Optional[str]) -> Optional[str]:
    """The picked checkpoint file when it is already on disk (local path or either Hub cache root), else None."""
    from pathlib import Path

    local = Path(repo).expanduser()
    if local.is_file():
        return str(local)
    if not filename or Path(filename).is_absolute() or ".." in Path(filename).parts:
        return None
    if local.is_dir():
        return str(local / filename) if (local / filename).is_file() else None
    from huggingface_hub import try_to_load_from_cache
    from core.inference.diffusion import hub_cache_dir

    for root in (hub_cache_dir(), None):
        hit = try_to_load_from_cache(repo, filename, cache_dir = root)
        if isinstance(hit, str) and Path(hit).is_file():
            return hit
    return None


def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:
    """(weights MiB, headroom MiB) for a diffusers image load, the way DiffusionBackend._plan_memory prices it."""
    from core.inference.diffusion import (
        DiffusionBackend as DB,
        _image_variant_hint,
        _single_file_fp8_upcast,
        resolve_model_kind,
    )
    from core.inference.diffusion_auto_policy import resident_bytes_from_declared
    from core.inference.diffusion_engine_router import (
        has_active_loras,
        mlx_load_blocker,
        predict_engine,
    )
    from core.inference.diffusion_families import (
        detect_family_for_pick,
        local_pipeline_components_are_complete,
        named_variant_base,
        prefer_ungated_mirror,
        resolve_base_repo,
    )
    from core.inference.diffusion_gguf_pipeline import whole_pipeline_gguf_resident_mib
    from core.inference.diffusion_memory import (
        estimate_gguf_resident_mib,
        estimate_image_runtime_mib,
        estimate_safetensors_dense_mib,
        file_size_mib,
    )
    from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS, ENGINE_MLX

    repo, name = request.model_path, request.gguf_filename
    if getattr(request, "text_encoder_file", None) or getattr(request, "vae_file", None):
        return None  # supplied companions are priced from their headers at load time only
    kind = resolve_model_kind(name, request.model_kind)
    fam = detect_family_for_pick(repo, name, request.family_override)
    if fam is None:
        return None
    engine = predict_engine(
        fam,
        model_kind = kind,
        gpu_ordinal = gpu_ordinal,
        gguf_filename = name,
        engine = getattr(request, "engine", None),
        with_loras = has_active_loras(getattr(request, "loras", None)),
        mlx_blocker = mlx_load_blocker(
            transformer_quant = getattr(request, "transformer_quant", None),
            text_encoder_quant = getattr(request, "text_encoder_quant", None),
        ),
    )
    if engine == ENGINE_MLX:
        return _mlx_image_need(request, fam)
    if engine != ENGINE_DIFFUSERS:
        return None
    mib = 1024 * 1024
    if kind == "pipeline":
        base = repo
        files: dict[str, int] = {}
        dirs: list = []

        def grab(d):
            dirs.append(d)
            sizes = DB._local_dir_weight_sizes(d, exclude_transformer = False)
            for rel, size in sizes.items():
                files[rel] = max(files.get(rel, 0), size)
            return sizes

        DB._union_over_cached_revs(prefer_ungated_mirror(repo), grab)
        # A partial download would be priced as the whole pipeline.
        if not any(local_pipeline_components_are_complete(d, "model_index.json") for d in dirs):
            return None
        weights = (resident_bytes_from_declared(repo, files.items()) or 0) // mib
    else:
        path = _cached_pick(repo, name)
        if path is None:
            return None
        whole = getattr(fam, "single_file_is_pipeline", False)
        if kind == "single_file":
            weights = estimate_safetensors_dense_mib(
                file_size_mib(path), fp8_upcast = _single_file_fp8_upcast(path, fam)
            )
        elif whole:
            weights = whole_pipeline_gguf_resident_mib(path) or estimate_gguf_resident_mib(
                file_size_mib(path)
            )
        else:
            weights = estimate_gguf_resident_mib(file_size_mib(path))
        base = (request.base_repo or "").strip()
        if (
            not base
        ):  # offline only: the link a past load recorded, else the curated variant the name spells
            from hub.utils.companion_assets import read_companion_links
            linked = read_companion_links().get(str(repo).strip().lower()) or []
            base = linked[0] if linked else (named_variant_base(fam, repo, name) or "")
        base = resolve_base_repo(fam, base)
        if not whole:
            companions = max(
                DB._companion_cache_bytes(prefer_ungated_mirror(base)),
                DB._companion_cache_bytes(base),
            )
            if not companions:
                return None  # the text encoder and VAE are not downloaded yet
            weights = (weights or 0) + companions // mib
    hint = _image_variant_hint(fam.name, name, repo, base)
    return weights, estimate_image_runtime_mib(width = None, height = None, family = hint)


def _mlx_image_need(request: Any, fam: Any) -> Optional[tuple[int, int]]:
    """(weights MiB, decode headroom MiB) of an MLX image load, from headers."""
    from core.inference.diffusion_families import resolve_base_repo
    from core.inference.mlx_diffusion_backend import (
        decode_headroom_bytes,
        fetch_base_snapshot,
        get_mlx_diffusion_backend,
        plan_for_load,
        resolve_variant_base,
    )

    gguf = _cached_pick(request.model_path, request.gguf_filename)
    if gguf is None:
        return None
    base = resolve_base_repo(fam, request.base_repo)
    if not request.base_repo:
        base = resolve_variant_base(
            request.model_path,
            fam,
            None,
            request.gguf_filename,
            fallback = base,
            local_files_only = True,
        )
    base_dir = fetch_base_snapshot(base, None, local_files_only = True)
    headroom = decode_headroom_bytes(get_mlx_diffusion_backend()._rt())
    plan = plan_for_load(gguf, base_dir, headroom_bytes = headroom)
    if plan.source != "plan":
        return None
    mib = 1024 * 1024
    # Tiled peak suffices: the renderer tiles once the pool cannot take an untiled decode.
    return (plan.planned_bytes - headroom) // mib, -(-headroom // mib)


def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:
    """(weights MiB, headroom MiB) for a video load: checkpoint file sizes plus the family's companion table."""
    from core.inference.diffusion import DiffusionBackend as DB
    from core.inference.diffusion_component_sizes import video_components_gb
    from core.inference.diffusion_memory import (
        estimate_gguf_resident_mib,
        estimate_safetensors_dense_mib,
        estimate_video_runtime_mib,
        file_size_mib,
    )
    from core.inference.video import _detect_load_family, resolve_video_model_kind
    from core.inference.video_families import resolve_video_base_repo
    from core.inference.video_minimax_h3 import is_h3_native
    from core.inference.video_moe_pair import moe_partner_filename

    repo, name = request.model_path, request.gguf_filename
    kind = resolve_video_model_kind(name, request.model_kind)
    fam = _detect_load_family(
        repo, name, request.family_override, request.display_repo_id, request.model_kind
    )
    if fam is None or is_h3_native(fam, kind):
        return None  # H3 native runs its own server processes, priced elsewhere
    gb = 1000.0**3 / (1024 * 1024)  # MiB per GB of the family tables
    if kind == "pipeline":
        components = video_components_gb(fam, repo, allow_hub = False)
        if components is None or not DB._cache_bytes(repo):
            return None
        weights = int(sum(components) * gb)
    else:
        names = [name] + ([moe_partner_filename(name)] if getattr(fam, "is_moe", False) else [])
        paths = [_cached_pick(repo, n) if n else None for n in names]
        components = video_components_gb(
            fam, resolve_video_base_repo(fam, request.base_repo, repo), allow_hub = False
        )
        if None in paths or components is None:
            return None
        sizes = [file_size_mib(p) for p in paths]
        price = estimate_gguf_resident_mib if kind == "gguf" else estimate_safetensors_dense_mib
        if None in sizes:
            return None
        weights = sum(price(s) for s in sizes) + int((components[1] + components[2]) * gb)
    preset = fam.resolution_presets[0]
    return weights, estimate_video_runtime_mib(
        width = preset[0], height = preset[1], num_frames = fam.default_num_frames
    )


def _on_metal() -> bool:
    try:
        from utils.hardware import DeviceType, get_device
        return get_device() == DeviceType.MLX
    except Exception:  # noqa: BLE001
        return False


def shared_metal_headroom_bytes() -> Optional[int]:
    """Headroom on Apple's GPU less resident MLX chat growth; None unless priced media is on Metal."""
    if not metal_media_priced():
        return None
    import psutil
    from core.inference.llama_cpp import _apple_wired_headroom_bytes

    wired = _apple_wired_headroom_bytes()
    if wired is None:
        return None
    pool = min(wired, int(psutil.virtual_memory().available))
    return max(0, pool - (_mlx_chat_unallocated_mib() or 0) * 1024 * 1024)


def metal_media_priced() -> bool:
    """Whether a media model is kept on Metal at a price: one that reserved the tiled decode, so it must tile."""
    return _on_metal() and bool(media_held_mib())


def _metal_pool() -> tuple[Optional[dict[int, dict]], Optional[str]]:
    """Unified memory as GPU 0: wired headroom capped at available RAM (both exclude loaded models)."""
    try:
        import psutil
        from core.inference.llama_cpp import _apple_wired_headroom_bytes

        wired = _apple_wired_headroom_bytes()
        available = int(psutil.virtual_memory().available)
    except Exception as exc:  # noqa: BLE001
        logger.warning("gpu_residency: Metal memory probe failed: %s", exc)
        return None, "probe_error"
    if wired is None or available <= 0:
        return None, "probe_error"
    return {0: {"free_mib": min(wired, available) // (1024 * 1024), "metal": True}}, None


def _free_mib_by_gpu() -> tuple[Optional[dict[int, dict]], Optional[str]]:
    """{physical id: {free_mib, total_mib}} or (None, reason) when the host can't be priced."""
    try:
        from utils.hardware import DeviceType, get_device, get_visible_gpu_utilization

        if get_device() == DeviceType.MLX:
            return _metal_pool()
        if get_device() != DeviceType.CUDA:
            return None, "non_accelerator"
        util = get_visible_gpu_utilization() or {}
    except Exception as exc:  # noqa: BLE001
        logger.warning("gpu_residency: GPU probe failed: %s", exc)
        return None, "probe_error"
    if util.get("index_kind") != "physical":
        return None, "unmapped_gpus"
    rows: dict[int, dict] = {}
    for device in util.get("devices") or ():
        total_gb, used_gb = device.get("vram_total_gb"), device.get("vram_used_gb")
        if device.get("unified_memory") or device.get("shared_memory") or not total_gb:
            rows[device["index"]] = {"unified": True}
            continue
        if used_gb is None:
            continue
        total = int(total_gb * 1024)
        rows[device["index"]] = {
            "free_mib": max(int(total - used_gb * 1024), 0),
            "total_mib": total,
        }
    return rows, None


def _keeps_transformer_resident(need: MediaNeed, gpu: int, free_mib: int, total_mib: int) -> bool:
    """The diffusion planner's verdict for this load at the free memory left: "fits" never means streaming weights."""
    from core.inference.diffusion_memory import (
        DeviceMemory,
        plan_diffusion_memory,
        plan_keeps_transformer_resident,
    )

    ordinal = _ordinal_for_physical(gpu)
    plan = plan_diffusion_memory(
        target = SimpleNamespace(supports_model_cpu_offload = True),
        device_memory = DeviceMemory("cuda", f"cuda:{ordinal}", "discrete_vram", free_mib, total_mib),
        model_dense_mib = need.weights_mib,
        runtime_headroom_mib = need.headroom_mib,
        # Planned as auto whatever the load asks: Balanced and Low VRAM hold less than an auto plan that stays resident.
        requested_mode = None,
    )
    return plan_keeps_transformer_resident(plan)


def _ordinal_for_physical(gpu: int) -> int:
    try:
        from utils.hardware.hardware import get_parent_visible_gpu_ids
        parent = list(get_parent_visible_gpu_ids() or ())
        return parent.index(gpu) if gpu in parent else gpu
    except Exception:  # noqa: BLE001
        return gpu


def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dict]:
    """Whether ``owner`` fits next to the residents already loaded. Side-effect free; default-deny with a reason."""
    if not need_mib:
        return False, {"reason": "estimate_unavailable"}
    rows, reason = _free_mib_by_gpu()
    if rows is None:
        return False, {"reason": reason}
    records = _media_records()
    own = records.get(owner)
    # A resident still loading has not allocated its weights yet either, so live free VRAM misses all of it.
    loading = {name for name in records if name != owner and _media_busy(name)[0]}
    info: dict[str, Any] = {"need_mib": {}, "free_mib": {}, "short_mib": {}, "reason": "fits"}
    for gpu, mib in need_mib.items():
        row = rows.get(gpu)
        if row is None:
            return False, {"reason": "gpu_unknown", "gpu": gpu}
        if row.get("unified"):
            return False, {"reason": "unified_memory", "gpu": gpu}
        free = row["free_mib"]
        # An idle resident's generation headroom is not allocated yet; it will be when it generates.
        free -= sum(
            int(need[gpu]) if name in loading else getattr(need, "headroom_mib", 0)
            for name, need in records.items()
            if name != owner and gpu in need
        )
        if row.get("metal"):
            free -= _mlx_chat_unallocated_mib() or 0
        free += getattr(need_mib, "credit_mib", 0)
        if own is not None and gpu in own:
            # Reloading the same owner frees what it holds now.
            free += max(own[gpu] - getattr(own, "headroom_mib", 0), 0)
        info["need_mib"][gpu], info["free_mib"][gpu] = int(mib), max(free, 0)
        if free < mib + FIT_MARGIN_MIB:
            info["short_mib"][gpu] = int(mib + FIT_MARGIN_MIB - free)
            info["reason"] = "short"
    if info["short_mib"]:
        return False, info
    if getattr(need_mib, "kind", None) == "image":
        for gpu in [g for g in need_mib if not rows[g].get("metal")]:
            if not _keeps_transformer_resident(
                need_mib, gpu, info["free_mib"][gpu], rows[gpu]["total_mib"]
            ):
                info["reason"] = "would_stream"
                return False, info
    return True, info


def chat_load_in_flight() -> bool:
    """Chat load or swap running; its GPUs are unknown before launch, so it overlaps all GPUs."""
    try:
        from core.inference import llama_keepwarm, model_slots
        from core.inference.llama_cpp import chat_load_active
        return bool(
            llama_keepwarm._load_lock.locked()
            or chat_load_active()
            or model_slots.loading is not None
        )
    except Exception:  # noqa: BLE001
        return False


def chat_growth_unpriced() -> bool:
    """On Metal, whether a resident MLX chat model may still grow its KV / prompt cache by an unpriced amount."""
    return _on_metal() and _mlx_chat_unallocated_mib() is None


def _mlx_chat_unallocated_mib() -> Optional[int]:
    """MiB resident MLX chat models priced but have not allocated yet; None when any is unpriced."""
    total = 0
    try:
        from core.inference import model_slots, orchestrator
        for orch in [
            orchestrator._inference_backend,
            *(s.orchestrator for s in model_slots.resident()),
        ]:
            mib = _orch_unallocated_mib(orch)
            if mib is None:
                return None
            total += mib
        return total
    except Exception:  # noqa: BLE001
        return None


def _orch_unallocated_mib(orch) -> Optional[int]:
    total = 0
    for entry in list((getattr(orch, "models", None) or {}).values()):
        if isinstance(entry, dict) and entry.get("is_mlx"):
            growth = entry.get("mlx_growth")
            if growth is None:
                return None
            rows = max(1, int(orch.effective_parallel_slots), int(growth.get("slots", 1)))
            priced = growth["mib"] + (rows - 1) * growth["row_mib"]
            at_load, now = growth.get("footprint_mib"), mlx_worker_footprint_mib(orch)
            grown = now - at_load if at_load is not None and now is not None else 0
            total += max(0, priced - max(0, grown))
    return total


def mlx_replaced_credit_mib(orch) -> int:
    """MiB a load into ``orch`` gets back from the MLX model it replaces (each load spawns a fresh worker)."""
    try:
        if not any(
            isinstance(e, dict) and e.get("is_mlx")
            for e in list((getattr(orch, "models", None) or {}).values())
        ):
            return 0
        return (mlx_worker_footprint_mib(orch) or 0) + (_orch_unallocated_mib(orch) or 0)
    except Exception:  # noqa: BLE001
        return 0


def mlx_cache_bounded(entry: dict, slots: int = 1) -> bool:
    """Whether the MLX cache stops at its priced context; batched vision rows drop the window."""
    if slots > 1 and entry.get("context_unbounded_when_batched"):
        return False
    return entry.get("context_length_enforced") is True or bool(entry.get("mlx_context_budget"))


def mlx_slots_outgrow_price(orch, n_parallel) -> bool:
    """Whether serving ``n_parallel`` rows grows a resident MLX model beside media past what it was admitted at."""
    try:
        from core.inference.llama_server_args import clamp_parallel_slots
        entry = (getattr(orch, "models", None) or {}).get(orch.active_model_name or "") or {}
        return bool(
            _on_metal()
            and entry.get("is_mlx")
            and media_held_mib()
            and clamp_parallel_slots(n_parallel) > int(orch.effective_parallel_slots)
        )
    except Exception:  # noqa: BLE001
        return False


def mlx_worker_footprint_mib(orch) -> Optional[int]:
    """MiB the MLX worker process holds now, Metal buffers included, or None when unreadable."""
    proc = getattr(orch, "_proc", None)
    if proc is None or proc.pid is None:
        return None
    from core.inference.llama_cpp import LlamaCppBackend

    usage = LlamaCppBackend._darwin_rusage(proc.pid)
    return None if usage is None else usage[1] // (1024 * 1024)


_CHAT_LOADING_MESSAGE = (
    "Unsloth Studio is loading a chat model on this GPU. Retry once it has finished loading."
)


class ChatLoadInFlightRefusal(Exception):
    """A media load under Always met a chat load whose size isn't known yet. Retryable, unlike the Always refusal."""

    def __init__(self) -> None:
        super().__init__(_CHAT_LOADING_MESSAGE)

    def as_http_exception(
        self,
        path: Optional[str] = None,
        *,
        openai_errors: bool = False,
    ):
        return chat_loading_http_exception(path, openai_errors = openai_errors)


def chat_loading_http_exception(path: Optional[str] = None, *, openai_errors: bool = False):
    """The transient 409, shaped like GpuBusyForAnotherAccountError.as_http_exception."""
    from fastapi import HTTPException
    from utils.api_errors import error_body_for_path

    retry_after = K3_RETRY_AFTER_S
    if openai_errors and not (path and path.startswith("/v1/")):
        path = "/v1/"
    detail = (
        error_body_for_path(path, _CHAT_LOADING_MESSAGE, status = 409, code = "gpu_busy", param = "model")
        if path and path.startswith("/v1/")
        else {"error": "gpu_busy", "message": _CHAT_LOADING_MESSAGE, "retry_after": retry_after}
    )
    return HTTPException(status_code = 409, detail = detail, headers = {"Retry-After": str(retry_after)})


def _gb(mib: int) -> str:
    return f"{mib / 1024:.1f}"


def need_free_refusal(info: dict, slots: list[dict]) -> Exception:
    """The Always refusal for a load that doesn't fit next to the models already loaded."""
    from utils.account_context import is_owner_context

    short = info.get("short_mib") or {}
    gpu = max(short, key = short.get) if short else None
    if gpu is None or not is_owner_context():
        # No numbers (masked host, unified memory, probe error) or a managed account: the plain refusal.
        return arb.refusal_for(slots)
    message = (
        f"This model needs about {_gb(info['need_mib'][gpu])} GB of GPU memory, and {_gb(info['free_mib'][gpu])} GB "
        "is free next to the models already loaded. Keep multiple models loaded is set to Always, so nothing is "
        "unloaded to make room. Unload one of them, or choose When they fit."
    )
    first = slots[0] if slots else {}
    return arb.MultiModelAlwaysRefusal(
        message, kind = first.get("kind"), model = first.get("model") or None
    )


def foreign_on(owner: str, account_id: Optional[str]) -> int:
    """Generations another account runs on ``owner``'s model; only these block evicting it."""
    try:
        if owner == arb.CHAT:
            from state import active_generations
            return active_generations.foreign_count(account_id)
        if owner == arb.DIFFUSION:
            from hub.services.models import account_access
            if account_access.generation_is_foreign("diffusion"):
                return 1
            return 0
        if owner == arb.VIDEO:
            import sys

            video = sys.modules.get("core.inference.video")
            running = video.generation_account_in_flight() if video is not None else None
            return int(running is not None and running != account_id)
    except Exception as exc:  # noqa: BLE001 - unknown reads as busy, so nothing is torn down under a stream
        logger.warning("gpu_residency: could not count generations on %s: %s", owner, exc)
        return 1
    return 0


def _overlaps(gpus: Optional[Iterable[int]], target: Optional[Iterable[int]]) -> bool:
    return gpus is None or target is None or bool(set(gpus) & set(target))


def _record_evicted(slots: list[dict], evicted: Optional[list], freed: list[str]) -> None:
    """Report evictions the way acquire_for does (to ``evicted`` or the route's sink) and log them."""
    try:
        from core.inference.api_monitor import api_monitor
    except Exception:  # noqa: BLE001
        api_monitor = None
    arb.note_evictions(slots, evicted)
    for slot in slots:
        model = slot.get("model") or slot.get("kind") or ""
        freed.append(model)
        if api_monitor is not None:
            try:
                api_monitor.record_lifecycle(event = "unload", model = model, reason = "evicted")
            except Exception:  # noqa: BLE001
                pass


def _evict_resident(
    name: str, acting: Optional[str], evicted: Optional[list], freed: list[str]
) -> None:
    """Evict one whole resident. Caller holds the arbiter lock."""
    busy = foreign_on(name, acting)
    if busy:
        raise arb.GpuBusyForAnotherAccountError(name, busy)
    slots = arb.resident_slots(name) or [{"kind": _MEDIA_KIND.get(name, "chat"), "model": ""}]
    logger.info("gpu_arbiter: evicting %s to make room", name)
    arb._EVICTORS[name]()
    arb._drop_resident(name)
    _record_evicted(slots, evicted, freed)


def _free_steps(
    owner: str,
    gpu_ids: Optional[Iterable[int]],
    reason: str,
    evicted: Optional[list],
    *,
    need_mib: Optional[dict] = None,
    short_mib: int = 0,
    acting: Optional[str] = None,
    media_only: bool = False,
) -> list[str]:
    """Caller holds the arbiter lock; frees cheapest residents on gpu_ids only (never other GPUs),
    stopping once the load fits (or after one step when unpriced); active chat model last;
    media_only spares chat models; keep-in-memory does not protect."""
    target = set(gpu_ids) if gpu_ids is not None else None
    freed: list[str] = []

    def done() -> bool:
        if need_mib:
            return can_load_alongside(owner, need_mib)[0]
        return bool(freed)

    def overlapping() -> list[str]:
        return [n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)]

    # 1. Residents still loading can't be sized.
    for name in overlapping():
        if media_only and name not in MEDIA_OWNERS:
            continue
        loading = (
            chat_load_in_flight()
            if name == arb.CHAT
            else name in MEDIA_OWNERS and _media_busy(name)[0]
        )
        if loading:
            _evict_resident(name, acting, evicted, freed)
    if freed and done():
        return freed
    # 2. Idle H3 video servers sit outside every owner's teardown; one beside a video model on other GPUs stays.
    video_gpus = arb._view().get(arb.VIDEO, ...)
    if owner != arb.VIDEO and (video_gpus is ... or _overlaps(video_gpus, target)):
        try:
            from core.inference.video_minimax_h3 import release_h3_native_servers
            if release_h3_native_servers(reason):
                freed.append("video-server")
        except Exception as exc:  # noqa: BLE001
            logger.warning("gpu_residency: could not release the idle video server: %s", exc)
        if freed and done():
            return freed
    # 3. The idle image or video model on these GPUs.
    for name in overlapping():
        if name in MEDIA_OWNERS and not any(_media_busy(name)) and not foreign_on(name, acting):
            _evict_resident(name, acting, evicted, freed)
            if done():
                return freed
    if media_only:
        return freed
    # 4. Idle chat models kept alongside, least recently used first.
    if arb.CHAT in overlapping() or owner == arb.CHAT:
        from core.inference import model_slots

        short = short_mib or (
            max((can_load_alongside(owner, need_mib)[1].get("short_mib") or {0: 0}).values())
            if need_mib
            else 0
        )
        victims = model_slots.eviction_victims(None, short, gpu_ids)
        if victims:
            reports = {id(victim): _kept_slot_report(victim) for victim in victims}
            # A slot a request reached since it was picked stays loaded, so it isn't reported.
            dropped = model_slots.evict_these(victims) or []
            _record_evicted([reports[id(victim)] for victim in dropped], evicted, freed)
            if dropped and done():
                return freed
    # 5. The active chat model.
    if owner != arb.CHAT and arb.CHAT in overlapping():
        _evict_resident(arb.CHAT, acting, evicted, freed)
    return freed


def _kept_slot_report(slot: Any) -> dict:
    try:
        from routes.inference import _kept_slot_reports
        reports = _kept_slot_reports(slot)
        if reports:
            return reports[0]
    except Exception:  # noqa: BLE001
        pass
    from core.inference.model_ids import public_model_id

    request = getattr(slot, "request", None)
    raw = str(getattr(request, "model_path", "") or "")
    model = public_model_id(raw) or raw
    return {"kind": "chat", "model": model, "account": getattr(slot, "account", None)}


def media_fit_kwargs(
    owner: str,
    request: Any,
    gpu_ordinal: Optional[int],
    policy: Optional[str] = None,
) -> dict:
    """The acquire_for keywords for an image or video load: the policy and, when known, what it will hold."""
    policy = policy or current_policy()
    if policy == "off":
        return {}
    need = media_load_need_mib(owner, request, gpu_ordinal)
    kwargs: dict = {"policy": policy}
    if gpu_ordinal is not None:
        # Known even when the load can't be priced, so residents on other GPUs stay out of it.
        kwargs["gpu_ids"] = [gpu_ordinal]
    if need:
        kwargs["need_mib"] = need
    return kwargs


def settle_unpriced_chat_beside_media(policy: str) -> None:
    """Media beside an unpriced chat model: When they fit evicts it, Always refuses."""
    if policy not in ("fit", "always") or not _on_metal():
        return
    from utils.account_context import current_account_id

    with arb._lock:
        arb._sync_from_view()
        media = [name for name in MEDIA_OWNERS if name in arb._residents]
        if not media:
            return
        if policy == "always":
            raise arb.refusal_for(
                [slot for name in media for slot in arb.resident_slots(name)],
                arb.KIND_LABELS[media[0]],
            )
        for name in media:
            _evict_resident(name, current_account_id(), None, [])


def forget_media_price(owner: str, cancel: Any = None) -> None:
    """Drop ``owner``'s price; gives up on ``cancel`` since an evictor holding the lock awaits our drain."""
    while not arb._lock.acquire(timeout = 0.05):
        if cancel is not None and cancel.is_set():
            from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
            raise RuntimeError(DIFFUSION_CANCELLED_MSG)
    try:
        arb._media_needs.pop(owner, None)
    finally:
        arb._lock.release()


def refuse_unless_fits(
    owner: str,
    *,
    policy: str = "off",
    gpu_ids: Optional[Iterable[int]] = None,
    need_mib: Optional[dict] = None,
    cancel: Any = None,
) -> None:
    """acquire_for(allow_evict=False) check for loads inside a generation; mutates nothing; gives up
    on cancel because a lock-holding evictor awaits our drain."""
    from utils.account_context import current_account_id

    while not arb._lock.acquire(timeout = 0.05):
        if cancel is not None and cancel.is_set():
            from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
            raise RuntimeError(DIFFUSION_CANCELLED_MSG)
    try:
        arb._sync_from_view()
        others = [name for name in arb._residents if name != owner]
        if not others:
            return
        if policy != "off" and owner in arb._KEPT_UNDER_ALWAYS:
            coreside_locked(
                owner,
                policy = policy,
                need_mib = need_mib,
                gpu_ids = gpu_ids,
                allow_evict = False,
                evict_first = False,
                acting = current_account_id(),
                evicted = None,
            )
            return
        raise arb.GpuOwnerBusyError(others[-1])
    finally:
        arb._lock.release()


def free_for(
    owner: str,
    gpu_ids: Optional[Iterable[int]],
    reason: str,
    evicted: Optional[list] = None,
    *,
    need_mib: Optional[dict] = None,
    short_mib: int = 0,
    media_only: bool = False,
) -> list[str]:
    """Free residents on ``gpu_ids`` so ``owner`` fits, cheapest first. Appends {kind, model} to ``evicted``."""
    from utils.account_context import current_account_id

    acting = current_account_id()
    with arb._lock:
        arb._sync_from_view()
        return _free_steps(
            owner,
            gpu_ids,
            reason,
            evicted,
            need_mib = need_mib,
            short_mib = short_mib,
            acting = acting,
            media_only = media_only,
        )


def _claim_holds_anything(name: str) -> bool:
    """Whether a kept claim holds or is loading a model; unreadable counts as yes. Caller holds the lock."""
    if arb.resident_slots(name):
        return True
    return arb._chat_occupied() if name == arb.CHAT else arb._media_occupied(name)


def coreside_locked(
    owner: str,
    *,
    policy: str,
    need_mib: Optional[dict],
    gpu_ids: Optional[Iterable[int]],
    allow_evict: bool,
    evict_first: bool,
    acting: Optional[str],
    evicted: Optional[list],
) -> None:
    """Makes room for owner under When they fit / Always; under When they fit, unpriceable residents
    are evicted first."""
    target = (
        arb.physical_gpu_ids(gpu_ids)
        if gpu_ids is not None
        else (frozenset(need_mib) if need_mib else None)
    )
    overlap = [
        name for name, gpus in arb._view().items() if name != owner and _overlaps(gpus, target)
    ]
    for name in [n for n in overlap if n not in arb._KEPT_UNDER_ALWAYS]:
        # A Decision API server is evicted as before.
        if not allow_evict:
            raise arb.GpuOwnerBusyError(name)
        _evict_resident(name, acting, evicted, [])
    kept = [n for n in overlap if n in arb._KEPT_UNDER_ALWAYS]
    if policy == "always":
        # A claim whose load failed holds nothing, so Always evicts it rather than refusing (gpu_arbiter.acquire_for).
        for name in [n for n in kept if not _claim_holds_anything(n)]:
            if not allow_evict:
                raise arb.GpuOwnerBusyError(name)
            _evict_resident(name, acting, evicted, [])
            kept.remove(name)

    def evict_or_refuse(names: list[str]) -> None:
        if not names:
            return
        if policy == "always":
            raise arb.refusal_for(
                [slot for n in names for slot in arb.resident_slots(n)], arb.KIND_LABELS[names[0]]
            )
        if not allow_evict:
            raise arb.GpuOwnerBusyError(names[-1])
        for name in names:
            _evict_resident(name, acting, evicted, [])
            kept.remove(name)

    if not kept:
        return
    if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_load_in_flight():
        # Its size isn't known until it launches.
        if policy == "always":
            raise ChatLoadInFlightRefusal()
        evict_or_refuse([arb.CHAT])
    if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_growth_unpriced():
        evict_or_refuse([arb.CHAT])
    if owner == arb.CHAT and not need_mib:
        if evict_first:
            # No estimate before launch (a non-GGUF load, or a load with no retry to make room).
            return evict_or_refuse(list(kept))
        records = _media_records()
        evict_or_refuse([n for n in kept if n not in records or _media_busy(n)[0]])
        if kept and _free_mib_by_gpu()[0] is None:
            evict_or_refuse(list(kept))
        return
    if need_mib is None or evict_first:
        return evict_or_refuse(list(kept))
    if isinstance(need_mib, ChatNeed):
        evict_or_refuse([n for n in kept if n in MEDIA_OWNERS and n not in _media_records()])
        if not kept:
            return
    fits, info = can_load_alongside(owner, need_mib)
    if fits:
        return
    if policy == "always":
        raise need_free_refusal(info, [slot for n in kept for slot in arb.resident_slots(n)])
    if not allow_evict:
        raise arb.GpuOwnerBusyError(kept[-1])
    _free_steps(
        owner,
        target,
        f"GPU memory for {owner}",
        evicted,
        need_mib = need_mib,
        acting = acting,
        media_only = owner == arb.CHAT and _on_metal(),
    )
    if not can_load_alongside(owner, need_mib)[0]:
        # Still short: unload the rest on these GPUs, as Off does, and let the planner offload.
        for name in [
            n for n, gpus in arb._view().items() if n != owner and _overlaps(gpus, target)
        ]:
            _evict_resident(name, acting, evicted, [])