Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep image and video models loaded beside chat models on Apple Silicon (#15693)

Long Yixing556b28bb40d2
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index abc2b54c268..6614f90ded0 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -844,6 +844,9 @@ def would_evict(
in_flight = getattr(gpu_residency, "chat_load_in_flight", None)
if callable(in_flight) and in_flight():
return True # coreside_locked evicts a chat that is still loading
+ unpriced = getattr(gpu_residency, "chat_growth_unpriced", None)
+ if callable(unpriced) and unpriced():
+ return True
answer = fits(owner, need_mib)
ok, info = answer if isinstance(answer, tuple) else (answer, {})
return not (ok or _idle_media_frees_room(gpu_residency, owner, info))
diff --git a/studio/backend/core/inference/gpu_residency.py b/studio/backend/core/inference/gpu_residency.py
index c8e29dc029b..7577edc874a 100644
--- a/studio/backend/core/inference/gpu_residency.py
+++ b/studio/backend/core/inference/gpu_residency.py
@@ -34,6 +34,19 @@ class MediaNeed(dict):
self.kind = kind
+class ChatNeed(dict):
+ """Chat load {GPU: MiB}, priced against free memory plus ``credit_mib`` released by the model it replaces."""
+
+ def __init__(
+ self,
+ per_gpu: dict,
+ *,
+ credit_mib: int = 0,
+ ):
+ super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})
+ self.credit_mib = int(credit_mib)
+
+
def current_policy() -> str:
"""Keep multiple models loaded, read as the owner. Never call this under the arbiter lock (it reads a database)."""
try:
@@ -233,29 +246,30 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,
estimate_safetensors_dense_mib,
file_size_mib,
)
- from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS
+ from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS, ENGINE_MLX
repo, name = request.model_path, request.gguf_filename
if getattr(request, "text_encoder_file", None) or getattr(request, "vae_file", None):
return None # supplied companions are priced from their headers at load time only
kind = resolve_model_kind(name, request.model_kind)
fam = detect_family_for_pick(repo, name, request.family_override)
- if (
- fam is None
- or predict_engine(
- fam,
- model_kind = kind,
- gpu_ordinal = gpu_ordinal,
- gguf_filename = name,
- with_loras = has_active_loras(getattr(request, "loras", None)),
- # An explicit precision keeps a default-routed Apple Silicon load on diffusers, as the load does.
- mlx_blocker = mlx_load_blocker(
- transformer_quant = getattr(request, "transformer_quant", None),
- text_encoder_quant = getattr(request, "text_encoder_quant", None),
- ),
- )
- != ENGINE_DIFFUSERS
- ):
+ if fam is None:
+ return None
+ engine = predict_engine(
+ fam,
+ model_kind = kind,
+ gpu_ordinal = gpu_ordinal,
+ gguf_filename = name,
+ engine = getattr(request, "engine", None),
+ with_loras = has_active_loras(getattr(request, "loras", None)),
+ mlx_blocker = mlx_load_blocker(
+ transformer_quant = getattr(request, "transformer_quant", None),
+ text_encoder_quant = getattr(request, "text_encoder_quant", None),
+ ),
+ )
+ if engine == ENGINE_MLX:
+ return _mlx_image_need(request, fam)
+ if engine != ENGINE_DIFFUSERS:
return None
mib = 1024 * 1024
if kind == "pipeline":
@@ -310,6 +324,40 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,
return weights, estimate_image_runtime_mib(width = None, height = None, family = hint)
+def _mlx_image_need(request: Any, fam: Any) -> Optional[tuple[int, int]]:
+ """(weights MiB, decode headroom MiB) of an MLX image load, from headers."""
+ from core.inference.diffusion_families import resolve_base_repo
+ from core.inference.mlx_diffusion_backend import (
+ decode_headroom_bytes,
+ fetch_base_snapshot,
+ get_mlx_diffusion_backend,
+ plan_for_load,
+ resolve_variant_base,
+ )
+
+ gguf = _cached_pick(request.model_path, request.gguf_filename)
+ if gguf is None:
+ return None
+ base = resolve_base_repo(fam, request.base_repo)
+ if not request.base_repo:
+ base = resolve_variant_base(
+ request.model_path,
+ fam,
+ None,
+ request.gguf_filename,
+ fallback = base,
+ local_files_only = True,
+ )
+ base_dir = fetch_base_snapshot(base, None, local_files_only = True)
+ headroom = decode_headroom_bytes(get_mlx_diffusion_backend()._rt())
+ plan = plan_for_load(gguf, base_dir, headroom_bytes = headroom)
+ if plan.source != "plan":
+ return None
+ mib = 1024 * 1024
+ # Tiled peak suffices: the renderer tiles once the pool cannot take an untiled decode.
+ return (plan.planned_bytes - headroom) // mib, -(-headroom // mib)
+
+
def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:
"""(weights MiB, headroom MiB) for a video load: checkpoint file sizes plus the family's companion table."""
from core.inference.diffusion import DiffusionBackend as DB
@@ -357,12 +405,57 @@ def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,
)
+def _on_metal() -> bool:
+ try:
+ from utils.hardware import DeviceType, get_device
+ return get_device() == DeviceType.MLX
+ except Exception: # noqa: BLE001
+ return False
+
+
+def shared_metal_headroom_bytes() -> Optional[int]:
+ """Headroom on Apple's GPU less resident MLX chat growth; None unless priced media is on Metal."""
+ if not metal_media_priced():
+ return None
+ import psutil
+ from core.inference.llama_cpp import _apple_wired_headroom_bytes
+
+ wired = _apple_wired_headroom_bytes()
+ if wired is None:
+ return None
+ pool = min(wired, int(psutil.virtual_memory().available))
+ return max(0, pool - (_mlx_chat_unallocated_mib() or 0) * 1024 * 1024)
+
+
+def metal_media_priced() -> bool:
+ """Whether a media model is kept on Metal at a price: one that reserved the tiled decode, so it must tile."""
+ return _on_metal() and bool(media_held_mib())
+
+
+def _metal_pool() -> tuple[Optional[dict[int, dict]], Optional[str]]:
+ """Unified memory as GPU 0: wired headroom capped at available RAM (both exclude loaded models)."""
+ try:
+ import psutil
+ from core.inference.llama_cpp import _apple_wired_headroom_bytes
+
+ wired = _apple_wired_headroom_bytes()
+ available = int(psutil.virtual_memory().available)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("gpu_residency: Metal memory probe failed: %s", exc)
+ return None, "probe_error"
+ if wired is None or available <= 0:
+ return None, "probe_error"
+ return {0: {"free_mib": min(wired, available) // (1024 * 1024), "metal": True}}, None
+
+
def _free_mib_by_gpu() -> tuple[Optional[dict[int, dict]], Optional[str]]:
"""{physical id: {free_mib, total_mib}} or (None, reason) when the host can't be priced."""
try:
from utils.hardware import DeviceType, get_device, get_visible_gpu_utilization
+
+ if get_device() == DeviceType.MLX:
+ return _metal_pool()
if get_device() != DeviceType.CUDA:
- # MLX and other unified-memory hosts share the pool with the CPU: co-residency there is deferred.
return None, "non_accelerator"
util = get_visible_gpu_utilization() or {}
except Exception as exc: # noqa: BLE001
@@ -440,6 +533,9 @@ def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dict
for name, need in records.items()
if name != owner and gpu in need
)
+ if row.get("metal"):
+ free -= _mlx_chat_unallocated_mib() or 0
+ free += getattr(need_mib, "credit_mib", 0)
if own is not None and gpu in own:
# Reloading the same owner frees what it holds now.
free += max(own[gpu] - getattr(own, "headroom_mib", 0), 0)
@@ -450,7 +546,7 @@ def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dict
if info["short_mib"]:
return False, info
if getattr(need_mib, "kind", None) == "image":
- for gpu in need_mib:
+ for gpu in [g for g in need_mib if not rows[g].get("metal")]:
if not _keeps_transformer_resident(
need_mib, gpu, info["free_mib"][gpu], rows[gpu]["total_mib"]
):
@@ -474,6 +570,90 @@ def chat_load_in_flight() -> bool:
return False
+def chat_growth_unpriced() -> bool:
+ """On Metal, whether a resident MLX chat model may still grow its KV / prompt cache by an unpriced amount."""
+ return _on_metal() and _mlx_chat_unallocated_mib() is None
+
+
+def _mlx_chat_unallocated_mib() -> Optional[int]:
+ """MiB resident MLX chat models priced but have not allocated yet; None when any is unpriced."""
+ total = 0
+ try:
+ from core.inference import model_slots, orchestrator
+ for orch in [
+ orchestrator._inference_backend,
+ *(s.orchestrator for s in model_slots.resident()),
+ ]:
+ mib = _orch_unallocated_mib(orch)
+ if mib is None:
+ return None
+ total += mib
+ return total
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _orch_unallocated_mib(orch) -> Optional[int]:
+ total = 0
+ for entry in list((getattr(orch, "models", None) or {}).values()):
+ if isinstance(entry, dict) and entry.get("is_mlx"):
+ growth = entry.get("mlx_growth")
+ if growth is None:
+ return None
+ rows = max(1, int(orch.effective_parallel_slots), int(growth.get("slots", 1)))
+ priced = growth["mib"] + (rows - 1) * growth["row_mib"]
+ at_load, now = growth.get("footprint_mib"), mlx_worker_footprint_mib(orch)
+ grown = now - at_load if at_load is not None and now is not None else 0
+ total += max(0, priced - max(0, grown))
+ return total
+
+
+def mlx_replaced_credit_mib(orch) -> int:
+ """MiB a load into ``orch`` gets back from the MLX model it replaces (each load spawns a fresh worker)."""
+ try:
+ if not any(
+ isinstance(e, dict) and e.get("is_mlx")
+ for e in list((getattr(orch, "models", None) or {}).values())
+ ):
+ return 0
+ return (mlx_worker_footprint_mib(orch) or 0) + (_orch_unallocated_mib(orch) or 0)
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def mlx_cache_bounded(entry: dict, slots: int = 1) -> bool:
+ """Whether the MLX cache stops at its priced context; batched vision rows drop the window."""
+ if slots > 1 and entry.get("context_unbounded_when_batched"):
+ return False
+ return entry.get("context_length_enforced") is True or bool(entry.get("mlx_context_budget"))
+
+
+def mlx_slots_outgrow_price(orch, n_parallel) -> bool:
+ """Whether serving ``n_parallel`` rows grows a resident MLX model beside media past what it was admitted at."""
+ try:
+ from core.inference.llama_server_args import clamp_parallel_slots
+ entry = (getattr(orch, "models", None) or {}).get(orch.active_model_name or "") or {}
+ return bool(
+ _on_metal()
+ and entry.get("is_mlx")
+ and media_held_mib()
+ and clamp_parallel_slots(n_parallel) > int(orch.effective_parallel_slots)
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def mlx_worker_footprint_mib(orch) -> Optional[int]:
+ """MiB the MLX worker process holds now, Metal buffers included, or None when unreadable."""
+ proc = getattr(orch, "_proc", None)
+ if proc is None or proc.pid is None:
+ return None
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ usage = LlamaCppBackend._darwin_rusage(proc.pid)
+ return None if usage is None else usage[1] // (1024 * 1024)
+
+
_CHAT_LOADING_MESSAGE = (
"Unsloth Studio is loading a chat model on this GPU. Retry once it has finished loading."
)
@@ -712,6 +892,38 @@ def media_fit_kwargs(
return kwargs
+def settle_unpriced_chat_beside_media(policy: str) -> None:
+ """Media beside an unpriced chat model: When they fit evicts it, Always refuses."""
+ if policy not in ("fit", "always") or not _on_metal():
+ return
+ from utils.account_context import current_account_id
+
+ with arb._lock:
+ arb._sync_from_view()
+ media = [name for name in MEDIA_OWNERS if name in arb._residents]
+ if not media:
+ return
+ if policy == "always":
+ raise arb.refusal_for(
+ [slot for name in media for slot in arb.resident_slots(name)],
+ arb.KIND_LABELS[media[0]],
+ )
+ for name in media:
+ _evict_resident(name, current_account_id(), None, [])
+
+
+def forget_media_price(owner: str, cancel: Any = None) -> None:
+ """Drop ``owner``'s price; gives up on ``cancel`` since an evictor holding the lock awaits our drain."""
+ while not arb._lock.acquire(timeout = 0.05):
+ if cancel is not None and cancel.is_set():
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+ try:
+ arb._media_needs.pop(owner, None)
+ finally:
+ arb._lock.release()
+
+
def refuse_unless_fits(
owner: str,
*,
@@ -845,6 +1057,8 @@ def coreside_locked(
if policy == "always":
raise ChatLoadInFlightRefusal()
evict_or_refuse([arb.CHAT])
+ if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_growth_unpriced():
+ evict_or_refuse([arb.CHAT])
if owner == arb.CHAT and not need_mib:
if evict_first:
# No estimate before launch (a non-GGUF load, or a load with no retry to make room).
@@ -856,6 +1070,10 @@ def coreside_locked(
return
if need_mib is None or evict_first:
return evict_or_refuse(list(kept))
+ if isinstance(need_mib, ChatNeed):
+ evict_or_refuse([n for n in kept if n in MEDIA_OWNERS and n not in _media_records()])
+ if not kept:
+ return
fits, info = can_load_alongside(owner, need_mib)
if fits:
return
@@ -863,7 +1081,15 @@ def coreside_locked(
raise need_free_refusal(info, [slot for n in kept for slot in arb.resident_slots(n)])
if not allow_evict:
raise arb.GpuOwnerBusyError(kept[-1])
- _free_steps(owner, target, f"GPU memory for {owner}", evicted, need_mib = need_mib, acting = acting)
+ _free_steps(
+ owner,
+ target,
+ f"GPU memory for {owner}",
+ evicted,
+ need_mib = need_mib,
+ acting = acting,
+ media_only = owner == arb.CHAT and _on_metal(),
+ )
if not can_load_alongside(owner, need_mib)[0]:
# Still short: unload the rest on these GPUs, as Off does, and let the planner offload.
for name in [
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index fb87fce8546..2fb74c94790 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -599,6 +599,20 @@ def _apple_wired_headroom_bytes() -> Optional[int]:
return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))
+def _apple_budget_beside_media_mib(held_mib: int, unallocated_mib: int) -> int:
+ """Unified-memory budget beside resident media: price off the working set, unallocated part off available RAM."""
+ from utils.hardware.hardware import _metal_recommended_working_set_bytes
+ import psutil
+
+ mib = 1024 * 1024
+ working_set = _metal_recommended_working_set_bytes() or int(psutil.virtual_memory().total)
+ available = int(psutil.virtual_memory().available)
+ left = working_set // mib - held_mib
+ if available > 0:
+ left = min(left, available // mib - unallocated_mib)
+ return max(0, int(left * _APPLE_UNIFIED_MEMORY_FRACTION))
+
+
def _gpu_plan_mib(total_bytes: int, on: list[int], gpus) -> dict[int, int]:
"""MiB this load holds per GPU: an even share, capped at what each card had free, since
--fit on offloads the rest to CPU."""
@@ -25549,6 +25563,26 @@ class LlamaCppBackend:
# Unified-memory budget (0 off Apple Silicon) for the no-GPU Metal cap below.
_apple_budget_mib = self._apple_metal_memory_budget_bytes() // (1024 * 1024)
+ _metal_media_mib, _metal_media_unallocated_mib = (
+ self._metal_media_mib() if _apple_budget_mib > 0 else (0, 0)
+ )
+ if _metal_media_mib:
+ # 0 would mean "not Apple Silicon" and skip every Metal check.
+ _apple_budget_mib = max(
+ 1,
+ min(
+ _apple_budget_mib,
+ _apple_budget_beside_media_mib(
+ _metal_media_mib, _metal_media_unallocated_mib
+ ),
+ ),
+ )
+ _metal_media_wired_mib = int(
+ _metal_media_unallocated_mib * _APPLE_WIRED_CEILING_FRACTION
+ )
+
+ def _apple_wired_left_mib(wired_bytes: int) -> int:
+ return max(0, wired_bytes // (1024 * 1024) - _metal_media_wired_mib)
if tensor_parallel and tp_gpus:
# The planner can shrink ctx/KV, not weights or the MTP reserve.
@@ -26392,8 +26426,7 @@ class LlamaCppBackend:
)
# Past free memory macOS swaps; past the wired limit the machine panics.
_apple_wired_mib = int(
- self._apple_metal_wired_ceiling_bytes()
- // (1024 * 1024)
+ _apple_wired_left_mib(self._apple_metal_wired_ceiling_bytes())
* max(0.0, 1.0 - _flat_mtp_reserve)
)
if (
@@ -26447,8 +26480,7 @@ class LlamaCppBackend:
# Past the wired limit Metal refuses, so it is the harder ceiling when readable.
_fit_wired_mib = (
int(
- self._apple_metal_wired_ceiling_bytes()
- // (1024 * 1024)
+ _apple_wired_left_mib(self._apple_metal_wired_ceiling_bytes())
* max(0.0, 1.0 - _flat_mtp_reserve)
)
if _vram_fit_ctx
@@ -26497,6 +26529,8 @@ class LlamaCppBackend:
# them, so it must see the same wired limit the refusals do.
if not _caller_owns_budget and not _paravirtual_cpu_forced:
_metal_fit_target_floor_mib = self._metal_fit_target_floor_mib()
+ if _metal_fit_target_floor_mib > 0:
+ _metal_fit_target_floor_mib += _metal_media_wired_mib
if (
intent.refuse_partial_gpu_fit
@@ -26513,7 +26547,8 @@ class LlamaCppBackend:
wired_bytes = self._apple_metal_wired_headroom_bytes()
if wired_bytes is not None:
wired_mib = int(
- wired_bytes // (1024 * 1024) * max(0.0, 1.0 - _flat_mtp_reserve)
+ _apple_wired_left_mib(wired_bytes)
+ * max(0.0, 1.0 - _flat_mtp_reserve)
)
budget_mib = min(budget_mib, wired_mib)
need_mib = math.ceil(
@@ -33715,6 +33750,21 @@ class LlamaCppBackend:
return media_unallocated_mib()
+ def _metal_media_mib(self) -> tuple[int, int]:
+ """(price, unallocated) MiB of media resident on Apple's GPU, plus MLX chat growth beside them."""
+ if self not in _serving_backends_snapshot():
+ return 0, 0
+ from core.inference.gpu_residency import (
+ _mlx_chat_unallocated_mib,
+ media_held_mib,
+ media_unallocated_mib,
+ )
+
+ held = media_held_mib().get(0, 0)
+ if not held:
+ return 0, 0
+ return held, media_unallocated_mib().get(0, 0) + (_mlx_chat_unallocated_mib() or 0)
+
def _note_server_pid(self, pid: int) -> None:
if getattr(self, "_owns_pidfile", True):
self._record_server_pid(pid)
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.py
index 925962551b9..2702f1c2f87 100644
--- a/studio/backend/core/inference/mlx_diffusion_backend.py
+++ b/studio/backend/core/inference/mlx_diffusion_backend.py
@@ -480,7 +480,7 @@ class MfluxRuntime:
return image
def free_working_set_bytes(self) -> Optional[int]:
- """Metal's recommended working set minus what MLX holds live now (the resident model), or None."""
+ """Metal working set minus live MLX, capped by system headroom less a chat worker's reserved growth."""
import mlx.core as mx
try:
@@ -490,13 +490,26 @@ class MfluxRuntime:
return None
if budget <= 0:
return None
- return budget - int(mx.get_active_memory())
+ free = budget - int(mx.get_active_memory())
+ try:
+ from core.inference.gpu_residency import shared_metal_headroom_bytes
+ shared = shared_metal_headroom_bytes()
+ except Exception: # noqa: BLE001 -- this process's reading alone, as before
+ shared = None
+ return free if shared is None else min(free, shared)
def _plan_vae_tiling(self, model: Any, width: int, height: int) -> None:
from mflux.models.common.vae.tiling_config import TilingConfig
+ try:
+ from core.inference.gpu_residency import metal_media_priced
+
+ # Price reserved the tiled decode; a chat model may join mid-generation on it.
+ priced = metal_media_priced()
+ except Exception: # noqa: BLE001
+ priced = False
tiled = TilingConfig.may_tile_implicitly(model) and vae_tiling_needed(
- width, height, self.free_working_set_bytes()
+ width, height, 0 if priced else self.free_working_set_bytes()
)
model.tiling_config = TilingConfig() if tiled else None
self.last_vae_tiled = tiled
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index ed5f9dddc54..72e78ac2b0a 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -11812,13 +11812,16 @@ def _chat_fit_kwargs(
placement,
policy: Optional[str] = None,
credit_gb: Optional[dict] = None,
+ mlx_need_mib: Optional[int] = None,
+ mlx_credit_mib: int = 0,
) -> dict:
"""How the CHAT acquire shares the GPU under When they fit / Always.
- audio.cpp and native audio are priced from the placement estimate. A GGUF launch prices itself (it refuses a
- partial fit and load_model_gated makes room). Anything else has no estimate before launch: When they fit
- evicts first, Always tries the load and turns an out-of-memory failure into its refusal. ``credit_gb`` is what
- the outgoing primary frees per GPU when this load replaces it.
+ audio.cpp and native audio are priced from the placement estimate, an MLX load on Apple Silicon from
+ ``mlx_need_mib``. A GGUF launch prices itself (it refuses a partial fit and load_model_gated makes room).
+ Anything else has no estimate before launch: When they fit evicts first, Always tries the load and turns an
+ out-of-memory failure into its refusal. ``credit_gb`` is what the outgoing primary frees per GPU when this
+ load replaces it.
"""
if policy is None:
from core.inference.gpu_residency import current_policy
@@ -11840,11 +11843,42 @@ def _chat_fit_kwargs(
int(g): int((placement.native_required_gb / len(gpus) - credit.get(int(g), 0.0)) * 1024)
for g in gpus
}
- elif policy == "fit" and not (config.is_gguf and _primary_can_retry_fit.get()):
+ elif mlx_need_mib is not None:
+ from core.inference.gpu_residency import ChatNeed
+ kwargs["need_mib"] = ChatNeed({0: int(mlx_need_mib)}, credit_mib = mlx_credit_mib)
+ elif not (config.is_gguf and _primary_can_retry_fit.get()) and (
+ policy == "fit" or (not config.is_gguf and _unified_metal_host())
+ ):
+ # Unified memory pages past the budget: Always gets no OOM to refuse on.
kwargs["evict_first"] = True
return kwargs
+def _unified_metal_host() -> bool:
+ from core.inference.gpu_residency import _on_metal
+ return _on_metal()
+
+
+def _mlx_cache_bounded(entry: dict, slots: int = 1) -> bool:
+ from core.inference.gpu_residency import mlx_cache_bounded
+ return mlx_cache_bounded(entry, slots)
+
+
+def _note_mlx_slots(backend, slots: int) -> None:
+ """A batch opened at a wider setting keeps its width after the setting drops, so the reservation never shrinks."""
+ entry = backend.models.get(backend.active_model_name or "") or {}
+ growth = entry.get("mlx_growth")
+ if isinstance(growth, dict):
+ growth["slots"] = max(int(growth.get("slots", 1)), int(slots))
+ if not _mlx_cache_bounded(entry, growth["slots"]):
+ entry.pop("mlx_growth")
+
+
+def _mlx_slots_outgrow_price(backend, n_parallel) -> bool:
+ from core.inference.gpu_residency import mlx_slots_outgrow_price
+ return mlx_slots_outgrow_price(backend, n_parallel)
+
+
def _outgoing_chat_credit_gb(
placement, policy: str, replacing: bool, availability: "Optional[_NativeAudioAvailability]"
) -> tuple[Optional[dict], Optional[dict]]:
@@ -14707,6 +14741,118 @@ def _mlx_estimate_drafter(
return None, None, True
+class _MlxLoadEstimate(NamedTuple):
+ breakdown: Any
+ fitted_ctx: Optional[int]
+ offloadable: tuple
+ retains_history: bool
+ offloaded: bool = False
+
+
+def _mlx_load_estimate(
+ request, config, model_identifier
+) -> tuple[Optional[str], Optional[_MlxLoadEstimate]]:
+ """``(refusal reason, None)`` or ``(None, estimate)`` of what an MLX load of *request* occupies."""
+ if not _mlx_estimate_available():
+ return "not_gguf", None
+ from core.inference.native_audio import is_native_audio_model
+
+ if getattr(config, "audio_cpp", None) is not None or is_native_audio_model(model_identifier):
+ return "not_gguf", None
+ if getattr(config, "is_lora", False):
+ return "unsizable", None
+ model_dir = _local_mlx_model_dir(config)
+ if not model_dir:
+ return "not_downloaded", None
+ from core.inference.mlx_inference import (
+ mlx_kv_quant_is_refused,
+ mlx_vlm_snapshot_store_available,
+ )
+ from core.inference.mlx_memory import mlx_memory_breakdown
+
+ load_in_4bit = _mlx_estimate_load_in_4bit(config, request)
+ kv_bits = _mlx_estimate_kv_bits(request.mlx_kv_quant, request.mlx_kv_bits)
+ # A width the cache takes makes the window a budget, pinned or fit.
+ if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):
+ kv_bits = None
+ named_ctx = request.max_seq_length or 0
+ offload, offloadable, offload_baseline = _mlx_estimate_offload(
+ request, config, model_identifier, model_dir, load_in_4bit, kv_bits
+ )
+ drafter, fitted_ctx, speculates = _mlx_estimate_drafter(
+ request,
+ model_identifier,
+ model_dir,
+ _mlx_estimate_ceiling(model_dir),
+ load_in_4bit,
+ offload,
+ offload_baseline,
+ )
+ route = {**({"vision": True} if speculates else {}), **offload}
+ if not named_ctx and drafter is None:
+ fitted_ctx = _mlx_estimate_fitted_context(config, model_dir, load_in_4bit, kv_bits, **route)
+ priced_ctx = named_ctx or fitted_ctx or _mlx_estimate_ceiling(model_dir)
+ if not priced_ctx:
+ return "unsizable", None
+ breakdown = mlx_memory_breakdown(
+ model_dir,
+ n_ctx = priced_ctx,
+ kv_bits = kv_bits,
+ load_in_4bit = load_in_4bit,
+ **route,
+ **({} if drafter is None else {"drafter": drafter}),
+ )
+ if breakdown is None:
+ return "unsizable", None
+ retains_history = (
+ not (getattr(config, "is_vision", False) or speculates)
+ or mlx_vlm_snapshot_store_available()
+ )
+ return None, _MlxLoadEstimate(
+ breakdown, fitted_ctx, offloadable, retains_history, "offload" in offload
+ )
+
+
+def _mlx_chat_need(
+ request,
+ config,
+ model_identifier,
+ slots: int = 1,
+) -> Optional[tuple[int, dict]]:
+ """``(need MiB, growth)`` of an MLX chat load serving ``slots`` rows, or None; rows prefill as one batch."""
+ if getattr(config, "is_gguf", False) or not _unified_metal_host():
+ return None
+ try:
+ _, estimate = _mlx_load_estimate(request, config, model_identifier)
+ # Offload workspaces allocate at prefill but are priced as weights.
+ if estimate is None or estimate.offloaded:
+ return None
+ from core.inference.mlx_inference import mlx_memory_budget
+
+ reserve = 0
+ if estimate.retains_history:
+ without, within = (
+ mlx_memory_budget(retains_history = False),
+ mlx_memory_budget(retains_history = True),
+ )
+ reserve = max(0, (without or 0) - (within or 0))
+ except Exception as exc:
+ logger.info("MLX load of %s left unpriced: %s", model_identifier, exc)
+ return None
+ from core.inference.llama_server_args import clamp_parallel_slots
+
+ mib = 1024 * 1024
+ one_row = -(-(int(estimate.breakdown.total_bytes) + reserve) // mib)
+ row = -(-int(estimate.breakdown.kv_bytes + estimate.breakdown.compute_bytes) // mib)
+ slots = clamp_parallel_slots(slots)
+ growth = {
+ "mib": max(0, one_row - int(estimate.breakdown.weights_bytes) // mib),
+ "row_mib": row,
+ "slots": slots,
+ }
+ return one_row + (slots - 1) * row, growth
+
+
def _mlx_estimate_available() -> bool:
"""Whether this host would actually run the load through MLX."""
try:
@@ -18453,6 +18599,21 @@ async def _load_model_impl(
if not (request.gguf_variant or is_direct_gguf_request):
_inherit_resident_load_in_4bit(backend, request, model_identifier)
await asyncio.to_thread(reap_dead_worker, backend)
+ if (
+ not request.max_seq_length
+ and _same_loaded_identifier(backend.active_model_name, model_identifier)
+ and _mlx_slots_outgrow_price(backend, _n_parallel)
+ ):
+ # A reload for more rows keeps the context the resident was asked for.
+ request.max_seq_length = (
+ int(
+ (backend.models.get(backend.active_model_name) or {}).get(
+ "max_seq_length_requested"
+ )
+ or 0
+ )
+ or request.max_seq_length
+ )
if (
_same_loaded_identifier(backend.active_model_name, model_identifier)
and (backend.models.get(backend.active_model_name) or {}).get("engine", "auto")
@@ -18469,10 +18630,11 @@ async def _load_model_impl(
)
# Without this the route reports already_loaded and nothing moves.
and _resident_audio_placement_matches(backend, request)
+ and not _mlx_slots_outgrow_price(backend, _n_parallel)
):
api_monitor.discard(_load_event) # nothing loaded, no monitor row
logger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")
- backend.set_parallel_slots(_n_parallel)
+ _note_mlx_slots(backend, backend.set_parallel_slots(_n_parallel))
# A no-op Unsloth load of a preview-owned checkpoint still claims it.
_set_preview_resident(None)
inference_config = load_inference_config(backend.active_model_name)
@@ -18822,6 +18984,7 @@ async def _load_model_impl(
logger.warning("Loading an oversized GGUF: %s", _host_offload_warning)
expected_native_owner = None
+ mlx_chat_need = None
post_media_handoff_gpu = None
post_media_handoff_needed_gb = None
post_chat_handoff_expected_free_gb = None
@@ -18875,8 +19038,24 @@ async def _load_model_impl(
)
if post_chat_handoff_expected_free_gb is None:
post_chat_handoff_expected_free_gb = chat_wait_gb
+ mlx_chat_need = await asyncio.to_thread(
+ _mlx_chat_need, request, config, model_identifier, _n_parallel
+ )
+ mlx_credit_mib = 0
+ if mlx_chat_need is not None:
+ from core.inference.gpu_residency import mlx_replaced_credit_mib
+
+ # No llama-server credit: its footprint mixes CPU-only buffers with Metal ones.
+ mlx_credit_mib = await asyncio.to_thread(mlx_replaced_credit_mib, backend)
handoff_kwargs.update(
- _chat_fit_kwargs(config, placement, multi_model_mode, outgoing_chat_gb)
+ _chat_fit_kwargs(
+ config,
+ placement,
+ multi_model_mode,
+ outgoing_chat_gb,
+ mlx_need_mib = None if mlx_chat_need is None else mlx_chat_need[0],
+ mlx_credit_mib = mlx_credit_mib,
+ )
)
await asyncio.to_thread(
acquire_for_request,
@@ -19255,6 +19434,29 @@ async def _load_model_impl(
_resident_entry["gpu_ids_requested"] = (
list(request.gpu_ids) if request.gpu_ids else None
)
+ if mlx_chat_need is not None and _mlx_cache_bounded(
+ _resident_entry, mlx_chat_need[1]["slots"]
+ ):
+ from core.inference.gpu_residency import mlx_worker_footprint_mib
+ _resident_entry["mlx_growth"] = {
+ **mlx_chat_need[1],
+ "footprint_mib": mlx_worker_footprint_mib(backend),
+ }
+ elif mlx_chat_need is not None:
+ # Admitted at a price the loaded cache does not keep to: settle as an unpriced load would have.
+ from core.inference.gpu_arbiter import (
+ GpuBusyForAnotherAccountError,
+ MultiModelAlwaysRefusal,
+ )
+ from core.inference.gpu_residency import settle_unpriced_chat_beside_media
+ try:
+ await asyncio.to_thread(settle_unpriced_chat_beside_media, multi_model_mode)
+ except (MultiModelAlwaysRefusal, GpuBusyForAnotherAccountError) as exc:
+ await asyncio.to_thread(backend.unload_model, config.identifier)
+ await asyncio.to_thread(backend._shutdown_subprocess, 5.0)
+ if isinstance(exc, GpuBusyForAnotherAccountError):
+ raise exc.as_http_exception() from exc
+ raise
if request._advertised_id:
backend._openai_advertised_id = request._advertised_id
@@ -20618,64 +20820,15 @@ async def estimate_memory(
if config is None:
return EstimateMemoryResponse(available = False, reason = "unsizable")
if not getattr(config, "is_gguf", False):
- if not _mlx_estimate_available():
- return EstimateMemoryResponse(available = False, reason = "not_gguf")
- from core.inference.native_audio import is_native_audio_model
-
- if getattr(config, "audio_cpp", None) is not None or is_native_audio_model(
- model_identifier
- ):
- return EstimateMemoryResponse(available = False, reason = "not_gguf")
- if getattr(config, "is_lora", False):
- return EstimateMemoryResponse(available = False, reason = "unsizable")
- model_dir = _local_mlx_model_dir(config)
- if not model_dir:
- return EstimateMemoryResponse(available = False, reason = "not_downloaded")
- from core.inference.mlx_inference import mlx_kv_quant_is_refused
- from core.inference.mlx_memory import mlx_memory_breakdown
-
- mlx_load_in_4bit = _mlx_estimate_load_in_4bit(config, request)
- mlx_kv_bits = _mlx_estimate_kv_bits(request.mlx_kv_quant, request.mlx_kv_bits)
- # A width the cache takes applies under a pin or a fit alike: the window is then a budget.
- if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):
- mlx_kv_bits = None
- mlx_named_ctx = request.max_seq_length or 0
- mlx_offload, mlx_offloadable, mlx_offload_baseline = _mlx_estimate_offload(
- request, config, model_identifier, model_dir, mlx_load_in_4bit, mlx_kv_bits
- )
- mlx_drafter, mlx_fitted_ctx, mlx_speculates = _mlx_estimate_drafter(
- request,
- model_identifier,
- model_dir,
- _mlx_estimate_ceiling(model_dir),
- mlx_load_in_4bit,
- mlx_offload,
- mlx_offload_baseline,
- )
- mlx_route = {**({"vision": True} if mlx_speculates else {}), **mlx_offload}
- if not mlx_named_ctx and mlx_drafter is None:
- mlx_fitted_ctx = _mlx_estimate_fitted_context(
- config, model_dir, mlx_load_in_4bit, mlx_kv_bits, **mlx_route
- )
- mlx_priced_ctx = mlx_named_ctx or mlx_fitted_ctx or _mlx_estimate_ceiling(model_dir)
- if not mlx_priced_ctx:
- return EstimateMemoryResponse(available = False, reason = "unsizable")
- mlx_breakdown = mlx_memory_breakdown(
- model_dir,
- n_ctx = mlx_priced_ctx,
- kv_bits = mlx_kv_bits,
- load_in_4bit = mlx_load_in_4bit,
- **mlx_route,
- **({} if mlx_drafter is None else {"drafter": mlx_drafter}),
- )
- if mlx_breakdown is None:
- return EstimateMemoryResponse(available = False, reason = "unsizable")
+ reason, mlx = _mlx_load_estimate(request, config, model_identifier)
+ if mlx is None:
+ return EstimateMemoryResponse(available = False, reason = reason)
return EstimateMemoryResponse(
**project_estimate_memory_response(
build_memory_estimate(
- dataclasses.replace(mlx_breakdown, offloadable = mlx_offloadable),
+ dataclasses.replace(mlx.breakdown, offloadable = mlx.offloadable),
quant_file_bytes = 0,
- context_fitted = mlx_fitted_ctx,
+ context_fitted = mlx.fitted_ctx,
)
)
)
@@ -46821,6 +46974,7 @@ def _mlx_fallback_load_gate(request, gpu_ordinal, multi_model_mode):
from core.inference.gpu_arbiter import DIFFUSION
from core.inference.gpu_residency import (
ChatLoadInFlightRefusal,
+ forget_media_price,
media_fit_kwargs,
refuse_unless_fits,
)
@@ -46837,7 +46991,8 @@ def _mlx_fallback_load_gate(request, gpu_ordinal, multi_model_mode):
DIFFUSION,
policy = fit.get("policy", "off"),
gpu_ids = fit.get("gpu_ids"),
- need_mib = fit.get("need_mib"),
+ # The price is the MLX load's; this diffusers load has none.
+ need_mib = None,
cancel = cancel,
)
except (HTTPException, ChatLoadInFlightRefusal, RuntimeError) as exc:
@@ -46855,6 +47010,7 @@ def _mlx_fallback_load_gate(request, gpu_ordinal, multi_model_mode):
):
if ref and _repo_is_in_the_hub_cache(ref) is not True:
_note_load_fetched_with_a_request_token(ref, request.hf_token)
+ forget_media_price(DIFFUSION, cancel)
return start()
return gate
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index a402f71cd34..9593578f614 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -583,6 +583,13 @@ def _no_live_metal_wired_ceiling(monkeypatch):
)
[email protected](autouse = True)
+def _not_on_metal(monkeypatch):
+ """Keep co-residency verdicts off the host's device: on a Mac they take the Metal paths."""
+ from core.inference import gpu_residency
+ monkeypatch.setattr(gpu_residency, "_on_metal", lambda: False)
+
+
@pytest.fixture(autouse = True)
def _no_leftover_generation_account(monkeypatch):
"""Reset routes.video's _generation_account per test, since no route ever clears it."""
diff --git a/studio/backend/tests/test_gpu_arbiter.py b/studio/backend/tests/test_gpu_arbiter.py
index 4bcd94ea355..c52b70cf0f5 100644
--- a/studio/backend/tests/test_gpu_arbiter.py
+++ b/studio/backend/tests/test_gpu_arbiter.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import types
+from contextlib import nullcontext
import pytest
@@ -155,6 +156,87 @@ def test_image_load_beside_a_resident_chat_model_policy_matrix(cores, policy, ne
assert arb._media_needs.get(arb.DIFFUSION) == (need if policy != "off" else None)
+ "policy, growth, outcome",
+ [("always", 512, "join"), ("fit", None, "evict"), ("always", None, "refuse")],
+)
+def test_image_load_beside_a_chat_model_on_metal(cores, monkeypatch, policy, growth, outcome):
+ import core.inference.gpu_residency as res
+
+ monkeypatch.setattr(res, "_on_metal", lambda: True)
+ monkeypatch.setattr(res, "_mlx_chat_unallocated_mib", lambda: growth)
+ cores.fits()
+ arb.acquire_for(arb.CHAT)
+ assert arb.would_evict(arb.DIFFUSION, policy = policy, need_mib = {0: 4096}) is (outcome == "evict")
+ with pytest.raises(arb.MultiModelAlwaysRefusal) if outcome == "refuse" else nullcontext():
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, need_mib = {0: 4096})
+ gone = {"join": set(), "evict": {arb.CHAT}, "refuse": {arb.DIFFUSION}}[outcome]
+ assert {arb.CHAT, arb.DIFFUSION} - set(arb.residents()) == gone
+
+
[email protected]("metal", [True, False])
+def test_a_short_chat_load_on_metal_frees_media_but_no_kept_chat_model(cores, monkeypatch, metal):
+ import core.inference.gpu_residency as res
+ from core.inference import model_slots
+
+ monkeypatch.setattr(res, "_on_metal", lambda: metal)
+ monkeypatch.setattr(res, "_mlx_chat_unallocated_mib", lambda: 0)
+ monkeypatch.setattr(res, "can_load_alongside", lambda *a: (False, {"short_mib": {0: 1}}))
+ asked = []
+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *a, **k: asked.append(a) or [])
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})
+ arb.acquire_for(arb.CHAT, policy = "fit", need_mib = {0: 9000})
+ assert arb.DIFFUSION not in arb.residents() and bool(asked) is not metal
+ asked.clear()
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})
+ assert asked
+
+
[email protected]("policy", ["fit", "always"])
[email protected]("priced", [True, False])
+def test_a_priced_mlx_chat_load_does_not_join_an_unpriced_image(cores, monkeypatch, policy, priced):
+ import core.inference.gpu_residency as res
+
+ monkeypatch.setattr(res, "can_load_alongside", lambda *a: (True, {}))
+ arb.acquire_for(
+ arb.DIFFUSION, lambda: None, policy = "fit" if priced else "off", need_mib = {0: 4096}
+ )
+ refused = policy == "always" and not priced
+ with pytest.raises(arb.MultiModelAlwaysRefusal) if refused else nullcontext():
+ arb.acquire_for(arb.CHAT, policy = policy, need_mib = res.ChatNeed({0: 2048}))
+ assert (arb.DIFFUSION in arb.residents()) is (priced or refused)
+
+
[email protected]("policy", ["off", "fit", "always"])
+def test_an_unpriced_chat_load_settles_beside_media_as_its_policy_says(cores, monkeypatch, policy):
+ import core.inference.gpu_residency as res
+
+ monkeypatch.setattr(res, "_on_metal", lambda: True)
+ res.settle_unpriced_chat_beside_media(policy) # nothing resident
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})
+ with pytest.raises(arb.MultiModelAlwaysRefusal) if policy == "always" else nullcontext():
+ res.settle_unpriced_chat_beside_media(policy)
+ assert (arb.DIFFUSION in arb.residents()) is (policy != "fit")
+ monkeypatch.setattr(res, "_on_metal", lambda: False)
+ res.settle_unpriced_chat_beside_media("fit")
+ assert (arb.DIFFUSION in arb.residents()) is (policy != "fit")
+
+
+def test_a_renderer_change_forgets_the_media_price(cores):
+ import core.inference.gpu_residency as res
+
+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})
+ assert arb._media_needs.get(arb.DIFFUSION) == {0: 4096}
+ res.forget_media_price(arb.DIFFUSION)
+ assert arb.DIFFUSION not in arb._media_needs and arb.DIFFUSION in arb.residents()
+ import threading
+
+ cancel = threading.Event()
+ cancel.set()
+ with arb._lock, pytest.raises(RuntimeError):
+ res.forget_media_price(arb.DIFFUSION, cancel)
+
+
@pytest.mark.parametrize("managed", [False, True])
def test_always_refuses_an_image_load_that_does_not_fit_without_evicting(cores, managed):
from utils.account_context import AccountContext, run_as
diff --git a/studio/backend/tests/test_gpu_residency_coexistence.py b/studio/backend/tests/test_gpu_residency_coexistence.py
index 1250333cc82..fb7e69d0508 100644
--- a/studio/backend/tests/test_gpu_residency_coexistence.py
+++ b/studio/backend/tests/test_gpu_residency_coexistence.py
@@ -266,7 +266,7 @@ class TestCanLoadAlongside(_ArbiterResetMixin, unittest.TestCase):
gr.can_load_alongside(arb.VIDEO, None), (False, {"reason": "estimate_unavailable"})
)
self.assertEqual(gr.can_load_alongside(arb.VIDEO, {})[1]["reason"], "estimate_unavailable")
- for device in (DeviceType.MLX, DeviceType.CPU, DeviceType.XPU):
+ for device in (DeviceType.CPU, DeviceType.XPU):
self.assertEqual(
self._check(need, device = device), (False, {"reason": "non_accelerator"})
)
@@ -364,6 +364,90 @@ class TestCanLoadAlongside(_ArbiterResetMixin, unittest.TestCase):
):
self.assertTrue(self._check(_need({0: 1000}, kind = "video"))[0])
+ def _metal(self, need, wired, available, *chat_growth):
+ with (
+ _cuda(DeviceType.MLX),
+ patch.object(gr, "_mlx_chat_unallocated_mib", return_value = sum(chat_growth)),
+ patch(
+ "core.inference.llama_cpp._apple_wired_headroom_bytes",
+ return_value = wired and wired * MIB,
+ ),
+ patch("psutil.virtual_memory", return_value = SimpleNamespace(available = available * MIB)),
+ ):
+ return gr.can_load_alongside(arb.VIDEO, need)
+
+ def test_metal_is_one_pool_at_the_smaller_reading_less_idle_headroom(self):
+ self._resident(arb.DIFFUSION, {0}, _need({0: 9000}, weights = 6000, headroom = 3000))
+ free = 12288 - 3000
+ for w, a in ((20480, 12288), (12288, 20480)):
+ self.assertEqual(self._metal(_need({0: 1}, kind = "image"), w, a)[1]["free_mib"][0], free)
+ self.assertTrue(self._metal(_need({0: free - gr.FIT_MARGIN_MIB}), 20480, 12288)[0])
+ self.assertFalse(self._metal(_need({0: free - gr.FIT_MARGIN_MIB + 1}), 20480, 12288)[0])
+ self.assertEqual(self._metal(_need({0: 1}), None, 12288)[1], {"reason": "probe_error"})
+ self.assertEqual(self._metal(_need({0: 1}), w, a, 2500)[1]["free_mib"], {0: free - 2500})
+ credited = gr.ChatNeed({0: 8192}, credit_mib = 12288)
+ self.assertTrue(self._metal(credited, 6144, 6144, 4096)[0])
+ self.assertFalse(self._metal(gr.ChatNeed({0: 8192}), 6144, 6144, 4096)[0])
+
+ def test_mlx_chat_growth_adds_a_kv_cache_per_row_less_what_each_worker_grew(self):
+ from core.inference import llama_cpp, model_slots, orchestrator
+
+ def orch(growth, rows, pid):
+ entry = {"is_mlx": True, "mlx_growth": growth} if growth else {"is_mlx": growth is None}
+ proc = SimpleNamespace(pid = pid)
+ return SimpleNamespace(models = {"m": entry}, effective_parallel_slots = rows, _proc = proc)
+
+ def total(grown, primary, *kept):
+ slots = [SimpleNamespace(orchestrator = o) for o in kept]
+ with (
+ patch.object(orchestrator, "_inference_backend", primary),
+ patch.object(model_slots, "resident", lambda: slots),
+ patch.object(gr, "mlx_worker_footprint_mib", lambda o: grown.get(o._proc.pid)),
+ ):
+ return gr._mlx_chat_unallocated_mib()
+
+ g = {"mib": 512, "row_mib": 100, "footprint_mib": 2000}
+ self.assertEqual(total({}, orch(g, 4, 7), orch({**g, "mib": 256}, 1, 8)), 1068)
+ self.assertEqual(total({7: 2300}, orch(g, 4, 7), orch(False, 1, 9)), 512)
+ self.assertEqual(total({7: 1900}, orch(g, 4, 7)), 812)
+ self.assertEqual(total({7: 9000}, orch(g, 4, 7)), 0)
+ self.assertEqual(total({}, orch({**g, "slots": 4}, 1, 7)), 812)
+ with patch.object(llama_cpp.LlamaCppBackend, "_darwin_rusage", return_value = (1, 3 * MIB)):
+ self.assertEqual(gr.mlx_worker_footprint_mib(orch(g, 1, 7)), 3)
+ with patch.object(model_slots, "resident", side_effect = RuntimeError):
+ self.assertIsNone(gr._mlx_chat_unallocated_mib())
+ self.assertIsNone(total({}, orch(g, 1, 7), orch(None, 1, 8)))
+
+ def test_a_replaced_mlx_model_credits_its_worker_and_reserved_growth(self):
+ g = {"mib": 512, "row_mib": 100, "footprint_mib": 2000}
+
+ def orch(entry, rows = 2):
+ return SimpleNamespace(
+ models = {"m": entry}, effective_parallel_slots = rows, _proc = SimpleNamespace(pid = 7)
+ )
+
+ with patch.object(gr, "mlx_worker_footprint_mib", lambda o: 2100):
+ self.assertEqual(
+ gr.mlx_replaced_credit_mib(orch({"is_mlx": True, "mlx_growth": g})), 2100 + 512
+ )
+ self.assertEqual(gr.mlx_replaced_credit_mib(orch({"is_mlx": True})), 2100)
+ self.assertEqual(gr.mlx_replaced_credit_mib(orch({"is_mlx": False})), 0)
+
+ def test_more_parallel_rows_beside_media_outgrow_the_admitted_price(self):
+ o = SimpleNamespace(
+ models = {"m": {"is_mlx": True}}, active_model_name = "m", effective_parallel_slots = 2
+ )
+ with (
+ patch.object(gr, "_on_metal", return_value = True),
+ patch.object(gr, "media_held_mib", return_value = {0: 9000}),
+ ):
+ self.assertTrue(gr.mlx_slots_outgrow_price(o, 4))
+ self.assertFalse(gr.mlx_slots_outgrow_price(o, 2))
+ with patch.object(gr, "media_held_mib", return_value = {}):
+ self.assertFalse(gr.mlx_slots_outgrow_price(o, 4))
+ with patch.object(gr, "_on_metal", return_value = False):
+ self.assertFalse(gr.mlx_slots_outgrow_price(o, 4))
+
class TestPipelineNeed(_GpuCacheResetMixin, unittest.TestCase):
def setUp(self):
@@ -731,3 +815,54 @@ def test_single_file_image_need_reads_fp8_from_the_header(tmp_path, monkeypatch)
save_file({"blocks.0.weight": torch.zeros(1024, 2048, dtype = torch.bfloat16)}, str(dense))
# Both files are 4 MiB on disk: only the one that stores fp8 widens on load.
assert need(plain) > need(dense)
+
+
+def test_an_mlx_routed_image_load_is_priced_from_its_quantization_plan(monkeypatch):
+ import types
+
+ import core.inference.diffusion_engine_router as router
+ import core.inference.diffusion_families as dfam
+ import core.inference.gpu_residency as gr
+ import core.inference.mlx_diffusion_backend as mdb
+ from core.inference.sd_cpp_engine import ENGINE_MLX
+
+ picks = []
+ monkeypatch.setattr(
+ dfam, "detect_family_for_pick", lambda *a, **k: types.SimpleNamespace(name = "z")
+ )
+ monkeypatch.setattr(dfam, "resolve_base_repo", lambda f, b: b)
+ monkeypatch.setattr(
+ router, "predict_engine", lambda *a, **k: picks.append(k["engine"]) or ENGINE_MLX
+ )
+ monkeypatch.setattr(gr, "_cached_pick", lambda repo, name: "/m.gguf")
+ fetched = []
+ monkeypatch.setattr(
+ mdb, "fetch_base_snapshot", lambda base, *a, **k: fetched.append(base) or "/base"
+ )
+ monkeypatch.setattr(
+ mdb,
+ "resolve_variant_base",
+ lambda repo, f, tok, name, *, fallback, local_files_only: "org/variant",
+ )
+ monkeypatch.setattr(
+ mdb, "get_mlx_diffusion_backend", lambda: types.SimpleNamespace(_rt = lambda: None)
+ )
+ monkeypatch.setattr(mdb, "decode_headroom_bytes", lambda rt: 3 * MIB + 1)
+ plan = types.SimpleNamespace(source = "plan", planned_bytes = 4099 * MIB + 1)
+ monkeypatch.setattr(mdb, "plan_for_load", lambda *a, **k: plan)
+ request = types.SimpleNamespace(
+ model_path = "org/z",
+ gguf_filename = "z.gguf",
+ text_encoder_file = None,
+ vae_file = None,
+ model_kind = "gguf",
+ family_override = None,
+ base_repo = "org/base",
+ engine = "auto",
+ )
+ assert gr._image_need(request, 0) == (4096, 4)
+ assert picks == ["auto"] and fetched == ["org/base"]
+ request.base_repo = None
+ assert gr._image_need(request, 0) == (4096, 4) and fetched[-1] == "org/variant"
+ plan.source = "ram_rule"
+ assert gr._image_need(request, 0) is None
diff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.py
index 9637af4cade..866899c17aa 100644
--- a/studio/backend/tests/test_metal_explicit_context_guard.py
+++ b/studio/backend/tests/test_metal_explicit_context_guard.py
@@ -1864,6 +1864,33 @@ class TestMetalAlongsideAdmission:
**kwargs,
)["cmd"]
+ @pytest.mark.parametrize(
+ "media, gib",
+ [((20480, 0), 22), ((1, 20480), 22), ((6144, 65536), 128), ((30720, 0), 22)],
+ )
+ def test_media_comes_off_the_metal_budget(self, tmp_path, monkeypatch, media, gib):
+ import core.inference.llama_cpp as llama_cpp
+ import utils.hardware.hardware as hardware
+
+ memory = _types.SimpleNamespace(total = gib << 30, available = gib << 30)
+ monkeypatch.setattr("psutil.virtual_memory", lambda: memory)
+ monkeypatch.setattr(hardware, "_metal_recommended_working_set_bytes", lambda: gib << 30)
+ monkeypatch.setattr(LlamaCppBackend, "_metal_media_mib", lambda self: media)
+ args = dict(n_ctx = 0, real_fit = True, refuse_partial_gpu_fit = True, weights_bytes = 4 << 30)
+ with pytest.raises(llama_cpp.GpuMemoryShortError):
+ _launch(tmp_path, monkeypatch, budget_bytes = 32 << 30, wired_bytes = 64 << 30, **args)
+
+ def test_an_mlx_chat_models_growth_comes_off_beside_media(self, monkeypatch):
+ import core.inference.gpu_residency as residency
+ import core.inference.llama_cpp as llama_cpp
+
+ backend = LlamaCppBackend()
+ monkeypatch.setattr(llama_cpp, "_serving_backends_snapshot", lambda: [backend])
+ monkeypatch.setattr(residency, "media_held_mib", lambda: {0: 6144})
+ monkeypatch.setattr(residency, "media_unallocated_mib", lambda: {0: 1024})
+ monkeypatch.setattr(residency, "_mlx_chat_unallocated_mib", lambda: 2048)
+ assert backend._metal_media_mib() == (6144, 3072)
+
def test_retry_rechecks_memory_after_eviction(self, tmp_path, monkeypatch):
from core.inference.llama_cpp import GpuMemoryShortError
diff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.py
index 744ac42ff33..4422dd1a933 100644
--- a/studio/backend/tests/test_mlx_diffusion_backend.py
+++ b/studio/backend/tests/test_mlx_diffusion_backend.py
@@ -1876,3 +1876,56 @@ def test_the_mlx_base_fetch_keeps_the_fallback_scheduler_config():
from core.inference.mlx_diffusion_backend import BASE_ALLOW_PATTERNS
for name in ("scheduler/scheduler_config.json", "model_index.json"):
assert any(fnmatch.fnmatch(name, pattern) for pattern in BASE_ALLOW_PATTERNS), name
+
+
+def test_the_renderer_sees_the_shared_pool_beside_a_chat_model(monkeypatch):
+ """Its own working set cannot see a chat worker, so tiling decides on the system-wide headroom too."""
+ import sys
+ import types
+
+ import core.inference.gpu_residency as gr
+ import core.inference.llama_cpp as llama_cpp
+
+ core = types.SimpleNamespace(
+ device_info = lambda: {"max_recommended_working_set_size": 96 * _GIB},
+ get_active_memory = lambda: 8 * _GIB,
+ )
+ monkeypatch.setitem(sys.modules, "mlx", types.SimpleNamespace(core = core))
+ monkeypatch.setitem(sys.modules, "mlx.core", core)
+ monkeypatch.setattr(gr, "_on_metal", lambda: True)
+ monkeypatch.setattr(llama_cpp, "_apple_wired_headroom_bytes", lambda: 16 * _GIB)
+ monkeypatch.setattr(gr, "_mlx_chat_unallocated_mib", lambda: 4096)
+ runtime = mb.MfluxRuntime()
+ monkeypatch.setattr(gr, "media_held_mib", lambda: {0: 9000})
+ monkeypatch.setattr("psutil.virtual_memory", lambda: types.SimpleNamespace(available = 64 * _GIB))
+ assert runtime.free_working_set_bytes() == 12 * _GIB
+ monkeypatch.setattr("psutil.virtual_memory", lambda: types.SimpleNamespace(available = 5 * _GIB))
+ assert runtime.free_working_set_bytes() == 1 * _GIB
+ assert gr.metal_media_priced()
+ monkeypatch.setattr(gr, "media_held_mib", lambda: {})
+ assert runtime.free_working_set_bytes() == 88 * _GIB
+
+
[email protected]("priced, free_gib, expected", [(True, 40.0, True), (False, 40.0, False)])
+def test_a_priced_media_model_on_metal_always_tiles(monkeypatch, priced, free_gib, expected):
+ """Its price reserved the tiled decode, and a chat model may join on it while this render runs."""
+ import sys
+ import types
+
+ import core.inference.gpu_residency as gr
+
+ class _TilingConfig:
+ @staticmethod
+ def may_tile_implicitly(model):
+ return True
+
+ module = types.ModuleType("mflux.models.common.vae.tiling_config")
+ module.TilingConfig = _TilingConfig
+ monkeypatch.setitem(sys.modules, "mflux.models.common.vae.tiling_config", module)
+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)
+ monkeypatch.setattr(gr, "metal_media_priced", lambda: priced)
+ runtime = mb.MfluxRuntime()
+ monkeypatch.setattr(runtime, "free_working_set_bytes", lambda: int(free_gib * _GIB))
+ model = types.SimpleNamespace(tiling_config = None)
+ runtime._plan_vae_tiling(model, 1024, 1024)
+ assert runtime.vae_tiled() is expected
diff --git a/studio/backend/tests/test_multi_model_primary_fit.py b/studio/backend/tests/test_multi_model_primary_fit.py
index a58450345d5..f2f6032a1c2 100644
--- a/studio/backend/tests/test_multi_model_primary_fit.py
+++ b/studio/backend/tests/test_multi_model_primary_fit.py
@@ -12,7 +12,7 @@ from fastapi import HTTPException
import core.inference.model_slots as model_slots
import core.inference.orchestrator as orchestrator
import routes.inference as inf
-from core.inference import gpu_arbiter, gpu_residency
+from core.inference import gpu_arbiter, gpu_residency, mlx_inference
from core.inference.llama_cpp import GpuMemoryShortError
from models.inference import LoadRequest, LoadResponse
from utils import multi_model_settings
@@ -297,6 +297,26 @@ def test_media_on_target_gpus(monkeypatch, held, placement, expected):
assert inf._media_on_target_gpus(placement) is expected
+def test_mlx_chat_need_prices_one_row_the_prompt_cache_and_a_batch_row_per_extra_slot(monkeypatch):
+ gib = 1024**3
+ breakdown = SimpleNamespace(
+ total_bytes = 3 * gib, weights_bytes = gib, kv_bytes = gib // 2, compute_bytes = gib // 4
+ )
+ estimate = inf._MlxLoadEstimate(breakdown, None, (), True)
+ monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, estimate))
+ monkeypatch.setattr(
+ mlx_inference, "mlx_memory_budget", lambda **k: (10 - k["retains_history"]) << 30
+ )
+ mlx = SimpleNamespace(is_gguf = False)
+ assert inf._mlx_chat_need(None, mlx, "m") is None # not on Metal
+ monkeypatch.setattr(inf, "_unified_metal_host", lambda: True)
+ need, growth = inf._mlx_chat_need(None, mlx, "m", 4)
+ assert (need, growth) == (4096 + 3 * 768, {"mib": 3072, "row_mib": 768, "slots": 4})
+ offloaded = estimate._replace(offloaded = True)
+ monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, offloaded))
+ assert inf._mlx_chat_need(None, mlx, "m", 4) is None
+
+
def test_chat_fit_kwargs(monkeypatch):
policy = SimpleNamespace(value = "off")
monkeypatch.setattr(gpu_residency, "current_policy", lambda: policy.value)
@@ -328,6 +348,11 @@ def test_chat_fit_kwargs(monkeypatch):
# No target GPUs (or Vulkan ordinals) leaves every GPU in play.
assert inf._chat_fit_kwargs(gguf, _placement(None)) == {"policy": "always"}
assert inf._chat_fit_kwargs(gguf, _placement([0], vulkan = True)) == {"policy": "always"}
+ monkeypatch.setattr(inf, "_unified_metal_host", lambda: True)
+ assert inf._chat_fit_kwargs(native, _placement(None)).get("evict_first") is True
+ priced = inf._chat_fit_kwargs(native, _placement(None), mlx_need_mib = 700, mlx_credit_mib = 50)
+ assert priced.get("need_mib") == {0: 700} and "evict_first" not in priced
+ assert priced["need_mib"].credit_mib == 50
@pytest.mark.parametrize(
@@ -357,6 +382,62 @@ def test_the_replace_fallback_never_refuses_a_partial_fit():
)
+def test_only_a_bounded_mlx_cache_is_priced_and_its_admitted_width_never_shrinks():
+ assert inf._mlx_cache_bounded({"context_length_enforced": True})
+ assert inf._mlx_cache_bounded({"context_length_enforced": False, "mlx_context_budget": 4096})
+ assert not inf._mlx_cache_bounded({"context_length_enforced": None})
+ batching = {"context_length_enforced": True, "context_unbounded_when_batched": True}
+ assert inf._mlx_cache_bounded(batching) and not inf._mlx_cache_bounded(batching, 2)
+ assert not inf._mlx_cache_bounded(
+ {"context_length_enforced": False, "mlx_context_budget": None}
+ )
+ entry = {"context_length_enforced": True, "mlx_growth": {"mib": 1, "row_mib": 1, "slots": 1}}
+ backend = SimpleNamespace(models = {"m": entry}, active_model_name = "m")
+ for slots in (4, 1):
+ inf._note_mlx_slots(backend, slots)
+ assert entry["mlx_growth"]["slots"] == 4
+ vision = {**batching, "mlx_growth": {"mib": 1, "row_mib": 1, "slots": 1}}
+ inf._note_mlx_slots(SimpleNamespace(models = {"v": vision}, active_model_name = "v"), 2)
+ assert "mlx_growth" not in vision
+ inf._note_mlx_slots(SimpleNamespace(models = {}, active_model_name = None), 2)
+ import inspect
+
+ source = inspect.getsource(inf._load_model_impl)
+ assert "_note_mlx_slots(backend, backend.set_parallel_slots(_n_parallel))" in source
+ assert (
+ '_mlx_cache_bounded(\n _resident_entry, mlx_chat_need[1]["slots"]' in source
+ )
+
+
+def test_the_mlx_diffusers_fallback_forgets_the_mlx_price():
+ import inspect
+ source = inspect.getsource(inf._mlx_fallback_load_gate)
+ assert (
+ source.index("refuse_unless_fits(")
+ < source.index("forget_media_price(DIFFUSION, cancel)")
+ < source.index("return start()")
+ )
+
+
+def test_an_mlx_load_is_credited_what_it_replaces_and_more_rows_reprice_it():
+ import inspect
+
+ source = inspect.getsource(inf._load_model_impl)
+ shortcut = source.index('logger.info(f"Model already loaded (Unsloth)')
+ assert (
+ "not _mlx_slots_outgrow_price(backend, _n_parallel)" in source[shortcut - 1200 : shortcut]
+ )
+ inherit = source.index("# A reload for more rows keeps the context the resident was asked for.")
+ assert inherit < shortcut and '"max_seq_length_requested"' in source[inherit : inherit + 400]
+ credit = source.index(
+ "mlx_credit_mib = await asyncio.to_thread(mlx_replaced_credit_mib, backend)"
+ )
+ assert "gguf_replaced_credit_mib" not in source
+ assert credit < source.index("mlx_credit_mib = mlx_credit_mib", credit)
+ settle = source.index("settle_unpriced_chat_beside_media, multi_model_mode")
+ assert settle < source.index("backend.unload_model, config.identifier", settle)
+
+
def test_a_pinned_non_gguf_chat_load_keeps_its_requested_gpus(monkeypatch):
# Only GGUF and native audio resolve their GPUs: a Transformers load pinned to GPU 1 still targets GPU 1.
pinned = inf._LoadPlacement([1], None, False, None)