Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep image and video models loaded beside chat models on Apple Silicon (#15693)
Long Yixing556b28bb40d2
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.pyindex abc2b54c268..6614f90ded0 100644--- a/studio/backend/core/inference/gpu_arbiter.py+++ b/studio/backend/core/inference/gpu_arbiter.py@@ -844,6 +844,9 @@ def would_evict(in_flight = getattr(gpu_residency, "chat_load_in_flight", None)if callable(in_flight) and in_flight():return True # coreside_locked evicts a chat that is still loading+ unpriced = getattr(gpu_residency, "chat_growth_unpriced", None)+ if callable(unpriced) and unpriced():+ return Trueanswer = fits(owner, need_mib)ok, info = answer if isinstance(answer, tuple) else (answer, {})return not (ok or _idle_media_frees_room(gpu_residency, owner, info))diff --git a/studio/backend/core/inference/gpu_residency.py b/studio/backend/core/inference/gpu_residency.pyindex c8e29dc029b..7577edc874a 100644--- a/studio/backend/core/inference/gpu_residency.py+++ b/studio/backend/core/inference/gpu_residency.py@@ -34,6 +34,19 @@ class MediaNeed(dict):self.kind = kind+class ChatNeed(dict):+ """Chat load {GPU: MiB}, priced against free memory plus ``credit_mib`` released by the model it replaces."""++ def __init__(+ self,+ per_gpu: dict,+ *,+ credit_mib: int = 0,+ ):+ super().__init__({int(gpu): int(mib) for gpu, mib in per_gpu.items()})+ self.credit_mib = int(credit_mib)++def current_policy() -> str:"""Keep multiple models loaded, read as the owner. Never call this under the arbiter lock (it reads a database)."""try:@@ -233,29 +246,30 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,estimate_safetensors_dense_mib,file_size_mib,)- from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS+ from core.inference.sd_cpp_engine import ENGINE_DIFFUSERS, ENGINE_MLXrepo, name = request.model_path, request.gguf_filenameif getattr(request, "text_encoder_file", None) or getattr(request, "vae_file", None):return None # supplied companions are priced from their headers at load time onlykind = resolve_model_kind(name, request.model_kind)fam = detect_family_for_pick(repo, name, request.family_override)- if (- fam is None- or predict_engine(- fam,- model_kind = kind,- gpu_ordinal = gpu_ordinal,- gguf_filename = name,- with_loras = has_active_loras(getattr(request, "loras", None)),- # An explicit precision keeps a default-routed Apple Silicon load on diffusers, as the load does.- mlx_blocker = mlx_load_blocker(- transformer_quant = getattr(request, "transformer_quant", None),- text_encoder_quant = getattr(request, "text_encoder_quant", None),- ),- )- != ENGINE_DIFFUSERS- ):+ if fam is None:+ return None+ engine = predict_engine(+ fam,+ model_kind = kind,+ gpu_ordinal = gpu_ordinal,+ gguf_filename = name,+ engine = getattr(request, "engine", None),+ with_loras = has_active_loras(getattr(request, "loras", None)),+ mlx_blocker = mlx_load_blocker(+ transformer_quant = getattr(request, "transformer_quant", None),+ text_encoder_quant = getattr(request, "text_encoder_quant", None),+ ),+ )+ if engine == ENGINE_MLX:+ return _mlx_image_need(request, fam)+ if engine != ENGINE_DIFFUSERS:return Nonemib = 1024 * 1024if kind == "pipeline":@@ -310,6 +324,40 @@ def _image_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,return weights, estimate_image_runtime_mib(width = None, height = None, family = hint)+def _mlx_image_need(request: Any, fam: Any) -> Optional[tuple[int, int]]:+ """(weights MiB, decode headroom MiB) of an MLX image load, from headers."""+ from core.inference.diffusion_families import resolve_base_repo+ from core.inference.mlx_diffusion_backend import (+ decode_headroom_bytes,+ fetch_base_snapshot,+ get_mlx_diffusion_backend,+ plan_for_load,+ resolve_variant_base,+ )++ gguf = _cached_pick(request.model_path, request.gguf_filename)+ if gguf is None:+ return None+ base = resolve_base_repo(fam, request.base_repo)+ if not request.base_repo:+ base = resolve_variant_base(+ request.model_path,+ fam,+ None,+ request.gguf_filename,+ fallback = base,+ local_files_only = True,+ )+ base_dir = fetch_base_snapshot(base, None, local_files_only = True)+ headroom = decode_headroom_bytes(get_mlx_diffusion_backend()._rt())+ plan = plan_for_load(gguf, base_dir, headroom_bytes = headroom)+ if plan.source != "plan":+ return None+ mib = 1024 * 1024+ # Tiled peak suffices: the renderer tiles once the pool cannot take an untiled decode.+ return (plan.planned_bytes - headroom) // mib, -(-headroom // mib)++def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int, int]]:"""(weights MiB, headroom MiB) for a video load: checkpoint file sizes plus the family's companion table."""from core.inference.diffusion import DiffusionBackend as DB@@ -357,12 +405,57 @@ def _video_need(request: Any, gpu_ordinal: Optional[int]) -> Optional[tuple[int,)+def _on_metal() -> bool:+ try:+ from utils.hardware import DeviceType, get_device+ return get_device() == DeviceType.MLX+ except Exception: # noqa: BLE001+ return False+++def shared_metal_headroom_bytes() -> Optional[int]:+ """Headroom on Apple's GPU less resident MLX chat growth; None unless priced media is on Metal."""+ if not metal_media_priced():+ return None+ import psutil+ from core.inference.llama_cpp import _apple_wired_headroom_bytes++ wired = _apple_wired_headroom_bytes()+ if wired is None:+ return None+ pool = min(wired, int(psutil.virtual_memory().available))+ return max(0, pool - (_mlx_chat_unallocated_mib() or 0) * 1024 * 1024)+++def metal_media_priced() -> bool:+ """Whether a media model is kept on Metal at a price: one that reserved the tiled decode, so it must tile."""+ return _on_metal() and bool(media_held_mib())+++def _metal_pool() -> tuple[Optional[dict[int, dict]], Optional[str]]:+ """Unified memory as GPU 0: wired headroom capped at available RAM (both exclude loaded models)."""+ try:+ import psutil+ from core.inference.llama_cpp import _apple_wired_headroom_bytes++ wired = _apple_wired_headroom_bytes()+ available = int(psutil.virtual_memory().available)+ except Exception as exc: # noqa: BLE001+ logger.warning("gpu_residency: Metal memory probe failed: %s", exc)+ return None, "probe_error"+ if wired is None or available <= 0:+ return None, "probe_error"+ return {0: {"free_mib": min(wired, available) // (1024 * 1024), "metal": True}}, None++def _free_mib_by_gpu() -> tuple[Optional[dict[int, dict]], Optional[str]]:"""{physical id: {free_mib, total_mib}} or (None, reason) when the host can't be priced."""try:from utils.hardware import DeviceType, get_device, get_visible_gpu_utilization++ if get_device() == DeviceType.MLX:+ return _metal_pool()if get_device() != DeviceType.CUDA:- # MLX and other unified-memory hosts share the pool with the CPU: co-residency there is deferred.return None, "non_accelerator"util = get_visible_gpu_utilization() or {}except Exception as exc: # noqa: BLE001@@ -440,6 +533,9 @@ def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dictfor name, need in records.items()if name != owner and gpu in need)+ if row.get("metal"):+ free -= _mlx_chat_unallocated_mib() or 0+ free += getattr(need_mib, "credit_mib", 0)if own is not None and gpu in own:# Reloading the same owner frees what it holds now.free += max(own[gpu] - getattr(own, "headroom_mib", 0), 0)@@ -450,7 +546,7 @@ def can_load_alongside(owner: str, need_mib: Optional[dict]) -> tuple[bool, dictif info["short_mib"]:return False, infoif getattr(need_mib, "kind", None) == "image":- for gpu in need_mib:+ for gpu in [g for g in need_mib if not rows[g].get("metal")]:if not _keeps_transformer_resident(need_mib, gpu, info["free_mib"][gpu], rows[gpu]["total_mib"]):@@ -474,6 +570,90 @@ def chat_load_in_flight() -> bool:return False+def chat_growth_unpriced() -> bool:+ """On Metal, whether a resident MLX chat model may still grow its KV / prompt cache by an unpriced amount."""+ return _on_metal() and _mlx_chat_unallocated_mib() is None+++def _mlx_chat_unallocated_mib() -> Optional[int]:+ """MiB resident MLX chat models priced but have not allocated yet; None when any is unpriced."""+ total = 0+ try:+ from core.inference import model_slots, orchestrator+ for orch in [+ orchestrator._inference_backend,+ *(s.orchestrator for s in model_slots.resident()),+ ]:+ mib = _orch_unallocated_mib(orch)+ if mib is None:+ return None+ total += mib+ return total+ except Exception: # noqa: BLE001+ return None+++def _orch_unallocated_mib(orch) -> Optional[int]:+ total = 0+ for entry in list((getattr(orch, "models", None) or {}).values()):+ if isinstance(entry, dict) and entry.get("is_mlx"):+ growth = entry.get("mlx_growth")+ if growth is None:+ return None+ rows = max(1, int(orch.effective_parallel_slots), int(growth.get("slots", 1)))+ priced = growth["mib"] + (rows - 1) * growth["row_mib"]+ at_load, now = growth.get("footprint_mib"), mlx_worker_footprint_mib(orch)+ grown = now - at_load if at_load is not None and now is not None else 0+ total += max(0, priced - max(0, grown))+ return total+++def mlx_replaced_credit_mib(orch) -> int:+ """MiB a load into ``orch`` gets back from the MLX model it replaces (each load spawns a fresh worker)."""+ try:+ if not any(+ isinstance(e, dict) and e.get("is_mlx")+ for e in list((getattr(orch, "models", None) or {}).values())+ ):+ return 0+ return (mlx_worker_footprint_mib(orch) or 0) + (_orch_unallocated_mib(orch) or 0)+ except Exception: # noqa: BLE001+ return 0+++def mlx_cache_bounded(entry: dict, slots: int = 1) -> bool:+ """Whether the MLX cache stops at its priced context; batched vision rows drop the window."""+ if slots > 1 and entry.get("context_unbounded_when_batched"):+ return False+ return entry.get("context_length_enforced") is True or bool(entry.get("mlx_context_budget"))+++def mlx_slots_outgrow_price(orch, n_parallel) -> bool:+ """Whether serving ``n_parallel`` rows grows a resident MLX model beside media past what it was admitted at."""+ try:+ from core.inference.llama_server_args import clamp_parallel_slots+ entry = (getattr(orch, "models", None) or {}).get(orch.active_model_name or "") or {}+ return bool(+ _on_metal()+ and entry.get("is_mlx")+ and media_held_mib()+ and clamp_parallel_slots(n_parallel) > int(orch.effective_parallel_slots)+ )+ except Exception: # noqa: BLE001+ return False+++def mlx_worker_footprint_mib(orch) -> Optional[int]:+ """MiB the MLX worker process holds now, Metal buffers included, or None when unreadable."""+ proc = getattr(orch, "_proc", None)+ if proc is None or proc.pid is None:+ return None+ from core.inference.llama_cpp import LlamaCppBackend++ usage = LlamaCppBackend._darwin_rusage(proc.pid)+ return None if usage is None else usage[1] // (1024 * 1024)++_CHAT_LOADING_MESSAGE = ("Unsloth Studio is loading a chat model on this GPU. Retry once it has finished loading.")@@ -712,6 +892,38 @@ def media_fit_kwargs(return kwargs+def settle_unpriced_chat_beside_media(policy: str) -> None:+ """Media beside an unpriced chat model: When they fit evicts it, Always refuses."""+ if policy not in ("fit", "always") or not _on_metal():+ return+ from utils.account_context import current_account_id++ with arb._lock:+ arb._sync_from_view()+ media = [name for name in MEDIA_OWNERS if name in arb._residents]+ if not media:+ return+ if policy == "always":+ raise arb.refusal_for(+ [slot for name in media for slot in arb.resident_slots(name)],+ arb.KIND_LABELS[media[0]],+ )+ for name in media:+ _evict_resident(name, current_account_id(), None, [])+++def forget_media_price(owner: str, cancel: Any = None) -> None:+ """Drop ``owner``'s price; gives up on ``cancel`` since an evictor holding the lock awaits our drain."""+ while not arb._lock.acquire(timeout = 0.05):+ if cancel is not None and cancel.is_set():+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)+ try:+ arb._media_needs.pop(owner, None)+ finally:+ arb._lock.release()++def refuse_unless_fits(owner: str,*,@@ -845,6 +1057,8 @@ def coreside_locked(if policy == "always":raise ChatLoadInFlightRefusal()evict_or_refuse([arb.CHAT])+ if owner in MEDIA_OWNERS and arb.CHAT in kept and chat_growth_unpriced():+ evict_or_refuse([arb.CHAT])if owner == arb.CHAT and not need_mib:if evict_first:# No estimate before launch (a non-GGUF load, or a load with no retry to make room).@@ -856,6 +1070,10 @@ def coreside_locked(returnif need_mib is None or evict_first:return evict_or_refuse(list(kept))+ if isinstance(need_mib, ChatNeed):+ evict_or_refuse([n for n in kept if n in MEDIA_OWNERS and n not in _media_records()])+ if not kept:+ returnfits, info = can_load_alongside(owner, need_mib)if fits:return@@ -863,7 +1081,15 @@ def coreside_locked(raise need_free_refusal(info, [slot for n in kept for slot in arb.resident_slots(n)])if not allow_evict:raise arb.GpuOwnerBusyError(kept[-1])- _free_steps(owner, target, f"GPU memory for {owner}", evicted, need_mib = need_mib, acting = acting)+ _free_steps(+ owner,+ target,+ f"GPU memory for {owner}",+ evicted,+ need_mib = need_mib,+ acting = acting,+ media_only = owner == arb.CHAT and _on_metal(),+ )if not can_load_alongside(owner, need_mib)[0]:# Still short: unload the rest on these GPUs, as Off does, and let the planner offload.for name in [diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex fb87fce8546..2fb74c94790 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -599,6 +599,20 @@ def _apple_wired_headroom_bytes() -> Optional[int]:return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))+def _apple_budget_beside_media_mib(held_mib: int, unallocated_mib: int) -> int:+ """Unified-memory budget beside resident media: price off the working set, unallocated part off available RAM."""+ from utils.hardware.hardware import _metal_recommended_working_set_bytes+ import psutil++ mib = 1024 * 1024+ working_set = _metal_recommended_working_set_bytes() or int(psutil.virtual_memory().total)+ available = int(psutil.virtual_memory().available)+ left = working_set // mib - held_mib+ if available > 0:+ left = min(left, available // mib - unallocated_mib)+ return max(0, int(left * _APPLE_UNIFIED_MEMORY_FRACTION))++def _gpu_plan_mib(total_bytes: int, on: list[int], gpus) -> dict[int, int]:"""MiB this load holds per GPU: an even share, capped at what each card had free, since--fit on offloads the rest to CPU."""@@ -25549,6 +25563,26 @@ class LlamaCppBackend:# Unified-memory budget (0 off Apple Silicon) for the no-GPU Metal cap below._apple_budget_mib = self._apple_metal_memory_budget_bytes() // (1024 * 1024)+ _metal_media_mib, _metal_media_unallocated_mib = (+ self._metal_media_mib() if _apple_budget_mib > 0 else (0, 0)+ )+ if _metal_media_mib:+ # 0 would mean "not Apple Silicon" and skip every Metal check.+ _apple_budget_mib = max(+ 1,+ min(+ _apple_budget_mib,+ _apple_budget_beside_media_mib(+ _metal_media_mib, _metal_media_unallocated_mib+ ),+ ),+ )+ _metal_media_wired_mib = int(+ _metal_media_unallocated_mib * _APPLE_WIRED_CEILING_FRACTION+ )++ def _apple_wired_left_mib(wired_bytes: int) -> int:+ return max(0, wired_bytes // (1024 * 1024) - _metal_media_wired_mib)if tensor_parallel and tp_gpus:# The planner can shrink ctx/KV, not weights or the MTP reserve.@@ -26392,8 +26426,7 @@ class LlamaCppBackend:)# Past free memory macOS swaps; past the wired limit the machine panics._apple_wired_mib = int(- self._apple_metal_wired_ceiling_bytes()- // (1024 * 1024)+ _apple_wired_left_mib(self._apple_metal_wired_ceiling_bytes())* max(0.0, 1.0 - _flat_mtp_reserve))if (@@ -26447,8 +26480,7 @@ class LlamaCppBackend:# Past the wired limit Metal refuses, so it is the harder ceiling when readable._fit_wired_mib = (int(- self._apple_metal_wired_ceiling_bytes()- // (1024 * 1024)+ _apple_wired_left_mib(self._apple_metal_wired_ceiling_bytes())* max(0.0, 1.0 - _flat_mtp_reserve))if _vram_fit_ctx@@ -26497,6 +26529,8 @@ class LlamaCppBackend:# them, so it must see the same wired limit the refusals do.if not _caller_owns_budget and not _paravirtual_cpu_forced:_metal_fit_target_floor_mib = self._metal_fit_target_floor_mib()+ if _metal_fit_target_floor_mib > 0:+ _metal_fit_target_floor_mib += _metal_media_wired_mibif (intent.refuse_partial_gpu_fit@@ -26513,7 +26547,8 @@ class LlamaCppBackend:wired_bytes = self._apple_metal_wired_headroom_bytes()if wired_bytes is not None:wired_mib = int(- wired_bytes // (1024 * 1024) * max(0.0, 1.0 - _flat_mtp_reserve)+ _apple_wired_left_mib(wired_bytes)+ * max(0.0, 1.0 - _flat_mtp_reserve))budget_mib = min(budget_mib, wired_mib)need_mib = math.ceil(@@ -33715,6 +33750,21 @@ class LlamaCppBackend:return media_unallocated_mib()+ def _metal_media_mib(self) -> tuple[int, int]:+ """(price, unallocated) MiB of media resident on Apple's GPU, plus MLX chat growth beside them."""+ if self not in _serving_backends_snapshot():+ return 0, 0+ from core.inference.gpu_residency import (+ _mlx_chat_unallocated_mib,+ media_held_mib,+ media_unallocated_mib,+ )++ held = media_held_mib().get(0, 0)+ if not held:+ return 0, 0+ return held, media_unallocated_mib().get(0, 0) + (_mlx_chat_unallocated_mib() or 0)+def _note_server_pid(self, pid: int) -> None:if getattr(self, "_owns_pidfile", True):self._record_server_pid(pid)diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.pyindex 925962551b9..2702f1c2f87 100644--- a/studio/backend/core/inference/mlx_diffusion_backend.py+++ b/studio/backend/core/inference/mlx_diffusion_backend.py@@ -480,7 +480,7 @@ class MfluxRuntime:return imagedef free_working_set_bytes(self) -> Optional[int]:- """Metal's recommended working set minus what MLX holds live now (the resident model), or None."""+ """Metal working set minus live MLX, capped by system headroom less a chat worker's reserved growth."""import mlx.core as mxtry:@@ -490,13 +490,26 @@ class MfluxRuntime:return Noneif budget <= 0:return None- return budget - int(mx.get_active_memory())+ free = budget - int(mx.get_active_memory())+ try:+ from core.inference.gpu_residency import shared_metal_headroom_bytes+ shared = shared_metal_headroom_bytes()+ except Exception: # noqa: BLE001 -- this process's reading alone, as before+ shared = None+ return free if shared is None else min(free, shared)def _plan_vae_tiling(self, model: Any, width: int, height: int) -> None:from mflux.models.common.vae.tiling_config import TilingConfig+ try:+ from core.inference.gpu_residency import metal_media_priced++ # Price reserved the tiled decode; a chat model may join mid-generation on it.+ priced = metal_media_priced()+ except Exception: # noqa: BLE001+ priced = Falsetiled = TilingConfig.may_tile_implicitly(model) and vae_tiling_needed(- width, height, self.free_working_set_bytes()+ width, height, 0 if priced else self.free_working_set_bytes())model.tiling_config = TilingConfig() if tiled else Noneself.last_vae_tiled = tileddiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex ed5f9dddc54..72e78ac2b0a 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -11812,13 +11812,16 @@ def _chat_fit_kwargs(placement,policy: Optional[str] = None,credit_gb: Optional[dict] = None,+ mlx_need_mib: Optional[int] = None,+ mlx_credit_mib: int = 0,) -> dict:"""How the CHAT acquire shares the GPU under When they fit / Always.- audio.cpp and native audio are priced from the placement estimate. A GGUF launch prices itself (it refuses a- partial fit and load_model_gated makes room). Anything else has no estimate before launch: When they fit- evicts first, Always tries the load and turns an out-of-memory failure into its refusal. ``credit_gb`` is what- the outgoing primary frees per GPU when this load replaces it.+ audio.cpp and native audio are priced from the placement estimate, an MLX load on Apple Silicon from+ ``mlx_need_mib``. A GGUF launch prices itself (it refuses a partial fit and load_model_gated makes room).+ Anything else has no estimate before launch: When they fit evicts first, Always tries the load and turns an+ out-of-memory failure into its refusal. ``credit_gb`` is what the outgoing primary frees per GPU when this+ load replaces it."""if policy is None:from core.inference.gpu_residency import current_policy@@ -11840,11 +11843,42 @@ def _chat_fit_kwargs(int(g): int((placement.native_required_gb / len(gpus) - credit.get(int(g), 0.0)) * 1024)for g in gpus}- elif policy == "fit" and not (config.is_gguf and _primary_can_retry_fit.get()):+ elif mlx_need_mib is not None:+ from core.inference.gpu_residency import ChatNeed+ kwargs["need_mib"] = ChatNeed({0: int(mlx_need_mib)}, credit_mib = mlx_credit_mib)+ elif not (config.is_gguf and _primary_can_retry_fit.get()) and (+ policy == "fit" or (not config.is_gguf and _unified_metal_host())+ ):+ # Unified memory pages past the budget: Always gets no OOM to refuse on.kwargs["evict_first"] = Truereturn kwargs+def _unified_metal_host() -> bool:+ from core.inference.gpu_residency import _on_metal+ return _on_metal()+++def _mlx_cache_bounded(entry: dict, slots: int = 1) -> bool:+ from core.inference.gpu_residency import mlx_cache_bounded+ return mlx_cache_bounded(entry, slots)+++def _note_mlx_slots(backend, slots: int) -> None:+ """A batch opened at a wider setting keeps its width after the setting drops, so the reservation never shrinks."""+ entry = backend.models.get(backend.active_model_name or "") or {}+ growth = entry.get("mlx_growth")+ if isinstance(growth, dict):+ growth["slots"] = max(int(growth.get("slots", 1)), int(slots))+ if not _mlx_cache_bounded(entry, growth["slots"]):+ entry.pop("mlx_growth")+++def _mlx_slots_outgrow_price(backend, n_parallel) -> bool:+ from core.inference.gpu_residency import mlx_slots_outgrow_price+ return mlx_slots_outgrow_price(backend, n_parallel)++def _outgoing_chat_credit_gb(placement, policy: str, replacing: bool, availability: "Optional[_NativeAudioAvailability]") -> tuple[Optional[dict], Optional[dict]]:@@ -14707,6 +14741,118 @@ def _mlx_estimate_drafter(return None, None, True+class _MlxLoadEstimate(NamedTuple):+ breakdown: Any+ fitted_ctx: Optional[int]+ offloadable: tuple+ retains_history: bool+ offloaded: bool = False+++def _mlx_load_estimate(+ request, config, model_identifier+) -> tuple[Optional[str], Optional[_MlxLoadEstimate]]:+ """``(refusal reason, None)`` or ``(None, estimate)`` of what an MLX load of *request* occupies."""+ if not _mlx_estimate_available():+ return "not_gguf", None+ from core.inference.native_audio import is_native_audio_model++ if getattr(config, "audio_cpp", None) is not None or is_native_audio_model(model_identifier):+ return "not_gguf", None+ if getattr(config, "is_lora", False):+ return "unsizable", None+ model_dir = _local_mlx_model_dir(config)+ if not model_dir:+ return "not_downloaded", None+ from core.inference.mlx_inference import (+ mlx_kv_quant_is_refused,+ mlx_vlm_snapshot_store_available,+ )+ from core.inference.mlx_memory import mlx_memory_breakdown++ load_in_4bit = _mlx_estimate_load_in_4bit(config, request)+ kv_bits = _mlx_estimate_kv_bits(request.mlx_kv_quant, request.mlx_kv_bits)+ # A width the cache takes makes the window a budget, pinned or fit.+ if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):+ kv_bits = None+ named_ctx = request.max_seq_length or 0+ offload, offloadable, offload_baseline = _mlx_estimate_offload(+ request, config, model_identifier, model_dir, load_in_4bit, kv_bits+ )+ drafter, fitted_ctx, speculates = _mlx_estimate_drafter(+ request,+ model_identifier,+ model_dir,+ _mlx_estimate_ceiling(model_dir),+ load_in_4bit,+ offload,+ offload_baseline,+ )+ route = {**({"vision": True} if speculates else {}), **offload}+ if not named_ctx and drafter is None:+ fitted_ctx = _mlx_estimate_fitted_context(config, model_dir, load_in_4bit, kv_bits, **route)+ priced_ctx = named_ctx or fitted_ctx or _mlx_estimate_ceiling(model_dir)+ if not priced_ctx:+ return "unsizable", None+ breakdown = mlx_memory_breakdown(+ model_dir,+ n_ctx = priced_ctx,+ kv_bits = kv_bits,+ load_in_4bit = load_in_4bit,+ **route,+ **({} if drafter is None else {"drafter": drafter}),+ )+ if breakdown is None:+ return "unsizable", None+ retains_history = (+ not (getattr(config, "is_vision", False) or speculates)+ or mlx_vlm_snapshot_store_available()+ )+ return None, _MlxLoadEstimate(+ breakdown, fitted_ctx, offloadable, retains_history, "offload" in offload+ )+++def _mlx_chat_need(+ request,+ config,+ model_identifier,+ slots: int = 1,+) -> Optional[tuple[int, dict]]:+ """``(need MiB, growth)`` of an MLX chat load serving ``slots`` rows, or None; rows prefill as one batch."""+ if getattr(config, "is_gguf", False) or not _unified_metal_host():+ return None+ try:+ _, estimate = _mlx_load_estimate(request, config, model_identifier)+ # Offload workspaces allocate at prefill but are priced as weights.+ if estimate is None or estimate.offloaded:+ return None+ from core.inference.mlx_inference import mlx_memory_budget++ reserve = 0+ if estimate.retains_history:+ without, within = (+ mlx_memory_budget(retains_history = False),+ mlx_memory_budget(retains_history = True),+ )+ reserve = max(0, (without or 0) - (within or 0))+ except Exception as exc:+ logger.info("MLX load of %s left unpriced: %s", model_identifier, exc)+ return None+ from core.inference.llama_server_args import clamp_parallel_slots++ mib = 1024 * 1024+ one_row = -(-(int(estimate.breakdown.total_bytes) + reserve) // mib)+ row = -(-int(estimate.breakdown.kv_bytes + estimate.breakdown.compute_bytes) // mib)+ slots = clamp_parallel_slots(slots)+ growth = {+ "mib": max(0, one_row - int(estimate.breakdown.weights_bytes) // mib),+ "row_mib": row,+ "slots": slots,+ }+ return one_row + (slots - 1) * row, growth++def _mlx_estimate_available() -> bool:"""Whether this host would actually run the load through MLX."""try:@@ -18453,6 +18599,21 @@ async def _load_model_impl(if not (request.gguf_variant or is_direct_gguf_request):_inherit_resident_load_in_4bit(backend, request, model_identifier)await asyncio.to_thread(reap_dead_worker, backend)+ if (+ not request.max_seq_length+ and _same_loaded_identifier(backend.active_model_name, model_identifier)+ and _mlx_slots_outgrow_price(backend, _n_parallel)+ ):+ # A reload for more rows keeps the context the resident was asked for.+ request.max_seq_length = (+ int(+ (backend.models.get(backend.active_model_name) or {}).get(+ "max_seq_length_requested"+ )+ or 0+ )+ or request.max_seq_length+ )if (_same_loaded_identifier(backend.active_model_name, model_identifier)and (backend.models.get(backend.active_model_name) or {}).get("engine", "auto")@@ -18469,10 +18630,11 @@ async def _load_model_impl()# Without this the route reports already_loaded and nothing moves.and _resident_audio_placement_matches(backend, request)+ and not _mlx_slots_outgrow_price(backend, _n_parallel)):api_monitor.discard(_load_event) # nothing loaded, no monitor rowlogger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")- backend.set_parallel_slots(_n_parallel)+ _note_mlx_slots(backend, backend.set_parallel_slots(_n_parallel))# A no-op Unsloth load of a preview-owned checkpoint still claims it._set_preview_resident(None)inference_config = load_inference_config(backend.active_model_name)@@ -18822,6 +18984,7 @@ async def _load_model_impl(logger.warning("Loading an oversized GGUF: %s", _host_offload_warning)expected_native_owner = None+ mlx_chat_need = Nonepost_media_handoff_gpu = Nonepost_media_handoff_needed_gb = Nonepost_chat_handoff_expected_free_gb = None@@ -18875,8 +19038,24 @@ async def _load_model_impl()if post_chat_handoff_expected_free_gb is None:post_chat_handoff_expected_free_gb = chat_wait_gb+ mlx_chat_need = await asyncio.to_thread(+ _mlx_chat_need, request, config, model_identifier, _n_parallel+ )+ mlx_credit_mib = 0+ if mlx_chat_need is not None:+ from core.inference.gpu_residency import mlx_replaced_credit_mib++ # No llama-server credit: its footprint mixes CPU-only buffers with Metal ones.+ mlx_credit_mib = await asyncio.to_thread(mlx_replaced_credit_mib, backend)handoff_kwargs.update(- _chat_fit_kwargs(config, placement, multi_model_mode, outgoing_chat_gb)+ _chat_fit_kwargs(+ config,+ placement,+ multi_model_mode,+ outgoing_chat_gb,+ mlx_need_mib = None if mlx_chat_need is None else mlx_chat_need[0],+ mlx_credit_mib = mlx_credit_mib,+ ))await asyncio.to_thread(acquire_for_request,@@ -19255,6 +19434,29 @@ async def _load_model_impl(_resident_entry["gpu_ids_requested"] = (list(request.gpu_ids) if request.gpu_ids else None)+ if mlx_chat_need is not None and _mlx_cache_bounded(+ _resident_entry, mlx_chat_need[1]["slots"]+ ):+ from core.inference.gpu_residency import mlx_worker_footprint_mib+ _resident_entry["mlx_growth"] = {+ **mlx_chat_need[1],+ "footprint_mib": mlx_worker_footprint_mib(backend),+ }+ elif mlx_chat_need is not None:+ # Admitted at a price the loaded cache does not keep to: settle as an unpriced load would have.+ from core.inference.gpu_arbiter import (+ GpuBusyForAnotherAccountError,+ MultiModelAlwaysRefusal,+ )+ from core.inference.gpu_residency import settle_unpriced_chat_beside_media+ try:+ await asyncio.to_thread(settle_unpriced_chat_beside_media, multi_model_mode)+ except (MultiModelAlwaysRefusal, GpuBusyForAnotherAccountError) as exc:+ await asyncio.to_thread(backend.unload_model, config.identifier)+ await asyncio.to_thread(backend._shutdown_subprocess, 5.0)+ if isinstance(exc, GpuBusyForAnotherAccountError):+ raise exc.as_http_exception() from exc+ raiseif request._advertised_id:backend._openai_advertised_id = request._advertised_id@@ -20618,64 +20820,15 @@ async def estimate_memory(if config is None:return EstimateMemoryResponse(available = False, reason = "unsizable")if not getattr(config, "is_gguf", False):- if not _mlx_estimate_available():- return EstimateMemoryResponse(available = False, reason = "not_gguf")- from core.inference.native_audio import is_native_audio_model-- if getattr(config, "audio_cpp", None) is not None or is_native_audio_model(- model_identifier- ):- return EstimateMemoryResponse(available = False, reason = "not_gguf")- if getattr(config, "is_lora", False):- return EstimateMemoryResponse(available = False, reason = "unsizable")- model_dir = _local_mlx_model_dir(config)- if not model_dir:- return EstimateMemoryResponse(available = False, reason = "not_downloaded")- from core.inference.mlx_inference import mlx_kv_quant_is_refused- from core.inference.mlx_memory import mlx_memory_breakdown-- mlx_load_in_4bit = _mlx_estimate_load_in_4bit(config, request)- mlx_kv_bits = _mlx_estimate_kv_bits(request.mlx_kv_quant, request.mlx_kv_bits)- # A width the cache takes applies under a pin or a fit alike: the window is then a budget.- if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):- mlx_kv_bits = None- mlx_named_ctx = request.max_seq_length or 0- mlx_offload, mlx_offloadable, mlx_offload_baseline = _mlx_estimate_offload(- request, config, model_identifier, model_dir, mlx_load_in_4bit, mlx_kv_bits- )- mlx_drafter, mlx_fitted_ctx, mlx_speculates = _mlx_estimate_drafter(- request,- model_identifier,- model_dir,- _mlx_estimate_ceiling(model_dir),- mlx_load_in_4bit,- mlx_offload,- mlx_offload_baseline,- )- mlx_route = {**({"vision": True} if mlx_speculates else {}), **mlx_offload}- if not mlx_named_ctx and mlx_drafter is None:- mlx_fitted_ctx = _mlx_estimate_fitted_context(- config, model_dir, mlx_load_in_4bit, mlx_kv_bits, **mlx_route- )- mlx_priced_ctx = mlx_named_ctx or mlx_fitted_ctx or _mlx_estimate_ceiling(model_dir)- if not mlx_priced_ctx:- return EstimateMemoryResponse(available = False, reason = "unsizable")- mlx_breakdown = mlx_memory_breakdown(- model_dir,- n_ctx = mlx_priced_ctx,- kv_bits = mlx_kv_bits,- load_in_4bit = mlx_load_in_4bit,- **mlx_route,- **({} if mlx_drafter is None else {"drafter": mlx_drafter}),- )- if mlx_breakdown is None:- return EstimateMemoryResponse(available = False, reason = "unsizable")+ reason, mlx = _mlx_load_estimate(request, config, model_identifier)+ if mlx is None:+ return EstimateMemoryResponse(available = False, reason = reason)return EstimateMemoryResponse(**project_estimate_memory_response(build_memory_estimate(- dataclasses.replace(mlx_breakdown, offloadable = mlx_offloadable),+ dataclasses.replace(mlx.breakdown, offloadable = mlx.offloadable),quant_file_bytes = 0,- context_fitted = mlx_fitted_ctx,+ context_fitted = mlx.fitted_ctx,)))@@ -46821,6 +46974,7 @@ def _mlx_fallback_load_gate(request, gpu_ordinal, multi_model_mode):from core.inference.gpu_arbiter import DIFFUSIONfrom core.inference.gpu_residency import (ChatLoadInFlightRefusal,+ forget_media_price,media_fit_kwargs,refuse_unless_fits,)@@ -46837,7 +46991,8 @@ def _mlx_fallback_load_gate(request, gpu_ordinal, multi_model_mode):DIFFUSION,policy = fit.get("policy", "off"),gpu_ids = fit.get("gpu_ids"),- need_mib = fit.get("need_mib"),+ # The price is the MLX load's; this diffusers load has none.+ need_mib = None,cancel = cancel,)except (HTTPException, ChatLoadInFlightRefusal, RuntimeError) as exc:@@ -46855,6 +47010,7 @@ def _mlx_fallback_load_gate(request, gpu_ordinal, multi_model_mode):):if ref and _repo_is_in_the_hub_cache(ref) is not True:_note_load_fetched_with_a_request_token(ref, request.hf_token)+ forget_media_price(DIFFUSION, cancel)return start()return gatediff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex a402f71cd34..9593578f614 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -583,6 +583,13 @@ def _no_live_metal_wired_ceiling(monkeypatch):)[email protected](autouse = True)+def _not_on_metal(monkeypatch):+ """Keep co-residency verdicts off the host's device: on a Mac they take the Metal paths."""+ from core.inference import gpu_residency+ monkeypatch.setattr(gpu_residency, "_on_metal", lambda: False)++@pytest.fixture(autouse = True)def _no_leftover_generation_account(monkeypatch):"""Reset routes.video's _generation_account per test, since no route ever clears it."""diff --git a/studio/backend/tests/test_gpu_arbiter.py b/studio/backend/tests/test_gpu_arbiter.pyindex 4bcd94ea355..c52b70cf0f5 100644--- a/studio/backend/tests/test_gpu_arbiter.py+++ b/studio/backend/tests/test_gpu_arbiter.py@@ -6,6 +6,7 @@from __future__ import annotationsimport types+from contextlib import nullcontextimport pytest@@ -155,6 +156,87 @@ def test_image_load_beside_a_resident_chat_model_policy_matrix(cores, policy, neassert arb._media_needs.get(arb.DIFFUSION) == (need if policy != "off" else None)+ "policy, growth, outcome",+ [("always", 512, "join"), ("fit", None, "evict"), ("always", None, "refuse")],+)+def test_image_load_beside_a_chat_model_on_metal(cores, monkeypatch, policy, growth, outcome):+ import core.inference.gpu_residency as res++ monkeypatch.setattr(res, "_on_metal", lambda: True)+ monkeypatch.setattr(res, "_mlx_chat_unallocated_mib", lambda: growth)+ cores.fits()+ arb.acquire_for(arb.CHAT)+ assert arb.would_evict(arb.DIFFUSION, policy = policy, need_mib = {0: 4096}) is (outcome == "evict")+ with pytest.raises(arb.MultiModelAlwaysRefusal) if outcome == "refuse" else nullcontext():+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = policy, need_mib = {0: 4096})+ gone = {"join": set(), "evict": {arb.CHAT}, "refuse": {arb.DIFFUSION}}[outcome]+ assert {arb.CHAT, arb.DIFFUSION} - set(arb.residents()) == gone++[email protected]("metal", [True, False])+def test_a_short_chat_load_on_metal_frees_media_but_no_kept_chat_model(cores, monkeypatch, metal):+ import core.inference.gpu_residency as res+ from core.inference import model_slots++ monkeypatch.setattr(res, "_on_metal", lambda: metal)+ monkeypatch.setattr(res, "_mlx_chat_unallocated_mib", lambda: 0)+ monkeypatch.setattr(res, "can_load_alongside", lambda *a: (False, {"short_mib": {0: 1}}))+ asked = []+ monkeypatch.setattr(model_slots, "eviction_victims", lambda *a, **k: asked.append(a) or [])+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})+ arb.acquire_for(arb.CHAT, policy = "fit", need_mib = {0: 9000})+ assert arb.DIFFUSION not in arb.residents() and bool(asked) is not metal+ asked.clear()+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})+ assert asked++[email protected]("policy", ["fit", "always"])[email protected]("priced", [True, False])+def test_a_priced_mlx_chat_load_does_not_join_an_unpriced_image(cores, monkeypatch, policy, priced):+ import core.inference.gpu_residency as res++ monkeypatch.setattr(res, "can_load_alongside", lambda *a: (True, {}))+ arb.acquire_for(+ arb.DIFFUSION, lambda: None, policy = "fit" if priced else "off", need_mib = {0: 4096}+ )+ refused = policy == "always" and not priced+ with pytest.raises(arb.MultiModelAlwaysRefusal) if refused else nullcontext():+ arb.acquire_for(arb.CHAT, policy = policy, need_mib = res.ChatNeed({0: 2048}))+ assert (arb.DIFFUSION in arb.residents()) is (priced or refused)++[email protected]("policy", ["off", "fit", "always"])+def test_an_unpriced_chat_load_settles_beside_media_as_its_policy_says(cores, monkeypatch, policy):+ import core.inference.gpu_residency as res++ monkeypatch.setattr(res, "_on_metal", lambda: True)+ res.settle_unpriced_chat_beside_media(policy) # nothing resident+ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})+ with pytest.raises(arb.MultiModelAlwaysRefusal) if policy == "always" else nullcontext():+ res.settle_unpriced_chat_beside_media(policy)+ assert (arb.DIFFUSION in arb.residents()) is (policy != "fit")+ monkeypatch.setattr(res, "_on_metal", lambda: False)+ res.settle_unpriced_chat_beside_media("fit")+ assert (arb.DIFFUSION in arb.residents()) is (policy != "fit")+++def test_a_renderer_change_forgets_the_media_price(cores):+ import core.inference.gpu_residency as res++ arb.acquire_for(arb.DIFFUSION, lambda: None, policy = "fit", need_mib = {0: 4096})+ assert arb._media_needs.get(arb.DIFFUSION) == {0: 4096}+ res.forget_media_price(arb.DIFFUSION)+ assert arb.DIFFUSION not in arb._media_needs and arb.DIFFUSION in arb.residents()+ import threading++ cancel = threading.Event()+ cancel.set()+ with arb._lock, pytest.raises(RuntimeError):+ res.forget_media_price(arb.DIFFUSION, cancel)++@pytest.mark.parametrize("managed", [False, True])def test_always_refuses_an_image_load_that_does_not_fit_without_evicting(cores, managed):from utils.account_context import AccountContext, run_asdiff --git a/studio/backend/tests/test_gpu_residency_coexistence.py b/studio/backend/tests/test_gpu_residency_coexistence.pyindex 1250333cc82..fb7e69d0508 100644--- a/studio/backend/tests/test_gpu_residency_coexistence.py+++ b/studio/backend/tests/test_gpu_residency_coexistence.py@@ -266,7 +266,7 @@ class TestCanLoadAlongside(_ArbiterResetMixin, unittest.TestCase):gr.can_load_alongside(arb.VIDEO, None), (False, {"reason": "estimate_unavailable"}))self.assertEqual(gr.can_load_alongside(arb.VIDEO, {})[1]["reason"], "estimate_unavailable")- for device in (DeviceType.MLX, DeviceType.CPU, DeviceType.XPU):+ for device in (DeviceType.CPU, DeviceType.XPU):self.assertEqual(self._check(need, device = device), (False, {"reason": "non_accelerator"}))@@ -364,6 +364,90 @@ class TestCanLoadAlongside(_ArbiterResetMixin, unittest.TestCase):):self.assertTrue(self._check(_need({0: 1000}, kind = "video"))[0])+ def _metal(self, need, wired, available, *chat_growth):+ with (+ _cuda(DeviceType.MLX),+ patch.object(gr, "_mlx_chat_unallocated_mib", return_value = sum(chat_growth)),+ patch(+ "core.inference.llama_cpp._apple_wired_headroom_bytes",+ return_value = wired and wired * MIB,+ ),+ patch("psutil.virtual_memory", return_value = SimpleNamespace(available = available * MIB)),+ ):+ return gr.can_load_alongside(arb.VIDEO, need)++ def test_metal_is_one_pool_at_the_smaller_reading_less_idle_headroom(self):+ self._resident(arb.DIFFUSION, {0}, _need({0: 9000}, weights = 6000, headroom = 3000))+ free = 12288 - 3000+ for w, a in ((20480, 12288), (12288, 20480)):+ self.assertEqual(self._metal(_need({0: 1}, kind = "image"), w, a)[1]["free_mib"][0], free)+ self.assertTrue(self._metal(_need({0: free - gr.FIT_MARGIN_MIB}), 20480, 12288)[0])+ self.assertFalse(self._metal(_need({0: free - gr.FIT_MARGIN_MIB + 1}), 20480, 12288)[0])+ self.assertEqual(self._metal(_need({0: 1}), None, 12288)[1], {"reason": "probe_error"})+ self.assertEqual(self._metal(_need({0: 1}), w, a, 2500)[1]["free_mib"], {0: free - 2500})+ credited = gr.ChatNeed({0: 8192}, credit_mib = 12288)+ self.assertTrue(self._metal(credited, 6144, 6144, 4096)[0])+ self.assertFalse(self._metal(gr.ChatNeed({0: 8192}), 6144, 6144, 4096)[0])++ def test_mlx_chat_growth_adds_a_kv_cache_per_row_less_what_each_worker_grew(self):+ from core.inference import llama_cpp, model_slots, orchestrator++ def orch(growth, rows, pid):+ entry = {"is_mlx": True, "mlx_growth": growth} if growth else {"is_mlx": growth is None}+ proc = SimpleNamespace(pid = pid)+ return SimpleNamespace(models = {"m": entry}, effective_parallel_slots = rows, _proc = proc)++ def total(grown, primary, *kept):+ slots = [SimpleNamespace(orchestrator = o) for o in kept]+ with (+ patch.object(orchestrator, "_inference_backend", primary),+ patch.object(model_slots, "resident", lambda: slots),+ patch.object(gr, "mlx_worker_footprint_mib", lambda o: grown.get(o._proc.pid)),+ ):+ return gr._mlx_chat_unallocated_mib()++ g = {"mib": 512, "row_mib": 100, "footprint_mib": 2000}+ self.assertEqual(total({}, orch(g, 4, 7), orch({**g, "mib": 256}, 1, 8)), 1068)+ self.assertEqual(total({7: 2300}, orch(g, 4, 7), orch(False, 1, 9)), 512)+ self.assertEqual(total({7: 1900}, orch(g, 4, 7)), 812)+ self.assertEqual(total({7: 9000}, orch(g, 4, 7)), 0)+ self.assertEqual(total({}, orch({**g, "slots": 4}, 1, 7)), 812)+ with patch.object(llama_cpp.LlamaCppBackend, "_darwin_rusage", return_value = (1, 3 * MIB)):+ self.assertEqual(gr.mlx_worker_footprint_mib(orch(g, 1, 7)), 3)+ with patch.object(model_slots, "resident", side_effect = RuntimeError):+ self.assertIsNone(gr._mlx_chat_unallocated_mib())+ self.assertIsNone(total({}, orch(g, 1, 7), orch(None, 1, 8)))++ def test_a_replaced_mlx_model_credits_its_worker_and_reserved_growth(self):+ g = {"mib": 512, "row_mib": 100, "footprint_mib": 2000}++ def orch(entry, rows = 2):+ return SimpleNamespace(+ models = {"m": entry}, effective_parallel_slots = rows, _proc = SimpleNamespace(pid = 7)+ )++ with patch.object(gr, "mlx_worker_footprint_mib", lambda o: 2100):+ self.assertEqual(+ gr.mlx_replaced_credit_mib(orch({"is_mlx": True, "mlx_growth": g})), 2100 + 512+ )+ self.assertEqual(gr.mlx_replaced_credit_mib(orch({"is_mlx": True})), 2100)+ self.assertEqual(gr.mlx_replaced_credit_mib(orch({"is_mlx": False})), 0)++ def test_more_parallel_rows_beside_media_outgrow_the_admitted_price(self):+ o = SimpleNamespace(+ models = {"m": {"is_mlx": True}}, active_model_name = "m", effective_parallel_slots = 2+ )+ with (+ patch.object(gr, "_on_metal", return_value = True),+ patch.object(gr, "media_held_mib", return_value = {0: 9000}),+ ):+ self.assertTrue(gr.mlx_slots_outgrow_price(o, 4))+ self.assertFalse(gr.mlx_slots_outgrow_price(o, 2))+ with patch.object(gr, "media_held_mib", return_value = {}):+ self.assertFalse(gr.mlx_slots_outgrow_price(o, 4))+ with patch.object(gr, "_on_metal", return_value = False):+ self.assertFalse(gr.mlx_slots_outgrow_price(o, 4))+class TestPipelineNeed(_GpuCacheResetMixin, unittest.TestCase):def setUp(self):@@ -731,3 +815,54 @@ def test_single_file_image_need_reads_fp8_from_the_header(tmp_path, monkeypatch)save_file({"blocks.0.weight": torch.zeros(1024, 2048, dtype = torch.bfloat16)}, str(dense))# Both files are 4 MiB on disk: only the one that stores fp8 widens on load.assert need(plain) > need(dense)+++def test_an_mlx_routed_image_load_is_priced_from_its_quantization_plan(monkeypatch):+ import types++ import core.inference.diffusion_engine_router as router+ import core.inference.diffusion_families as dfam+ import core.inference.gpu_residency as gr+ import core.inference.mlx_diffusion_backend as mdb+ from core.inference.sd_cpp_engine import ENGINE_MLX++ picks = []+ monkeypatch.setattr(+ dfam, "detect_family_for_pick", lambda *a, **k: types.SimpleNamespace(name = "z")+ )+ monkeypatch.setattr(dfam, "resolve_base_repo", lambda f, b: b)+ monkeypatch.setattr(+ router, "predict_engine", lambda *a, **k: picks.append(k["engine"]) or ENGINE_MLX+ )+ monkeypatch.setattr(gr, "_cached_pick", lambda repo, name: "/m.gguf")+ fetched = []+ monkeypatch.setattr(+ mdb, "fetch_base_snapshot", lambda base, *a, **k: fetched.append(base) or "/base"+ )+ monkeypatch.setattr(+ mdb,+ "resolve_variant_base",+ lambda repo, f, tok, name, *, fallback, local_files_only: "org/variant",+ )+ monkeypatch.setattr(+ mdb, "get_mlx_diffusion_backend", lambda: types.SimpleNamespace(_rt = lambda: None)+ )+ monkeypatch.setattr(mdb, "decode_headroom_bytes", lambda rt: 3 * MIB + 1)+ plan = types.SimpleNamespace(source = "plan", planned_bytes = 4099 * MIB + 1)+ monkeypatch.setattr(mdb, "plan_for_load", lambda *a, **k: plan)+ request = types.SimpleNamespace(+ model_path = "org/z",+ gguf_filename = "z.gguf",+ text_encoder_file = None,+ vae_file = None,+ model_kind = "gguf",+ family_override = None,+ base_repo = "org/base",+ engine = "auto",+ )+ assert gr._image_need(request, 0) == (4096, 4)+ assert picks == ["auto"] and fetched == ["org/base"]+ request.base_repo = None+ assert gr._image_need(request, 0) == (4096, 4) and fetched[-1] == "org/variant"+ plan.source = "ram_rule"+ assert gr._image_need(request, 0) is Nonediff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.pyindex 9637af4cade..866899c17aa 100644--- a/studio/backend/tests/test_metal_explicit_context_guard.py+++ b/studio/backend/tests/test_metal_explicit_context_guard.py@@ -1864,6 +1864,33 @@ class TestMetalAlongsideAdmission:**kwargs,)["cmd"]+ @pytest.mark.parametrize(+ "media, gib",+ [((20480, 0), 22), ((1, 20480), 22), ((6144, 65536), 128), ((30720, 0), 22)],+ )+ def test_media_comes_off_the_metal_budget(self, tmp_path, monkeypatch, media, gib):+ import core.inference.llama_cpp as llama_cpp+ import utils.hardware.hardware as hardware++ memory = _types.SimpleNamespace(total = gib << 30, available = gib << 30)+ monkeypatch.setattr("psutil.virtual_memory", lambda: memory)+ monkeypatch.setattr(hardware, "_metal_recommended_working_set_bytes", lambda: gib << 30)+ monkeypatch.setattr(LlamaCppBackend, "_metal_media_mib", lambda self: media)+ args = dict(n_ctx = 0, real_fit = True, refuse_partial_gpu_fit = True, weights_bytes = 4 << 30)+ with pytest.raises(llama_cpp.GpuMemoryShortError):+ _launch(tmp_path, monkeypatch, budget_bytes = 32 << 30, wired_bytes = 64 << 30, **args)++ def test_an_mlx_chat_models_growth_comes_off_beside_media(self, monkeypatch):+ import core.inference.gpu_residency as residency+ import core.inference.llama_cpp as llama_cpp++ backend = LlamaCppBackend()+ monkeypatch.setattr(llama_cpp, "_serving_backends_snapshot", lambda: [backend])+ monkeypatch.setattr(residency, "media_held_mib", lambda: {0: 6144})+ monkeypatch.setattr(residency, "media_unallocated_mib", lambda: {0: 1024})+ monkeypatch.setattr(residency, "_mlx_chat_unallocated_mib", lambda: 2048)+ assert backend._metal_media_mib() == (6144, 3072)+def test_retry_rechecks_memory_after_eviction(self, tmp_path, monkeypatch):from core.inference.llama_cpp import GpuMemoryShortErrordiff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.pyindex 744ac42ff33..4422dd1a933 100644--- a/studio/backend/tests/test_mlx_diffusion_backend.py+++ b/studio/backend/tests/test_mlx_diffusion_backend.py@@ -1876,3 +1876,56 @@ def test_the_mlx_base_fetch_keeps_the_fallback_scheduler_config():from core.inference.mlx_diffusion_backend import BASE_ALLOW_PATTERNSfor name in ("scheduler/scheduler_config.json", "model_index.json"):assert any(fnmatch.fnmatch(name, pattern) for pattern in BASE_ALLOW_PATTERNS), name+++def test_the_renderer_sees_the_shared_pool_beside_a_chat_model(monkeypatch):+ """Its own working set cannot see a chat worker, so tiling decides on the system-wide headroom too."""+ import sys+ import types++ import core.inference.gpu_residency as gr+ import core.inference.llama_cpp as llama_cpp++ core = types.SimpleNamespace(+ device_info = lambda: {"max_recommended_working_set_size": 96 * _GIB},+ get_active_memory = lambda: 8 * _GIB,+ )+ monkeypatch.setitem(sys.modules, "mlx", types.SimpleNamespace(core = core))+ monkeypatch.setitem(sys.modules, "mlx.core", core)+ monkeypatch.setattr(gr, "_on_metal", lambda: True)+ monkeypatch.setattr(llama_cpp, "_apple_wired_headroom_bytes", lambda: 16 * _GIB)+ monkeypatch.setattr(gr, "_mlx_chat_unallocated_mib", lambda: 4096)+ runtime = mb.MfluxRuntime()+ monkeypatch.setattr(gr, "media_held_mib", lambda: {0: 9000})+ monkeypatch.setattr("psutil.virtual_memory", lambda: types.SimpleNamespace(available = 64 * _GIB))+ assert runtime.free_working_set_bytes() == 12 * _GIB+ monkeypatch.setattr("psutil.virtual_memory", lambda: types.SimpleNamespace(available = 5 * _GIB))+ assert runtime.free_working_set_bytes() == 1 * _GIB+ assert gr.metal_media_priced()+ monkeypatch.setattr(gr, "media_held_mib", lambda: {})+ assert runtime.free_working_set_bytes() == 88 * _GIB++[email protected]("priced, free_gib, expected", [(True, 40.0, True), (False, 40.0, False)])+def test_a_priced_media_model_on_metal_always_tiles(monkeypatch, priced, free_gib, expected):+ """Its price reserved the tiled decode, and a chat model may join on it while this render runs."""+ import sys+ import types++ import core.inference.gpu_residency as gr++ class _TilingConfig:+ @staticmethod+ def may_tile_implicitly(model):+ return True++ module = types.ModuleType("mflux.models.common.vae.tiling_config")+ module.TilingConfig = _TilingConfig+ monkeypatch.setitem(sys.modules, "mflux.models.common.vae.tiling_config", module)+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)+ monkeypatch.setattr(gr, "metal_media_priced", lambda: priced)+ runtime = mb.MfluxRuntime()+ monkeypatch.setattr(runtime, "free_working_set_bytes", lambda: int(free_gib * _GIB))+ model = types.SimpleNamespace(tiling_config = None)+ runtime._plan_vae_tiling(model, 1024, 1024)+ assert runtime.vae_tiled() is expecteddiff --git a/studio/backend/tests/test_multi_model_primary_fit.py b/studio/backend/tests/test_multi_model_primary_fit.pyindex a58450345d5..f2f6032a1c2 100644--- a/studio/backend/tests/test_multi_model_primary_fit.py+++ b/studio/backend/tests/test_multi_model_primary_fit.py@@ -12,7 +12,7 @@ from fastapi import HTTPExceptionimport core.inference.model_slots as model_slotsimport core.inference.orchestrator as orchestratorimport routes.inference as inf-from core.inference import gpu_arbiter, gpu_residency+from core.inference import gpu_arbiter, gpu_residency, mlx_inferencefrom core.inference.llama_cpp import GpuMemoryShortErrorfrom models.inference import LoadRequest, LoadResponsefrom utils import multi_model_settings@@ -297,6 +297,26 @@ def test_media_on_target_gpus(monkeypatch, held, placement, expected):assert inf._media_on_target_gpus(placement) is expected+def test_mlx_chat_need_prices_one_row_the_prompt_cache_and_a_batch_row_per_extra_slot(monkeypatch):+ gib = 1024**3+ breakdown = SimpleNamespace(+ total_bytes = 3 * gib, weights_bytes = gib, kv_bytes = gib // 2, compute_bytes = gib // 4+ )+ estimate = inf._MlxLoadEstimate(breakdown, None, (), True)+ monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, estimate))+ monkeypatch.setattr(+ mlx_inference, "mlx_memory_budget", lambda **k: (10 - k["retains_history"]) << 30+ )+ mlx = SimpleNamespace(is_gguf = False)+ assert inf._mlx_chat_need(None, mlx, "m") is None # not on Metal+ monkeypatch.setattr(inf, "_unified_metal_host", lambda: True)+ need, growth = inf._mlx_chat_need(None, mlx, "m", 4)+ assert (need, growth) == (4096 + 3 * 768, {"mib": 3072, "row_mib": 768, "slots": 4})+ offloaded = estimate._replace(offloaded = True)+ monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, offloaded))+ assert inf._mlx_chat_need(None, mlx, "m", 4) is None++def test_chat_fit_kwargs(monkeypatch):policy = SimpleNamespace(value = "off")monkeypatch.setattr(gpu_residency, "current_policy", lambda: policy.value)@@ -328,6 +348,11 @@ def test_chat_fit_kwargs(monkeypatch):# No target GPUs (or Vulkan ordinals) leaves every GPU in play.assert inf._chat_fit_kwargs(gguf, _placement(None)) == {"policy": "always"}assert inf._chat_fit_kwargs(gguf, _placement([0], vulkan = True)) == {"policy": "always"}+ monkeypatch.setattr(inf, "_unified_metal_host", lambda: True)+ assert inf._chat_fit_kwargs(native, _placement(None)).get("evict_first") is True+ priced = inf._chat_fit_kwargs(native, _placement(None), mlx_need_mib = 700, mlx_credit_mib = 50)+ assert priced.get("need_mib") == {0: 700} and "evict_first" not in priced+ assert priced["need_mib"].credit_mib == 50@pytest.mark.parametrize(@@ -357,6 +382,62 @@ def test_the_replace_fallback_never_refuses_a_partial_fit():)+def test_only_a_bounded_mlx_cache_is_priced_and_its_admitted_width_never_shrinks():+ assert inf._mlx_cache_bounded({"context_length_enforced": True})+ assert inf._mlx_cache_bounded({"context_length_enforced": False, "mlx_context_budget": 4096})+ assert not inf._mlx_cache_bounded({"context_length_enforced": None})+ batching = {"context_length_enforced": True, "context_unbounded_when_batched": True}+ assert inf._mlx_cache_bounded(batching) and not inf._mlx_cache_bounded(batching, 2)+ assert not inf._mlx_cache_bounded(+ {"context_length_enforced": False, "mlx_context_budget": None}+ )+ entry = {"context_length_enforced": True, "mlx_growth": {"mib": 1, "row_mib": 1, "slots": 1}}+ backend = SimpleNamespace(models = {"m": entry}, active_model_name = "m")+ for slots in (4, 1):+ inf._note_mlx_slots(backend, slots)+ assert entry["mlx_growth"]["slots"] == 4+ vision = {**batching, "mlx_growth": {"mib": 1, "row_mib": 1, "slots": 1}}+ inf._note_mlx_slots(SimpleNamespace(models = {"v": vision}, active_model_name = "v"), 2)+ assert "mlx_growth" not in vision+ inf._note_mlx_slots(SimpleNamespace(models = {}, active_model_name = None), 2)+ import inspect++ source = inspect.getsource(inf._load_model_impl)+ assert "_note_mlx_slots(backend, backend.set_parallel_slots(_n_parallel))" in source+ assert (+ '_mlx_cache_bounded(\n _resident_entry, mlx_chat_need[1]["slots"]' in source+ )+++def test_the_mlx_diffusers_fallback_forgets_the_mlx_price():+ import inspect+ source = inspect.getsource(inf._mlx_fallback_load_gate)+ assert (+ source.index("refuse_unless_fits(")+ < source.index("forget_media_price(DIFFUSION, cancel)")+ < source.index("return start()")+ )+++def test_an_mlx_load_is_credited_what_it_replaces_and_more_rows_reprice_it():+ import inspect++ source = inspect.getsource(inf._load_model_impl)+ shortcut = source.index('logger.info(f"Model already loaded (Unsloth)')+ assert (+ "not _mlx_slots_outgrow_price(backend, _n_parallel)" in source[shortcut - 1200 : shortcut]+ )+ inherit = source.index("# A reload for more rows keeps the context the resident was asked for.")+ assert inherit < shortcut and '"max_seq_length_requested"' in source[inherit : inherit + 400]+ credit = source.index(+ "mlx_credit_mib = await asyncio.to_thread(mlx_replaced_credit_mib, backend)"+ )+ assert "gguf_replaced_credit_mib" not in source+ assert credit < source.index("mlx_credit_mib = mlx_credit_mib", credit)+ settle = source.index("settle_unpriced_chat_beside_media, multi_model_mode")+ assert settle < source.index("backend.unload_model, config.identifier", settle)++def test_a_pinned_non_gguf_chat_load_keeps_its_requested_gpus(monkeypatch):# Only GGUF and native audio resolve their GPUs: a Transformers load pinned to GPU 1 still targets GPU 1.pinned = inf._LoadPlacement([1], None, False, None)