Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): attach an explicitly chosen MLX drafter even when it is priced not to fit (#15813)
Long Yixing316e5757fe7b
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 3124bd9ff05..3502c480db7 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -1151,8 +1151,8 @@ def mlx_resolve_drafter_offload(model_dir, knobs, drafter, *, auto: bool, **pricroom = mlx_resolve_offload(model_dir, knobs, drafter = drafter, **priced)if not auto:- # A plan the load would then refuse the drafter on (nothing fits: all on disk, no step) helps nobody.- if "auto" in knobs and fits(room) is False:+ # Room a chosen drafter does not fit even under MLX's own cap would only slow the load.+ if "auto" in knobs and fits(room, reserve = False) is False:return mlx_resolve_offload(model_dir, knobs, **priced), Nonereturn room, Nonealone = mlx_resolve_offload(model_dir, knobs, **priced)@@ -2882,12 +2882,14 @@ def mlx_drafter_fit(offload = None,prefill_chunk = None,baseline = None,+ reserve = True,):"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fitat least the context the load fits without it, on the *baseline* route (``mlx_offload_route``)- when the offload made room for it. ``attaches`` is None when the fit could not be priced."""- budget = mlx_memory_budget(retains_history = mlx_vlm_snapshot_store_available())+ when the offload made room for it. ``attaches`` is None when the fit could not be priced.+ ``reserve=False`` prices against MLX's own cap, spending the prompt cache's share."""+ budget = mlx_memory_budget(retains_history = reserve and mlx_vlm_snapshot_store_available())ceiling = pinned or ceilingif budget is None or not ceiling or not model_dir:return None, None@@ -2916,6 +2918,37 @@ def mlx_drafter_fit(return outcome == "fits" or (outcome == "fitted" and not pinned), fitted+def mlx_prompt_cache_room(+ model_dir,+ n_ctx,+ source,+ *,+ load_in_4bit: bool,+ offload = None,+ prefill_chunk = None,+):+ """Bytes MLX's own cap leaves the prompt cache beside *source*'s drafter at *n_ctx*; None where unsized."""+ budget = mlx_memory_budget(retains_history = False)+ if budget is None or not n_ctx or not model_dir:+ return None+ with mlx_rng_preserved():+ try:+ from core.inference.mlx_memory import mlx_headroom+ return mlx_headroom(+ model_dir,+ budget_bytes = budget,+ n_ctx = int(n_ctx),+ load_in_4bit = load_in_4bit,+ vision = True,+ drafter = (source.path, source.builtin),+ offload = offload,+ prefill_chunk = prefill_chunk,+ )+ except Exception as exc:+ logger.info("MLX prompt cache room beside %s not priced: %s", source.path, exc)+ return None++def _kv_window_enforced(model, is_vlm, window):"""Whether a cache built at *window* is bounded in every layer, None where none could be builtto judge: both runtimes defer to a model's own make_cache and ignore max_kv_size there."""@@ -4629,6 +4662,7 @@ class MLXInferenceBackend:self._prompt_cache_history = Noneself._prompt_cache_unavailable = False+ self._prompt_cache_room = Noneself._vlm_snapshot_store = Noneself._vlm_snapshot_store_unavailable = Falseself._vlm_is_diffusion_model = None@@ -4636,7 +4670,7 @@ class MLXInferenceBackend:def _prompt_cache(self):if self._prompt_cache_history is not None or self._prompt_cache_unavailable:return self._prompt_cache_history- max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb"))+ max_bytes = self._prompt_cache_bytes()if max_bytes <= 0:self._prompt_cache_unavailable = Truelogger.info("MLX prompt cache disabled by budget")@@ -4659,9 +4693,15 @@ class MLXInferenceBackend:)return self._prompt_cache_history+ def _prompt_cache_bytes(self):+ max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb"))+ room = getattr(self, "_prompt_cache_room", None)+ return max_bytes if room is None else min(max_bytes, room)+def _clear_prompt_cache(self):self._prompt_cache_history = Noneself._prompt_cache_unavailable = False+ self._prompt_cache_room = Noneself._vlm_snapshot_store = Noneself._vlm_snapshot_store_unavailable = Falseself._vlm_is_diffusion_model = None@@ -4783,7 +4823,7 @@ class MLXInferenceBackend:def _vlm_prompt_cache_store(self):if self._vlm_snapshot_store is not None or self._vlm_snapshot_store_unavailable:return self._vlm_snapshot_store- max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb"))+ max_bytes = self._prompt_cache_bytes()if max_bytes <= 0:self._vlm_snapshot_store_unavailable = Truelogger.info("MLX VLM prompt cache disabled by budget")@@ -5574,7 +5614,7 @@ class MLXInferenceBackend:_offloaded["prefill_chunk"] = self._prefill_step_priceable = not (is_distributed or is_lora or dtype is not None)spec_kind = _drafter_ctx = None- _drafter_fitted = False+ _stretched = []_vlm_route = speculates and not is_visionif speculates and _vlm_generation_is_diffusion(self._model):# Neither the single path nor the resident batch may decode a diffusion model speculatively.@@ -5594,7 +5634,7 @@ class MLXInferenceBackend:allowed = spec_drafters_allowed,)- def _fits(source):+ def _fits(source, reserve = True):if not _priceable:return None, Nonereturn mlx_drafter_fit(@@ -5605,19 +5645,27 @@ class MLXInferenceBackend:load_in_4bit = load_in_4bit,costless = spec_mode == "auto",baseline = offload_baseline["route"],+ reserve = reserve,**_offloaded,)+ def _stretch(source):+ ok, context = _fits(source, reserve = False)+ if ok is not False:+ _stretched.append(source)+ return ok, context+(self._speculative_draft,spec_kind,spec_reason,_drafter_ctx,) = mlx_speculative.build_draft(- self._model, resolution, fits = _fits, draft_n_max = _positive_int(spec_draft_n_max)- )- _drafter_fitted = (- self._speculative_draft is not None and self._speculative_draft.drafter is not None+ self._model,+ resolution,+ fits = _fits,+ stretch = _stretch,+ draft_n_max = _positive_int(spec_draft_n_max),)logger.info("MLX speculative decoding: %s (mode %s, reason %s)",@@ -5630,7 +5678,7 @@ class MLXInferenceBackend:_requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)_fitted_ctx, _eligibility = ((_drafter_ctx, _eligibility)- if _drafter_fitted+ if _drafter_ctx is not Noneelse (None, _eligibility)if not _priceable or _positive_int(max_seq_length) is not Noneelse _fitted_context(@@ -5650,6 +5698,20 @@ class MLXInferenceBackend:logger.info("MLX context fitted to %d tokens from %d", _fitted_ctx, _served_ctx or 0)_served_ctx = _fitted_ctxself._served_context = _served_ctx+ # A stretched drafter spent the prompt cache's share: cap the cache to what is left beside it.+ if (+ _priceable+ and _stretched+ and getattr(self._speculative_draft, "drafter", None) is not None+ ):+ self._prompt_cache_room = mlx_prompt_cache_room(+ model_dir, _served_ctx, _stretched[-1], load_in_4bit = load_in_4bit, **_offloaded+ )+ if self._prompt_cache_room is not None:+ logger.info(+ "MLX prompt cache holds at most %.2f GB beside the chosen drafter",+ self._prompt_cache_room / 1e9,+ )# Classified now so a cache the model cannot attend over is refused here, not on a token.(self._kv_quant,diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex 233e8b94398..94b1c17a67e 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -1486,9 +1486,7 @@ def mlx_fit_outcome(sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)if sizing is None:return "unsizable", None- if any(sizing.offload):- least, share = mlx_offload_planner().SAFETY_MARGIN- budget_bytes = min(budget_bytes - least, budget_bytes / (1 + share))+ budget_bytes = _fit_budget(sizing, budget_bytes)ceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)at_ceiling = _priced_at(sizing, ceiling)if at_ceiling is None:@@ -1510,6 +1508,33 @@ def mlx_fit_outcome(return ("no_fit", None) if best is None else ("fitted", (best // MLX_KV_BLOCK) * MLX_KV_BLOCK)+def _fit_budget(sizing: _MlxSizing, budget_bytes: int) -> float:+ if not any(sizing.offload):+ return budget_bytes+ least, share = mlx_offload_planner().SAFETY_MARGIN+ return min(budget_bytes - least, budget_bytes / (1 + share))+++def mlx_headroom(+ model_dir: str,+ *,+ budget_bytes: int,+ n_ctx: int,+ kv_bits: Optional[int] = None,+ prefill_chunk: Optional[int] = None,+ load_in_4bit: bool = False,+ vision: Optional[bool] = None,+ drafter: Optional[tuple] = None,+ offload: Optional[tuple] = None,+) -> Optional[int]:+ """Bytes of ``budget_bytes`` a load at ``n_ctx`` leaves free (as ``mlx_fit_outcome`` prices it), None if unsized."""+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)+ priced = None if sizing is None else _priced_at(sizing, max(int(n_ctx or 0), MLX_KV_BLOCK))+ if priced is None:+ return None+ return max(int(_fit_budget(sizing, budget_bytes) - priced.total_bytes), 0)++def mlx_offload_plan(model_dir: str,*,diff --git a/studio/backend/core/inference/mlx_speculative.py b/studio/backend/core/inference/mlx_speculative.pyindex 58de629e6ed..30e708cc819 100644--- a/studio/backend/core/inference/mlx_speculative.py+++ b/studio/backend/core/inference/mlx_speculative.py@@ -373,35 +373,55 @@ def build_draft(resolution: SpecResolution,*,fits,+ stretch = None,draft_n_max: Optional[int] = None,) -> tuple:"""``(draft, kind, reason, context)`` for the first source ``fits`` accepts that builds, else copies- alone when the mode allows. ``reason`` names a passed-over choice, so a substitute never attaches silently."""+ alone when the mode allows. ``reason`` names a passed-over choice, so a substitute never attaches silently.+ A chosen mode with none that fits attaches an unfit source ``stretch`` (its fit under MLX's own cap) admits."""from unsloth_zoo.mlx.speculative import companion_drafter, native_mtp_drafterif _carries_encoder_state(target):logger.info("MLX speculative decoding: %s carries encoder state", type(target).__name__)return None, None, None if resolution.mode == "auto" else RUNTIME_ERROR, None- reason = resolution.reason- for source in resolution.sources:- ok, context = fits(source)- if not ok:- if resolution.mode != "auto":- reason = DRAFTER_NO_MEMORY- elif ok is not None: # priced, and the drafter would shrink it- reason = AUTO_CONTEXT_COST- continue+ reason, incompatible, unfit = resolution.reason, False, []++ def build(source):+ nonlocal reason, incompatibletry:drafter = (native_mtp_drafter if source.builtin else companion_drafter)(source.path, target)if drafter is None:raise ValueError("no MTP splitter knows this checkpoint")+ return drafterexcept Exception as exc:logger.warning("MLX drafter %s (%s) not attached: %s", source.path, source.kind, exc)- reason = DRAFTER_INCOMPATIBLE+ reason, incompatible = DRAFTER_INCOMPATIBLE, True++ for source in resolution.sources:+ ok, context = fits(source)+ if ok:+ if (drafter := build(source)) is not None:+ return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, context+ elif resolution.mode != "auto":+ reason = DRAFTER_NO_MEMORY+ unfit.append(source)+ elif ok is not None: # priced, and the drafter would shrink it+ reason = AUTO_CONTEXT_COST+ # Priced one at a time, so the last source stretch admitted is the one that attaches.+ for source in unfit if stretch else ():+ ok, context = stretch(source)+ if ok is False:continue- return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, context+ logger.warning(+ "MLX drafter %s (%s) fits only in the prompt cache's share; attaching it as chosen",+ source.path,+ source.kind,+ )+ if (drafter := build(source)) is not None:+ reason = DRAFTER_INCOMPATIBLE if incompatible else resolution.reason+ return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, contextif resolution.copies and resolution.mode != "auto": # auto copies only alongside a drafterreturn _draft(None, True, draft_n_max), "ngram", reason, Nonereturn None, None, reason, Nonediff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 741d5d978fd..817220e68ac 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1891,8 +1891,10 @@ class _InferenceRuntimeFields(BaseModel):"MTP in Settings to force it. ""MLX loads report 'drafter_not_found' (no drafter of the requested kind ""is cached), 'drafter_incompatible' (the named or found drafter does not "- "fit this model), 'drafter_no_memory' (its weights and cache leave no "- "context that fits), 'kv_quant' (KV cache quantization is on) or "+ "fit this model), 'drafter_no_memory' (a drafter whose weights and cache "+ "leave no context that fits was passed over; a chosen mode still attaches "+ "one that fits by spending the prompt cache's share of memory), 'kv_quant' (KV "+ "cache quantization is on) or ""'runtime_error' (the installed MLX packages cannot speculate, or cannot ""load this model the way speculation needs), also when a later drafter ""or n-gram copies stood in for the one asked for. Under auto an MLX load "diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 3371d64b73a..2c857832e35 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -14728,13 +14728,14 @@ def _mlx_estimate_drafter(offload_route: Optional[dict] = None,baseline: Optional[dict] = None,):- """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it- would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""+ """``((path, builtin), fitted context, speculates, stretched)`` of an MLX load of *request*: the drafter it+ would attach, chosen as the load chooses it, whether it speculates at all (so loads through mlx-vlm), and+ whether that drafter fits only by spending the prompt cache's share."""from core.inference import mlx_speculativefrom core.inference.mlx_inference import mlx_drafter_fitif not _mlx_estimate_speculates(request, model_dir):- return None, None, False+ return None, None, False, Falsemode = mlx_speculative.mlx_spec_mode(request.speculative_type)resolution = mlx_speculative.resolve_speculation(request.speculative_type,@@ -14743,8 +14744,9 @@ def _mlx_estimate_drafter(target_name = model_identifier,allowed = _discoverable_drafters(model_identifier),)- for source in resolution.sources:- attaches, fitted = mlx_drafter_fit(++ def fit(source, **kwargs):+ return mlx_drafter_fit(model_dir,ceiling,request.max_seq_length or None,@@ -14752,11 +14754,20 @@ def _mlx_estimate_drafter(load_in_4bit = load_in_4bit,costless = mode == "auto",baseline = baseline,+ **kwargs,**(offload_route or {}),)++ for source in resolution.sources:+ attaches, fitted = fit(source)if attaches:- return (source.path, source.builtin), fitted, True- return None, None, True+ return (source.path, source.builtin), fitted, True, False+ # Like the load: a chosen mode with none that fits attaches the first MLX's own cap admits.+ for source in resolution.sources if mode != "auto" else ():+ attaches, fitted = fit(source, reserve = False)+ if attaches is not False:+ return (source.path, source.builtin), fitted, True, True+ return None, None, True, Falseclass _MlxLoadEstimate(NamedTuple):@@ -14765,6 +14776,7 @@ class _MlxLoadEstimate(NamedTuple):offloadable: tupleretains_history: booloffloaded: bool = False+ cache_room: Optional[int] = Nonedef _mlx_load_estimate(@@ -14797,7 +14809,7 @@ def _mlx_load_estimate(offload, offloadable, offload_baseline = _mlx_estimate_offload(request, config, model_identifier, model_dir, load_in_4bit, kv_bits)- drafter, fitted_ctx, speculates = _mlx_estimate_drafter(+ drafter, fitted_ctx, speculates, stretched = _mlx_estimate_drafter(request,model_identifier,model_dir,@@ -14807,19 +14819,32 @@ def _mlx_load_estimate(offload_baseline,)route = {**({"vision": True} if speculates else {}), **offload}- if not named_ctx and drafter is None:+ if not named_ctx and fitted_ctx is None:fitted_ctx = _mlx_estimate_fitted_context(config, model_dir, load_in_4bit, kv_bits, **route)priced_ctx = named_ctx or fitted_ctx or _mlx_estimate_ceiling(model_dir)if not priced_ctx:return "unsizable", None+ priced = dict(n_ctx = priced_ctx, kv_bits = kv_bits, load_in_4bit = load_in_4bit, **route)breakdown = mlx_memory_breakdown(- model_dir,- n_ctx = priced_ctx,- kv_bits = kv_bits,- load_in_4bit = load_in_4bit,- **route,- **({} if drafter is None else {"drafter": drafter}),+ model_dir, **priced, **({} if drafter is None else {"drafter": drafter}))+ cache_room = None+ if breakdown is None and drafter is not None:+ # A chosen drafter attaches even where it cannot be sized: price the rest of the load.+ breakdown = mlx_memory_breakdown(model_dir, **priced)+ elif stretched:+ from types import SimpleNamespace++ from core.inference.mlx_inference import mlx_prompt_cache_room++ # The load caps the prompt cache to what MLX's own cap leaves beside this drafter.+ cache_room = mlx_prompt_cache_room(+ model_dir,+ priced_ctx,+ SimpleNamespace(path = drafter[0], builtin = drafter[1]),+ load_in_4bit = load_in_4bit,+ **offload,+ )if breakdown is None:return "unsizable", Noneretains_history = (@@ -14827,7 +14852,7 @@ def _mlx_load_estimate(or mlx_vlm_snapshot_store_available())return None, _MlxLoadEstimate(- breakdown, fitted_ctx, offloadable, retains_history, "offload" in offload+ breakdown, fitted_ctx, offloadable, retains_history, "offload" in offload, cache_room)@@ -14854,6 +14879,8 @@ def _mlx_chat_need(mlx_memory_budget(retains_history = True),)reserve = max(0, (without or 0) - (within or 0))+ if estimate.cache_room is not None:+ reserve = min(reserve, estimate.cache_room)except Exception as exc:logger.info("MLX load of %s left unpriced: %s", model_identifier, exc)return Nonediff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex 9a5d5e8c3f4..7d6ff7908f9 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -820,12 +820,17 @@ class TestEstimateMemoryRoute:monkeypatch.setattr(spec, "speculation_refusal", lambda kv_quant, **_: spec.KV_QUANT if kv_quant else None)+ fit = Falsemonkeypatch.setattr(mlx_inference,"mlx_drafter_fit",- lambda d, ceiling, pinned, source, **_: (- source.path == "/ok" and (pinned or 0) <= 8192,- None if pinned else 12288,+ lambda d, ceiling, pinned, source, reserve = True, **_: (+ None+ if fit in ("unpriced", "unsizable")+ else source.path == "/ok" and (pinned or 0) <= 8192 and not fit+ if reserve+ else (pinned or 0) <= 32768, # MLX's own cap, spending the prompt cache's share+ None if pinned or fit in ("unpriced", "unsizable") else 12288,),)monkeypatch.setattr(ri, "_mlx_estimate_ceiling", lambda model_dir: 32768)@@ -837,31 +842,66 @@ class TestEstimateMemoryRoute:mlx_memory,"mlx_memory_breakdown",lambda d, *, n_ctx, **kw: priced.append((n_ctx, kw.get("drafter"), kw.get("vision")))- or mlx_memory.MlxMemoryBreakdown(1, 0, 0, 1, 1, n_ctx = n_ctx),+ or (+ None+ if fit == "unsizable" and kw.get("drafter")+ else mlx_memory.MlxMemoryBreakdown(1, 0, 0, 1, 1, n_ctx = n_ctx)+ ),+ )+ rooms, estimates, load_estimate = [], [], ri._mlx_load_estimate+ monkeypatch.setattr(+ mlx_inference,+ "mlx_prompt_cache_room",+ lambda d, n_ctx, source, **_: rooms.append((n_ctx, source.path)) or 7,+ )+ monkeypatch.setattr(+ ri,+ "_mlx_load_estimate",+ lambda *a: (result := load_estimate(*a), estimates.append(result[1]))[0],)assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288- for pinned in (4096, 65536):+ for pinned in (4096, 16384, 65536):_estimate(model_path = "org/model", speculative_type = "mtp", max_seq_length = pinned)_estimate(model_path = "org/model", speculative_type = "mtp", mlx_kv_quant = "8")_estimate(model_path = "org/model")_estimate(model_path = "o/m", speculative_type = "ngram", mlx_kv_quant = "auto", mlx_kv_bits = 8)_estimate(model_path = "org/model", spec_draft_model = "org/d") # named under Auto- assert [route for *_, route in priced] == [True, True, True, None, None, True, True]- assert [row[:2] for row in priced[:6]] == [+ fit = "unfit"+ assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288+ fit = "unpriced" # no fit to go on: the drafter the load attaches is still charged+ _estimate(model_path = "org/model", speculative_type = "mtp")+ fit = "unsizable" # one the breakdown cannot size leaves the rest of the load priced+ assert _estimate(model_path = "org/model", speculative_type = "mtp").available+ assert [route for *_, route in priced] == [True] * 4 + [None] * 2 + [True] * 6+ assert [row[:2] for row in priced[:7] + priced[8:]] == [(12288, ("/ok", False)),(4096, ("/ok", False)),- (65536, None),+ (16384, ("/big", True)),+ (65536, None), # past MLX's own cap the load falls back, so no drafter is charged(20480, None),(20480, None),(24576, None),+ (12288, ("/big", True)),+ (24576, ("/big", True)),+ (24576, ("/big", True)),+ (24576, None),]+ # Only a drafter that spent the prompt cache's share caps the cache admission charges.+ assert rooms == [(16384, "/big"), (12288, "/big"), (24576, "/big")]+ assert [e.cache_room for e in estimates if e and e.cache_room] == [7, 7, 7]def test_a_drafter_attaches_where_its_priced_fit_holds_and_a_pin_fits_whole(self, monkeypatch):import core.inference.mlx_inference as mlx_inferenceimport core.inference.mlx_memory as mlx_memoryfrom core.inference.mlx_speculative import DrafterSource- monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 << 30)+ budgets = []+ monkeypatch.setattr(mlx_inference, "mlx_vlm_snapshot_store_available", lambda: True)+ monkeypatch.setattr(+ mlx_inference,+ "mlx_memory_budget",+ lambda *, retains_history: budgets.append(retains_history) or 8 << 30,+ )priced = []def fit(pinned, outcome, *bare):@@ -902,6 +942,9 @@ class TestEstimateMemoryRoute:prefill_chunk = 8192,)assert (asked["offload"], asked["prefill_chunk"]) == ((None, 0.6), 8192)+ assert set(budgets) == {True}+ mlx_inference.mlx_drafter_fit("/m", 32768, None, source, load_in_4bit = False, reserve = False)+ assert budgets[-1] is False # MLX's own cap: the prompt cache's share is spendableseen = []def costless(attached, alone):@@ -4795,6 +4838,10 @@ class TestPricingALoad:str(tmp_path), budget_bytes = price + (2 << 30), max_ctx = 1 << 20, offload = offload)assert fit((None, 0.5)) == ("fitted", 8192) != fit(None)+ room = lambda offload: mm.mlx_headroom(+ str(tmp_path), budget_bytes = price + (2 << 30) + 5, n_ctx = 8192, offload = offload+ )+ assert room((None, 0.5)) == 5 # the room keeps the same margin free as the fitdef test_a_quantizing_load_is_priced_below_the_shards(self, tmp_path):config = self._checkpoint(tmp_path)@@ -5205,8 +5252,12 @@ class TestTheContextSearch:"no_fit",None,)+ assert mm.mlx_headroom("x", budget_bytes = 9000, n_ctx = 8192) == 808+ assert mm.mlx_headroom("x", budget_bytes = 9000, n_ctx = 100) == 9000 - 256+ assert mm.mlx_headroom("x", budget_bytes = 7000, n_ctx = 8192) == 0monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("unsizable", None)+ assert mm.mlx_headroom("x", budget_bytes = 7000, n_ctx = 8192) is None@_NEEDS_MLXdiff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex d5c1c23e371..57d07b2cfdb 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1336,11 +1336,17 @@ def test_an_auto_drafter_gets_offload_room_only_within_the_cap_where_it_then_fit]plans.update({True: (0.5, 0.5, 8192)})assert resolve(True) == ((0.5, 0.5, 8192),) * 2 and len(asked) == 1- # An explicit drafter priced not to fit its room plan keeps the target-only plan, not all-on-disk.+ # An explicit drafter priced not to fit its room plan, even under MLX's own cap, keeps the+ # target-only plan, not all-on-disk.plans.update({True: (None, 1.0, None), False: (None, 0.78, 8192)})asked.clear()assert resolve(False) == ((None, 0.78, 8192), None)- assert asked[-1][4] == {"load_in_4bit": False, "offload": (None, 1.0), "prefill_chunk": None}+ assert asked[-1][4] == {+ "load_in_4bit": False,+ "offload": (None, 1.0),+ "prefill_chunk": None,+ "reserve": False,+ }fits.update(on = None) # unpriceable: the load keeps it, so does the planassert resolve(False) == ((None, 1.0, None), None)@@ -4995,18 +5001,27 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fimonkeypatch.setattr(mlx_speculative,"resolve_speculation",- lambda mode, *_a, **_k: mlx_speculative.SpecResolution(mode, (source,)),+ lambda mode, *_a, **_k: mlx_speculative.SpecResolution(+ mode, () if mode == "ngram" else (source,)+ ),)monkeypatch.setattr(mlx_inference, "mlx_drafter_fit", lambda *a, **k: fits.append(a[1:4]) or (True, 4096))monkeypatch.setattr(drafters, "companion_drafter", lambda path, target: SimpleNamespace())+ spent, rooms = [], {"/drafter": 123, "/cheap": 500, "/unused": 0}+ monkeypatch.setattr(+ mlx_inference,+ "mlx_prompt_cache_room",+ lambda model_dir, n_ctx, source, **_k: spent.append((n_ctx, source)) or rooms[source.path],+ )config = SimpleNamespace(identifier = "org/text", is_vision = False, is_lora = False)backend = mlx_inference.MLXInferenceBackend()backend.load_model(config, max_seq_length = 0, speculative_type = "dflash", spec_draft_n_max = 5)entry = backend.models["org/text"]assert attempts == [False] and fits == [(8192, None, source)]+ assert backend._prompt_cache_room is None and spent == [] # a drafter that fits spends no shareassert (entry["spec_drafter_kind"], entry["spec_fallback_reason"]) == ("dflash", None)assert entry["context_length"] == 4096 and entry["spec_draft_n_max"] == 5assert backend._speculative_draft.controller.max_depth == 5@@ -5022,9 +5037,57 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fifailing.clear()fit = lambda *a, **kw: fits.append(kw.get("vision")) or (None, None)monkeypatch.setattr(mlx_inference, "_fitted_context", fit)- monkeypatch.setattr(mlx_inference, "mlx_drafter_fit", lambda *a, **k: (False, None))+ room = [True] # whether the drafter fits under MLX's own cap, spending the prompt cache's share+ monkeypatch.setattr(+ mlx_inference,+ "mlx_drafter_fit",+ lambda *a, reserve = True, **k: (not reserve and room[-1], None),+ )backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")assert fits[-1] is True # copies alone still load through mlx-vlm, so the fit prices that route+ fits.clear()+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash")+ assert backend._speculative_draft.drafter is not None and fits == [True]+ assert backend.models["org/text"]["spec_fallback_reason"] is None+ # That drafter spent the prompt cache's share, so both caches hold only what the cap still leaves.+ assert spent == [(8192, source)] and backend._prompt_cache_room == 123+ sized = []+ monkeypatch.setattr(+ mlx_inference,+ "_MLXPromptCacheHistory",+ lambda n, max_bytes, *a, **k: sized.append(max_bytes),+ )+ monkeypatch.setattr(+ mlx_inference, "VLMPromptSnapshotStore", lambda max_bytes: sized.append(max_bytes)+ )+ backend._prompt_cache(), backend._vlm_prompt_cache_store()+ assert sized in ([123, 123], [123]) # the second only where this mlx-vlm builds the store+ # The room is the attached drafter's: not one that failed to build, nor one never tried.+ cheap = mlx_speculative.DrafterSource("dflash", "/cheap", False)+ unused = mlx_speculative.DrafterSource("dflash", "/unused", False)+ resolve, build = mlx_speculative.resolve_speculation, drafters.companion_drafter+ monkeypatch.setattr(+ mlx_speculative,+ "resolve_speculation",+ lambda *_a, **_k: mlx_speculative.SpecResolution("dflash", (cheap, source, unused)),+ )+ monkeypatch.setattr(+ drafters,+ "companion_drafter",+ lambda path, target: (_ for _ in ()).throw(ValueError("no"))+ if path == "/cheap"+ else build(path, target),+ )+ spent.clear()+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash")+ assert backend._speculative_draft.drafter is not None and backend._prompt_cache_room == 123+ assert spent == [(8192, source)]+ monkeypatch.setattr(mlx_speculative, "resolve_speculation", resolve)+ monkeypatch.setattr(drafters, "companion_drafter", build)+ room.append(False)+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash")+ assert backend._speculative_draft is None and backend._prompt_cache_room is None+ assert backend.models["org/text"]["spec_fallback_reason"] == mlx_speculative.DRAFTER_NO_MEMORYbackend.load_model(config, max_seq_length = 0, spec_draft_model = "org/d")assert fits[-2:] == [True, True] # a named drafter enters the speculative route under Auto# ... and is a request, so a load that cannot honour it says why.diff --git a/studio/backend/tests/test_mlx_speculative.py b/studio/backend/tests/test_mlx_speculative.pyindex d1144afa5e2..9bc14d8083a 100644--- a/studio/backend/tests/test_mlx_speculative.py+++ b/studio/backend/tests/test_mlx_speculative.py@@ -167,11 +167,36 @@ def test_a_drafter_passed_over_keeps_its_reason_on_the_one_that_attaches(monkeypassert (draft.controller.max_depth, draft.controller.max_copy) == (6, 0)for d, n, e in ((object(), 3, True), (object(), 4, False), (draft.drafter, 15, True)):assert spec._draft(d, False, n).controller.fixed_depth is e- draft, kind, reason, _ = spec.build_draft(- None, spec.SpecResolution("dflash", sources[:1], copies = True), fits = fits++ def chosen(+ picked,+ fit,+ stretch = lambda _: (None, None),+ ):+ built.clear()+ result = spec.build_draft(+ None, spec.SpecResolution("dflash", picked, copies = True), fits = fit, stretch = stretch+ )+ return result[1:] + (built[:],)++ assert chosen(sources[::2], fits) == ("dflash", spec.DRAFTER_NO_MEMORY, 4096, ["ok"])+ assert chosen(sources[:1], fits) == ("dflash", None, None, ["big"])+ assert chosen(sources[:1], lambda _: (None, None)) == ("dflash", None, None, ["big"])+ assert chosen(sources[:1], fits, lambda _: (True, 2048)) == ("dflash", None, 2048, ["big"])+ # Past MLX's own cap a chosen drafter is still passed over, as is one nothing can stretch.+ for stretch in (lambda _: (False, None), None):+ assert chosen(sources[:1], fits, stretch) == ("ngram", spec.DRAFTER_NO_MEMORY, None, [])+ assert chosen(sources[:2], fits) == ("dflash", spec.DRAFTER_INCOMPATIBLE, None, ["big"])+ assert chosen(sources[1:2], lambda _: (False, 4096)) == (+ "ngram",+ spec.DRAFTER_INCOMPATIBLE,+ None,+ [],+ )+ draft, *_ = spec.build_draft(+ None, spec.SpecResolution("dflash", sources[1:2], copies = True), fits = lambda _: (False, 4096))- assert (draft.drafter, kind, reason) == (None, "ngram", spec.DRAFTER_NO_MEMORY)- assert (draft.controller.max_depth, draft.controller.max_copy) == (0, 16)+ assert (draft.drafter, draft.controller.max_depth, draft.controller.max_copy) == (None, 0, 16)auto = spec.SpecResolution("auto", sources[:1], copies = True)unfit = [spec.build_draft(None, auto, fits = lambda _: (ok, None)) for ok in (False, None)]assert unfit == [(None, None, spec.AUTO_CONTEXT_COST, None), (None,) * 4]diff --git a/studio/backend/tests/test_multi_model_primary_fit.py b/studio/backend/tests/test_multi_model_primary_fit.pyindex f2f6032a1c2..dcea572a847 100644--- a/studio/backend/tests/test_multi_model_primary_fit.py+++ b/studio/backend/tests/test_multi_model_primary_fit.py@@ -312,6 +312,11 @@ def test_mlx_chat_need_prices_one_row_the_prompt_cache_and_a_batch_row_per_extramonkeypatch.setattr(inf, "_unified_metal_host", lambda: True)need, growth = inf._mlx_chat_need(None, mlx, "m", 4)assert (need, growth) == (4096 + 3 * 768, {"mib": 3072, "row_mib": 768, "slots": 4})+ # A drafter that spent the prompt cache's share leaves the cache only the room the load caps it to.+ for room, reserved in ((256 << 20, 256), (4 * gib, 1024)):+ stretched = estimate._replace(cache_room = room)+ monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, stretched))+ assert inf._mlx_chat_need(None, mlx, "m", 4)[0] == 3072 + reserved + 3 * 768offloaded = estimate._replace(offloaded = True)monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, offloaded))assert inf._mlx_chat_need(None, mlx, "m", 4) is None