Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): attach an explicitly chosen MLX drafter even when it is priced not to fit (#15813)

Long Yixing316e5757fe7b
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 3124bd9ff05..3502c480db7 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -1151,8 +1151,8 @@ def mlx_resolve_drafter_offload(model_dir, knobs, drafter, *, auto: bool, **pric
room = mlx_resolve_offload(model_dir, knobs, drafter = drafter, **priced)
if not auto:
- # A plan the load would then refuse the drafter on (nothing fits: all on disk, no step) helps nobody.
- if "auto" in knobs and fits(room) is False:
+ # Room a chosen drafter does not fit even under MLX's own cap would only slow the load.
+ if "auto" in knobs and fits(room, reserve = False) is False:
return mlx_resolve_offload(model_dir, knobs, **priced), None
return room, None
alone = mlx_resolve_offload(model_dir, knobs, **priced)
@@ -2882,12 +2882,14 @@ def mlx_drafter_fit(
offload = None,
prefill_chunk = None,
baseline = None,
+ reserve = True,
):
"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:
only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit
at least the context the load fits without it, on the *baseline* route (``mlx_offload_route``)
- when the offload made room for it. ``attaches`` is None when the fit could not be priced."""
- budget = mlx_memory_budget(retains_history = mlx_vlm_snapshot_store_available())
+ when the offload made room for it. ``attaches`` is None when the fit could not be priced.
+ ``reserve=False`` prices against MLX's own cap, spending the prompt cache's share."""
+ budget = mlx_memory_budget(retains_history = reserve and mlx_vlm_snapshot_store_available())
ceiling = pinned or ceiling
if budget is None or not ceiling or not model_dir:
return None, None
@@ -2916,6 +2918,37 @@ def mlx_drafter_fit(
return outcome == "fits" or (outcome == "fitted" and not pinned), fitted
+def mlx_prompt_cache_room(
+ model_dir,
+ n_ctx,
+ source,
+ *,
+ load_in_4bit: bool,
+ offload = None,
+ prefill_chunk = None,
+):
+ """Bytes MLX's own cap leaves the prompt cache beside *source*'s drafter at *n_ctx*; None where unsized."""
+ budget = mlx_memory_budget(retains_history = False)
+ if budget is None or not n_ctx or not model_dir:
+ return None
+ with mlx_rng_preserved():
+ try:
+ from core.inference.mlx_memory import mlx_headroom
+ return mlx_headroom(
+ model_dir,
+ budget_bytes = budget,
+ n_ctx = int(n_ctx),
+ load_in_4bit = load_in_4bit,
+ vision = True,
+ drafter = (source.path, source.builtin),
+ offload = offload,
+ prefill_chunk = prefill_chunk,
+ )
+ except Exception as exc:
+ logger.info("MLX prompt cache room beside %s not priced: %s", source.path, exc)
+ return None
+
+
def _kv_window_enforced(model, is_vlm, window):
"""Whether a cache built at *window* is bounded in every layer, None where none could be built
to judge: both runtimes defer to a model's own make_cache and ignore max_kv_size there."""
@@ -4629,6 +4662,7 @@ class MLXInferenceBackend:
self._prompt_cache_history = None
self._prompt_cache_unavailable = False
+ self._prompt_cache_room = None
self._vlm_snapshot_store = None
self._vlm_snapshot_store_unavailable = False
self._vlm_is_diffusion_model = None
@@ -4636,7 +4670,7 @@ class MLXInferenceBackend:
def _prompt_cache(self):
if self._prompt_cache_history is not None or self._prompt_cache_unavailable:
return self._prompt_cache_history
- max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb"))
+ max_bytes = self._prompt_cache_bytes()
if max_bytes <= 0:
self._prompt_cache_unavailable = True
logger.info("MLX prompt cache disabled by budget")
@@ -4659,9 +4693,15 @@ class MLXInferenceBackend:
)
return self._prompt_cache_history
+ def _prompt_cache_bytes(self):
+ max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb"))
+ room = getattr(self, "_prompt_cache_room", None)
+ return max_bytes if room is None else min(max_bytes, room)
+
def _clear_prompt_cache(self):
self._prompt_cache_history = None
self._prompt_cache_unavailable = False
+ self._prompt_cache_room = None
self._vlm_snapshot_store = None
self._vlm_snapshot_store_unavailable = False
self._vlm_is_diffusion_model = None
@@ -4783,7 +4823,7 @@ class MLXInferenceBackend:
def _vlm_prompt_cache_store(self):
if self._vlm_snapshot_store is not None or self._vlm_snapshot_store_unavailable:
return self._vlm_snapshot_store
- max_bytes = _prompt_cache_max_bytes(self._memory_limits_applied.get("recommended_gb"))
+ max_bytes = self._prompt_cache_bytes()
if max_bytes <= 0:
self._vlm_snapshot_store_unavailable = True
logger.info("MLX VLM prompt cache disabled by budget")
@@ -5574,7 +5614,7 @@ class MLXInferenceBackend:
_offloaded["prefill_chunk"] = self._prefill_step
_priceable = not (is_distributed or is_lora or dtype is not None)
spec_kind = _drafter_ctx = None
- _drafter_fitted = False
+ _stretched = []
_vlm_route = speculates and not is_vision
if speculates and _vlm_generation_is_diffusion(self._model):
# Neither the single path nor the resident batch may decode a diffusion model speculatively.
@@ -5594,7 +5634,7 @@ class MLXInferenceBackend:
allowed = spec_drafters_allowed,
)
- def _fits(source):
+ def _fits(source, reserve = True):
if not _priceable:
return None, None
return mlx_drafter_fit(
@@ -5605,19 +5645,27 @@ class MLXInferenceBackend:
load_in_4bit = load_in_4bit,
costless = spec_mode == "auto",
baseline = offload_baseline["route"],
+ reserve = reserve,
**_offloaded,
)
+ def _stretch(source):
+ ok, context = _fits(source, reserve = False)
+ if ok is not False:
+ _stretched.append(source)
+ return ok, context
+
(
self._speculative_draft,
spec_kind,
spec_reason,
_drafter_ctx,
) = mlx_speculative.build_draft(
- self._model, resolution, fits = _fits, draft_n_max = _positive_int(spec_draft_n_max)
- )
- _drafter_fitted = (
- self._speculative_draft is not None and self._speculative_draft.drafter is not None
+ self._model,
+ resolution,
+ fits = _fits,
+ stretch = _stretch,
+ draft_n_max = _positive_int(spec_draft_n_max),
)
logger.info(
"MLX speculative decoding: %s (mode %s, reason %s)",
@@ -5630,7 +5678,7 @@ class MLXInferenceBackend:
_requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)
_fitted_ctx, _eligibility = (
(_drafter_ctx, _eligibility)
- if _drafter_fitted
+ if _drafter_ctx is not None
else (None, _eligibility)
if not _priceable or _positive_int(max_seq_length) is not None
else _fitted_context(
@@ -5650,6 +5698,20 @@ class MLXInferenceBackend:
logger.info("MLX context fitted to %d tokens from %d", _fitted_ctx, _served_ctx or 0)
_served_ctx = _fitted_ctx
self._served_context = _served_ctx
+ # A stretched drafter spent the prompt cache's share: cap the cache to what is left beside it.
+ if (
+ _priceable
+ and _stretched
+ and getattr(self._speculative_draft, "drafter", None) is not None
+ ):
+ self._prompt_cache_room = mlx_prompt_cache_room(
+ model_dir, _served_ctx, _stretched[-1], load_in_4bit = load_in_4bit, **_offloaded
+ )
+ if self._prompt_cache_room is not None:
+ logger.info(
+ "MLX prompt cache holds at most %.2f GB beside the chosen drafter",
+ self._prompt_cache_room / 1e9,
+ )
# Classified now so a cache the model cannot attend over is refused here, not on a token.
(
self._kv_quant,
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
index 233e8b94398..94b1c17a67e 100644
--- a/studio/backend/core/inference/mlx_memory.py
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -1486,9 +1486,7 @@ def mlx_fit_outcome(
sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)
if sizing is None:
return "unsizable", None
- if any(sizing.offload):
- least, share = mlx_offload_planner().SAFETY_MARGIN
- budget_bytes = min(budget_bytes - least, budget_bytes / (1 + share))
+ budget_bytes = _fit_budget(sizing, budget_bytes)
ceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)
at_ceiling = _priced_at(sizing, ceiling)
if at_ceiling is None:
@@ -1510,6 +1508,33 @@ def mlx_fit_outcome(
return ("no_fit", None) if best is None else ("fitted", (best // MLX_KV_BLOCK) * MLX_KV_BLOCK)
+def _fit_budget(sizing: _MlxSizing, budget_bytes: int) -> float:
+ if not any(sizing.offload):
+ return budget_bytes
+ least, share = mlx_offload_planner().SAFETY_MARGIN
+ return min(budget_bytes - least, budget_bytes / (1 + share))
+
+
+def mlx_headroom(
+ model_dir: str,
+ *,
+ budget_bytes: int,
+ n_ctx: int,
+ kv_bits: Optional[int] = None,
+ prefill_chunk: Optional[int] = None,
+ load_in_4bit: bool = False,
+ vision: Optional[bool] = None,
+ drafter: Optional[tuple] = None,
+ offload: Optional[tuple] = None,
+) -> Optional[int]:
+ """Bytes of ``budget_bytes`` a load at ``n_ctx`` leaves free (as ``mlx_fit_outcome`` prices it), None if unsized."""
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)
+ priced = None if sizing is None else _priced_at(sizing, max(int(n_ctx or 0), MLX_KV_BLOCK))
+ if priced is None:
+ return None
+ return max(int(_fit_budget(sizing, budget_bytes) - priced.total_bytes), 0)
+
+
def mlx_offload_plan(
model_dir: str,
*,
diff --git a/studio/backend/core/inference/mlx_speculative.py b/studio/backend/core/inference/mlx_speculative.py
index 58de629e6ed..30e708cc819 100644
--- a/studio/backend/core/inference/mlx_speculative.py
+++ b/studio/backend/core/inference/mlx_speculative.py
@@ -373,35 +373,55 @@ def build_draft(
resolution: SpecResolution,
*,
fits,
+ stretch = None,
draft_n_max: Optional[int] = None,
) -> tuple:
"""``(draft, kind, reason, context)`` for the first source ``fits`` accepts that builds, else copies
- alone when the mode allows. ``reason`` names a passed-over choice, so a substitute never attaches silently."""
+ alone when the mode allows. ``reason`` names a passed-over choice, so a substitute never attaches silently.
+ A chosen mode with none that fits attaches an unfit source ``stretch`` (its fit under MLX's own cap) admits."""
from unsloth_zoo.mlx.speculative import companion_drafter, native_mtp_drafter
if _carries_encoder_state(target):
logger.info("MLX speculative decoding: %s carries encoder state", type(target).__name__)
return None, None, None if resolution.mode == "auto" else RUNTIME_ERROR, None
- reason = resolution.reason
- for source in resolution.sources:
- ok, context = fits(source)
- if not ok:
- if resolution.mode != "auto":
- reason = DRAFTER_NO_MEMORY
- elif ok is not None: # priced, and the drafter would shrink it
- reason = AUTO_CONTEXT_COST
- continue
+ reason, incompatible, unfit = resolution.reason, False, []
+
+ def build(source):
+ nonlocal reason, incompatible
try:
drafter = (native_mtp_drafter if source.builtin else companion_drafter)(
source.path, target
)
if drafter is None:
raise ValueError("no MTP splitter knows this checkpoint")
+ return drafter
except Exception as exc:
logger.warning("MLX drafter %s (%s) not attached: %s", source.path, source.kind, exc)
- reason = DRAFTER_INCOMPATIBLE
+ reason, incompatible = DRAFTER_INCOMPATIBLE, True
+
+ for source in resolution.sources:
+ ok, context = fits(source)
+ if ok:
+ if (drafter := build(source)) is not None:
+ return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, context
+ elif resolution.mode != "auto":
+ reason = DRAFTER_NO_MEMORY
+ unfit.append(source)
+ elif ok is not None: # priced, and the drafter would shrink it
+ reason = AUTO_CONTEXT_COST
+ # Priced one at a time, so the last source stretch admitted is the one that attaches.
+ for source in unfit if stretch else ():
+ ok, context = stretch(source)
+ if ok is False:
continue
- return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, context
+ logger.warning(
+ "MLX drafter %s (%s) fits only in the prompt cache's share; attaching it as chosen",
+ source.path,
+ source.kind,
+ )
+ if (drafter := build(source)) is not None:
+ reason = DRAFTER_INCOMPATIBLE if incompatible else resolution.reason
+ return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, context
if resolution.copies and resolution.mode != "auto": # auto copies only alongside a drafter
return _draft(None, True, draft_n_max), "ngram", reason, None
return None, None, reason, None
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 741d5d978fd..817220e68ac 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1891,8 +1891,10 @@ class _InferenceRuntimeFields(BaseModel):
"MTP in Settings to force it. "
"MLX loads report 'drafter_not_found' (no drafter of the requested kind "
"is cached), 'drafter_incompatible' (the named or found drafter does not "
- "fit this model), 'drafter_no_memory' (its weights and cache leave no "
- "context that fits), 'kv_quant' (KV cache quantization is on) or "
+ "fit this model), 'drafter_no_memory' (a drafter whose weights and cache "
+ "leave no context that fits was passed over; a chosen mode still attaches "
+ "one that fits by spending the prompt cache's share of memory), 'kv_quant' (KV "
+ "cache quantization is on) or "
"'runtime_error' (the installed MLX packages cannot speculate, or cannot "
"load this model the way speculation needs), also when a later drafter "
"or n-gram copies stood in for the one asked for. Under auto an MLX load "
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3371d64b73a..2c857832e35 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -14728,13 +14728,14 @@ def _mlx_estimate_drafter(
offload_route: Optional[dict] = None,
baseline: Optional[dict] = None,
):
- """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it
- would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""
+ """``((path, builtin), fitted context, speculates, stretched)`` of an MLX load of *request*: the drafter it
+ would attach, chosen as the load chooses it, whether it speculates at all (so loads through mlx-vlm), and
+ whether that drafter fits only by spending the prompt cache's share."""
from core.inference import mlx_speculative
from core.inference.mlx_inference import mlx_drafter_fit
if not _mlx_estimate_speculates(request, model_dir):
- return None, None, False
+ return None, None, False, False
mode = mlx_speculative.mlx_spec_mode(request.speculative_type)
resolution = mlx_speculative.resolve_speculation(
request.speculative_type,
@@ -14743,8 +14744,9 @@ def _mlx_estimate_drafter(
target_name = model_identifier,
allowed = _discoverable_drafters(model_identifier),
)
- for source in resolution.sources:
- attaches, fitted = mlx_drafter_fit(
+
+ def fit(source, **kwargs):
+ return mlx_drafter_fit(
model_dir,
ceiling,
request.max_seq_length or None,
@@ -14752,11 +14754,20 @@ def _mlx_estimate_drafter(
load_in_4bit = load_in_4bit,
costless = mode == "auto",
baseline = baseline,
+ **kwargs,
**(offload_route or {}),
)
+
+ for source in resolution.sources:
+ attaches, fitted = fit(source)
if attaches:
- return (source.path, source.builtin), fitted, True
- return None, None, True
+ return (source.path, source.builtin), fitted, True, False
+ # Like the load: a chosen mode with none that fits attaches the first MLX's own cap admits.
+ for source in resolution.sources if mode != "auto" else ():
+ attaches, fitted = fit(source, reserve = False)
+ if attaches is not False:
+ return (source.path, source.builtin), fitted, True, True
+ return None, None, True, False
class _MlxLoadEstimate(NamedTuple):
@@ -14765,6 +14776,7 @@ class _MlxLoadEstimate(NamedTuple):
offloadable: tuple
retains_history: bool
offloaded: bool = False
+ cache_room: Optional[int] = None
def _mlx_load_estimate(
@@ -14797,7 +14809,7 @@ def _mlx_load_estimate(
offload, offloadable, offload_baseline = _mlx_estimate_offload(
request, config, model_identifier, model_dir, load_in_4bit, kv_bits
)
- drafter, fitted_ctx, speculates = _mlx_estimate_drafter(
+ drafter, fitted_ctx, speculates, stretched = _mlx_estimate_drafter(
request,
model_identifier,
model_dir,
@@ -14807,19 +14819,32 @@ def _mlx_load_estimate(
offload_baseline,
)
route = {**({"vision": True} if speculates else {}), **offload}
- if not named_ctx and drafter is None:
+ if not named_ctx and fitted_ctx is None:
fitted_ctx = _mlx_estimate_fitted_context(config, model_dir, load_in_4bit, kv_bits, **route)
priced_ctx = named_ctx or fitted_ctx or _mlx_estimate_ceiling(model_dir)
if not priced_ctx:
return "unsizable", None
+ priced = dict(n_ctx = priced_ctx, kv_bits = kv_bits, load_in_4bit = load_in_4bit, **route)
breakdown = mlx_memory_breakdown(
- model_dir,
- n_ctx = priced_ctx,
- kv_bits = kv_bits,
- load_in_4bit = load_in_4bit,
- **route,
- **({} if drafter is None else {"drafter": drafter}),
+ model_dir, **priced, **({} if drafter is None else {"drafter": drafter})
)
+ cache_room = None
+ if breakdown is None and drafter is not None:
+ # A chosen drafter attaches even where it cannot be sized: price the rest of the load.
+ breakdown = mlx_memory_breakdown(model_dir, **priced)
+ elif stretched:
+ from types import SimpleNamespace
+
+ from core.inference.mlx_inference import mlx_prompt_cache_room
+
+ # The load caps the prompt cache to what MLX's own cap leaves beside this drafter.
+ cache_room = mlx_prompt_cache_room(
+ model_dir,
+ priced_ctx,
+ SimpleNamespace(path = drafter[0], builtin = drafter[1]),
+ load_in_4bit = load_in_4bit,
+ **offload,
+ )
if breakdown is None:
return "unsizable", None
retains_history = (
@@ -14827,7 +14852,7 @@ def _mlx_load_estimate(
or mlx_vlm_snapshot_store_available()
)
return None, _MlxLoadEstimate(
- breakdown, fitted_ctx, offloadable, retains_history, "offload" in offload
+ breakdown, fitted_ctx, offloadable, retains_history, "offload" in offload, cache_room
)
@@ -14854,6 +14879,8 @@ def _mlx_chat_need(
mlx_memory_budget(retains_history = True),
)
reserve = max(0, (without or 0) - (within or 0))
+ if estimate.cache_room is not None:
+ reserve = min(reserve, estimate.cache_room)
except Exception as exc:
logger.info("MLX load of %s left unpriced: %s", model_identifier, exc)
return None
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index 9a5d5e8c3f4..7d6ff7908f9 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -820,12 +820,17 @@ class TestEstimateMemoryRoute:
monkeypatch.setattr(
spec, "speculation_refusal", lambda kv_quant, **_: spec.KV_QUANT if kv_quant else None
)
+ fit = False
monkeypatch.setattr(
mlx_inference,
"mlx_drafter_fit",
- lambda d, ceiling, pinned, source, **_: (
- source.path == "/ok" and (pinned or 0) <= 8192,
- None if pinned else 12288,
+ lambda d, ceiling, pinned, source, reserve = True, **_: (
+ None
+ if fit in ("unpriced", "unsizable")
+ else source.path == "/ok" and (pinned or 0) <= 8192 and not fit
+ if reserve
+ else (pinned or 0) <= 32768, # MLX's own cap, spending the prompt cache's share
+ None if pinned or fit in ("unpriced", "unsizable") else 12288,
),
)
monkeypatch.setattr(ri, "_mlx_estimate_ceiling", lambda model_dir: 32768)
@@ -837,31 +842,66 @@ class TestEstimateMemoryRoute:
mlx_memory,
"mlx_memory_breakdown",
lambda d, *, n_ctx, **kw: priced.append((n_ctx, kw.get("drafter"), kw.get("vision")))
- or mlx_memory.MlxMemoryBreakdown(1, 0, 0, 1, 1, n_ctx = n_ctx),
+ or (
+ None
+ if fit == "unsizable" and kw.get("drafter")
+ else mlx_memory.MlxMemoryBreakdown(1, 0, 0, 1, 1, n_ctx = n_ctx)
+ ),
+ )
+ rooms, estimates, load_estimate = [], [], ri._mlx_load_estimate
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_prompt_cache_room",
+ lambda d, n_ctx, source, **_: rooms.append((n_ctx, source.path)) or 7,
+ )
+ monkeypatch.setattr(
+ ri,
+ "_mlx_load_estimate",
+ lambda *a: (result := load_estimate(*a), estimates.append(result[1]))[0],
)
assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288
- for pinned in (4096, 65536):
+ for pinned in (4096, 16384, 65536):
_estimate(model_path = "org/model", speculative_type = "mtp", max_seq_length = pinned)
_estimate(model_path = "org/model", speculative_type = "mtp", mlx_kv_quant = "8")
_estimate(model_path = "org/model")
_estimate(model_path = "o/m", speculative_type = "ngram", mlx_kv_quant = "auto", mlx_kv_bits = 8)
_estimate(model_path = "org/model", spec_draft_model = "org/d") # named under Auto
- assert [route for *_, route in priced] == [True, True, True, None, None, True, True]
- assert [row[:2] for row in priced[:6]] == [
+ fit = "unfit"
+ assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288
+ fit = "unpriced" # no fit to go on: the drafter the load attaches is still charged
+ _estimate(model_path = "org/model", speculative_type = "mtp")
+ fit = "unsizable" # one the breakdown cannot size leaves the rest of the load priced
+ assert _estimate(model_path = "org/model", speculative_type = "mtp").available
+ assert [route for *_, route in priced] == [True] * 4 + [None] * 2 + [True] * 6
+ assert [row[:2] for row in priced[:7] + priced[8:]] == [
(12288, ("/ok", False)),
(4096, ("/ok", False)),
- (65536, None),
+ (16384, ("/big", True)),
+ (65536, None), # past MLX's own cap the load falls back, so no drafter is charged
(20480, None),
(20480, None),
(24576, None),
+ (12288, ("/big", True)),
+ (24576, ("/big", True)),
+ (24576, ("/big", True)),
+ (24576, None),
]
+ # Only a drafter that spent the prompt cache's share caps the cache admission charges.
+ assert rooms == [(16384, "/big"), (12288, "/big"), (24576, "/big")]
+ assert [e.cache_room for e in estimates if e and e.cache_room] == [7, 7, 7]
def test_a_drafter_attaches_where_its_priced_fit_holds_and_a_pin_fits_whole(self, monkeypatch):
import core.inference.mlx_inference as mlx_inference
import core.inference.mlx_memory as mlx_memory
from core.inference.mlx_speculative import DrafterSource
- monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 << 30)
+ budgets = []
+ monkeypatch.setattr(mlx_inference, "mlx_vlm_snapshot_store_available", lambda: True)
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_memory_budget",
+ lambda *, retains_history: budgets.append(retains_history) or 8 << 30,
+ )
priced = []
def fit(pinned, outcome, *bare):
@@ -902,6 +942,9 @@ class TestEstimateMemoryRoute:
prefill_chunk = 8192,
)
assert (asked["offload"], asked["prefill_chunk"]) == ((None, 0.6), 8192)
+ assert set(budgets) == {True}
+ mlx_inference.mlx_drafter_fit("/m", 32768, None, source, load_in_4bit = False, reserve = False)
+ assert budgets[-1] is False # MLX's own cap: the prompt cache's share is spendable
seen = []
def costless(attached, alone):
@@ -4795,6 +4838,10 @@ class TestPricingALoad:
str(tmp_path), budget_bytes = price + (2 << 30), max_ctx = 1 << 20, offload = offload
)
assert fit((None, 0.5)) == ("fitted", 8192) != fit(None)
+ room = lambda offload: mm.mlx_headroom(
+ str(tmp_path), budget_bytes = price + (2 << 30) + 5, n_ctx = 8192, offload = offload
+ )
+ assert room((None, 0.5)) == 5 # the room keeps the same margin free as the fit
def test_a_quantizing_load_is_priced_below_the_shards(self, tmp_path):
config = self._checkpoint(tmp_path)
@@ -5205,8 +5252,12 @@ class TestTheContextSearch:
"no_fit",
None,
)
+ assert mm.mlx_headroom("x", budget_bytes = 9000, n_ctx = 8192) == 808
+ assert mm.mlx_headroom("x", budget_bytes = 9000, n_ctx = 100) == 9000 - 256
+ assert mm.mlx_headroom("x", budget_bytes = 7000, n_ctx = 8192) == 0
monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)
assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("unsizable", None)
+ assert mm.mlx_headroom("x", budget_bytes = 7000, n_ctx = 8192) is None
@_NEEDS_MLX
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index d5c1c23e371..57d07b2cfdb 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -1336,11 +1336,17 @@ def test_an_auto_drafter_gets_offload_room_only_within_the_cap_where_it_then_fit
]
plans.update({True: (0.5, 0.5, 8192)})
assert resolve(True) == ((0.5, 0.5, 8192),) * 2 and len(asked) == 1
- # An explicit drafter priced not to fit its room plan keeps the target-only plan, not all-on-disk.
+ # An explicit drafter priced not to fit its room plan, even under MLX's own cap, keeps the
+ # target-only plan, not all-on-disk.
plans.update({True: (None, 1.0, None), False: (None, 0.78, 8192)})
asked.clear()
assert resolve(False) == ((None, 0.78, 8192), None)
- assert asked[-1][4] == {"load_in_4bit": False, "offload": (None, 1.0), "prefill_chunk": None}
+ assert asked[-1][4] == {
+ "load_in_4bit": False,
+ "offload": (None, 1.0),
+ "prefill_chunk": None,
+ "reserve": False,
+ }
fits.update(on = None) # unpriceable: the load keeps it, so does the plan
assert resolve(False) == ((None, 1.0, None), None)
@@ -4995,18 +5001,27 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fi
monkeypatch.setattr(
mlx_speculative,
"resolve_speculation",
- lambda mode, *_a, **_k: mlx_speculative.SpecResolution(mode, (source,)),
+ lambda mode, *_a, **_k: mlx_speculative.SpecResolution(
+ mode, () if mode == "ngram" else (source,)
+ ),
)
monkeypatch.setattr(
mlx_inference, "mlx_drafter_fit", lambda *a, **k: fits.append(a[1:4]) or (True, 4096)
)
monkeypatch.setattr(drafters, "companion_drafter", lambda path, target: SimpleNamespace())
+ spent, rooms = [], {"/drafter": 123, "/cheap": 500, "/unused": 0}
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_prompt_cache_room",
+ lambda model_dir, n_ctx, source, **_k: spent.append((n_ctx, source)) or rooms[source.path],
+ )
config = SimpleNamespace(identifier = "org/text", is_vision = False, is_lora = False)
backend = mlx_inference.MLXInferenceBackend()
backend.load_model(config, max_seq_length = 0, speculative_type = "dflash", spec_draft_n_max = 5)
entry = backend.models["org/text"]
assert attempts == [False] and fits == [(8192, None, source)]
+ assert backend._prompt_cache_room is None and spent == [] # a drafter that fits spends no share
assert (entry["spec_drafter_kind"], entry["spec_fallback_reason"]) == ("dflash", None)
assert entry["context_length"] == 4096 and entry["spec_draft_n_max"] == 5
assert backend._speculative_draft.controller.max_depth == 5
@@ -5022,9 +5037,57 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fi
failing.clear()
fit = lambda *a, **kw: fits.append(kw.get("vision")) or (None, None)
monkeypatch.setattr(mlx_inference, "_fitted_context", fit)
- monkeypatch.setattr(mlx_inference, "mlx_drafter_fit", lambda *a, **k: (False, None))
+ room = [True] # whether the drafter fits under MLX's own cap, spending the prompt cache's share
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_drafter_fit",
+ lambda *a, reserve = True, **k: (not reserve and room[-1], None),
+ )
backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")
assert fits[-1] is True # copies alone still load through mlx-vlm, so the fit prices that route
+ fits.clear()
+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash")
+ assert backend._speculative_draft.drafter is not None and fits == [True]
+ assert backend.models["org/text"]["spec_fallback_reason"] is None
+ # That drafter spent the prompt cache's share, so both caches hold only what the cap still leaves.
+ assert spent == [(8192, source)] and backend._prompt_cache_room == 123
+ sized = []
+ monkeypatch.setattr(
+ mlx_inference,
+ "_MLXPromptCacheHistory",
+ lambda n, max_bytes, *a, **k: sized.append(max_bytes),
+ )
+ monkeypatch.setattr(
+ mlx_inference, "VLMPromptSnapshotStore", lambda max_bytes: sized.append(max_bytes)
+ )
+ backend._prompt_cache(), backend._vlm_prompt_cache_store()
+ assert sized in ([123, 123], [123]) # the second only where this mlx-vlm builds the store
+ # The room is the attached drafter's: not one that failed to build, nor one never tried.
+ cheap = mlx_speculative.DrafterSource("dflash", "/cheap", False)
+ unused = mlx_speculative.DrafterSource("dflash", "/unused", False)
+ resolve, build = mlx_speculative.resolve_speculation, drafters.companion_drafter
+ monkeypatch.setattr(
+ mlx_speculative,
+ "resolve_speculation",
+ lambda *_a, **_k: mlx_speculative.SpecResolution("dflash", (cheap, source, unused)),
+ )
+ monkeypatch.setattr(
+ drafters,
+ "companion_drafter",
+ lambda path, target: (_ for _ in ()).throw(ValueError("no"))
+ if path == "/cheap"
+ else build(path, target),
+ )
+ spent.clear()
+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash")
+ assert backend._speculative_draft.drafter is not None and backend._prompt_cache_room == 123
+ assert spent == [(8192, source)]
+ monkeypatch.setattr(mlx_speculative, "resolve_speculation", resolve)
+ monkeypatch.setattr(drafters, "companion_drafter", build)
+ room.append(False)
+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash")
+ assert backend._speculative_draft is None and backend._prompt_cache_room is None
+ assert backend.models["org/text"]["spec_fallback_reason"] == mlx_speculative.DRAFTER_NO_MEMORY
backend.load_model(config, max_seq_length = 0, spec_draft_model = "org/d")
assert fits[-2:] == [True, True] # a named drafter enters the speculative route under Auto
# ... and is a request, so a load that cannot honour it says why.
diff --git a/studio/backend/tests/test_mlx_speculative.py b/studio/backend/tests/test_mlx_speculative.py
index d1144afa5e2..9bc14d8083a 100644
--- a/studio/backend/tests/test_mlx_speculative.py
+++ b/studio/backend/tests/test_mlx_speculative.py
@@ -167,11 +167,36 @@ def test_a_drafter_passed_over_keeps_its_reason_on_the_one_that_attaches(monkeyp
assert (draft.controller.max_depth, draft.controller.max_copy) == (6, 0)
for d, n, e in ((object(), 3, True), (object(), 4, False), (draft.drafter, 15, True)):
assert spec._draft(d, False, n).controller.fixed_depth is e
- draft, kind, reason, _ = spec.build_draft(
- None, spec.SpecResolution("dflash", sources[:1], copies = True), fits = fits
+
+ def chosen(
+ picked,
+ fit,
+ stretch = lambda _: (None, None),
+ ):
+ built.clear()
+ result = spec.build_draft(
+ None, spec.SpecResolution("dflash", picked, copies = True), fits = fit, stretch = stretch
+ )
+ return result[1:] + (built[:],)
+
+ assert chosen(sources[::2], fits) == ("dflash", spec.DRAFTER_NO_MEMORY, 4096, ["ok"])
+ assert chosen(sources[:1], fits) == ("dflash", None, None, ["big"])
+ assert chosen(sources[:1], lambda _: (None, None)) == ("dflash", None, None, ["big"])
+ assert chosen(sources[:1], fits, lambda _: (True, 2048)) == ("dflash", None, 2048, ["big"])
+ # Past MLX's own cap a chosen drafter is still passed over, as is one nothing can stretch.
+ for stretch in (lambda _: (False, None), None):
+ assert chosen(sources[:1], fits, stretch) == ("ngram", spec.DRAFTER_NO_MEMORY, None, [])
+ assert chosen(sources[:2], fits) == ("dflash", spec.DRAFTER_INCOMPATIBLE, None, ["big"])
+ assert chosen(sources[1:2], lambda _: (False, 4096)) == (
+ "ngram",
+ spec.DRAFTER_INCOMPATIBLE,
+ None,
+ [],
+ )
+ draft, *_ = spec.build_draft(
+ None, spec.SpecResolution("dflash", sources[1:2], copies = True), fits = lambda _: (False, 4096)
)
- assert (draft.drafter, kind, reason) == (None, "ngram", spec.DRAFTER_NO_MEMORY)
- assert (draft.controller.max_depth, draft.controller.max_copy) == (0, 16)
+ assert (draft.drafter, draft.controller.max_depth, draft.controller.max_copy) == (None, 0, 16)
auto = spec.SpecResolution("auto", sources[:1], copies = True)
unfit = [spec.build_draft(None, auto, fits = lambda _: (ok, None)) for ok in (False, None)]
assert unfit == [(None, None, spec.AUTO_CONTEXT_COST, None), (None,) * 4]
diff --git a/studio/backend/tests/test_multi_model_primary_fit.py b/studio/backend/tests/test_multi_model_primary_fit.py
index f2f6032a1c2..dcea572a847 100644
--- a/studio/backend/tests/test_multi_model_primary_fit.py
+++ b/studio/backend/tests/test_multi_model_primary_fit.py
@@ -312,6 +312,11 @@ def test_mlx_chat_need_prices_one_row_the_prompt_cache_and_a_batch_row_per_extra
monkeypatch.setattr(inf, "_unified_metal_host", lambda: True)
need, growth = inf._mlx_chat_need(None, mlx, "m", 4)
assert (need, growth) == (4096 + 3 * 768, {"mib": 3072, "row_mib": 768, "slots": 4})
+ # A drafter that spent the prompt cache's share leaves the cache only the room the load caps it to.
+ for room, reserved in ((256 << 20, 256), (4 * gib, 1024)):
+ stretched = estimate._replace(cache_room = room)
+ monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, stretched))
+ assert inf._mlx_chat_need(None, mlx, "m", 4)[0] == 3072 + reserved + 3 * 768
offloaded = estimate._replace(offloaded = True)
monkeypatch.setattr(inf, "_mlx_load_estimate", lambda *a: (None, offloaded))
assert inf._mlx_chat_need(None, mlx, "m", 4) is None