Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: speculative decoding for the MLX inference backend (#13014)

Long Yixing6bd164faafbc
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 87e6c0525e5..d90294a907c 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -1722,9 +1722,10 @@ def _build_generation_stats(
gen_tps,
cached_n = 0,
finish_reason = None,
+ drafts = None,
):
"""Map mlx stream stats onto the usage/timings shape llama-server emits, plus the reason
- generation ended."""
+ generation ended. ``drafts`` is a speculative reply's (drafted, accepted) token counts."""
prompt_n = int(prompt_n or 0)
gen_n = int(gen_n or 0)
cached_n = int(cached_n or 0)
@@ -1752,6 +1753,7 @@ def _build_generation_stats(
"predicted_per_token_ms": (predicted_ms / gen_n) if gen_n > 0 else 0.0,
"predicted_per_second": gen_tps,
"cache_n": cached_n,
+ **({} if drafts is None else {"draft_n": drafts[0], "draft_n_accepted": drafts[1]}),
},
# Latched where generation exits, so a cancel arriving afterwards cannot rewrite the reason the completion
# actually ended for.
@@ -2568,8 +2570,10 @@ def mlx_fit_to_memory(
load_in_4bit: bool,
retains_history: bool,
kv_bits = None,
+ **route,
) -> Optional[int]:
- """The context a load of *model_dir* is held to, priced at the KV width its cache takes."""
+ """The context a load of *model_dir* is held to, priced at the KV width its cache takes.
+ ``vision=True`` in *route* prices the mlx-vlm route a speculative text load takes."""
budget = mlx_memory_budget(retains_history = retains_history)
if budget is None or not ceiling or not model_dir:
return None
@@ -2583,6 +2587,7 @@ def mlx_fit_to_memory(
max_ctx = int(ceiling),
kv_bits = kv_bits,
load_in_4bit = load_in_4bit,
+ **route,
)
logger.debug("MLX fit for %s: %s tokens under %.1f GB", model_dir, fitted, budget / 1e9)
return fitted
@@ -2601,6 +2606,7 @@ def _fitted_context(
kv_bits = None,
is_vlm = False,
eligibility = None,
+ **route,
):
"""The fit for a resident model and the eligibility verdict it was priced under."""
try:
@@ -2617,10 +2623,50 @@ def _fitted_context(
load_in_4bit = load_in_4bit,
retains_history = retains_history,
kv_bits = kv_bits if verdict is None or verdict[0] in ("full", "partial") else None,
+ **route,
)
return fitted, verdict
+def mlx_drafter_fit(
+ model_dir,
+ ceiling,
+ pinned,
+ source,
+ *,
+ load_in_4bit: bool,
+ costless = False,
+):
+ """``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:
+ only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit
+ the context the load fits without it. ``attaches`` is None when the fit could not be priced."""
+ budget = mlx_memory_budget(retains_history = mlx_vlm_snapshot_store_available())
+ ceiling = pinned or ceiling
+ if budget is None or not ceiling or not model_dir:
+ return None, None
+ with mlx_rng_preserved():
+ try:
+ from core.inference.mlx_memory import mlx_fit_outcome
+
+ fit = functools.partial(
+ mlx_fit_outcome,
+ model_dir,
+ budget_bytes = budget,
+ max_ctx = int(ceiling),
+ load_in_4bit = load_in_4bit,
+ vision = True,
+ )
+ outcome, fitted = fit(drafter = (source.path, source.builtin))
+ if outcome == "unsizable":
+ return None, fitted
+ if costless and (outcome, fitted) != fit():
+ return False, fitted
+ except Exception as exc:
+ logger.info("MLX drafter %s not priced: %s", source.path, exc)
+ return None, None
+ return outcome == "fits" or (outcome == "fitted" and not pinned), fitted
+
+
def _kv_window_enforced(model, is_vlm, window):
"""Whether a cache built at *window* is bounded in every layer, None where none could be built
to judge: both runtimes defer to a model's own make_cache and ignore max_kv_size there."""
@@ -3072,6 +3118,7 @@ def _make_mlx_presence_penalty_processor(penalty: float):
logits = logits - mask[:vocab]
return logits
+ _processor.history_only = True
return _processor
@@ -3116,6 +3163,7 @@ def _make_mlx_frequency_penalty_processor(penalty: float):
counts = mx.zeros((vocab + 1,), dtype = mx.float32).at[safe].add(1.0)
return logits - (penalty * counts[:vocab]).astype(logits.dtype)
+ _processor.history_only = True
return _processor
@@ -3140,6 +3188,7 @@ def _make_mlx_logit_bias_processor(logit_bias: dict):
mask = mx.zeros((vocab + 1,), dtype = mx.float32).at[state["safe"]].add(state["values"])
return logits + mask[:vocab].astype(logits.dtype)
+ _processor.history_only = True
return _processor
@@ -3203,7 +3252,11 @@ def _mlx_sampling_processors(
processors.append(_make_mlx_logit_bias_processor(logit_bias))
if repetition_penalty is not None and float(repetition_penalty) not in (0.0, 1.0):
from mlx_lm.sample_utils import make_logits_processors
- processors.extend(make_logits_processors(repetition_penalty = float(repetition_penalty)))
+
+ repetition = make_logits_processors(repetition_penalty = float(repetition_penalty))
+ for processor in repetition:
+ processor.history_only = True
+ processors.extend(repetition)
if presence_penalty:
processors.append(_make_mlx_presence_penalty_processor(float(presence_penalty)))
if frequency_penalty:
@@ -3232,34 +3285,44 @@ def _vlm_add_special_tokens(model_type, processor):
)
+def _vlm_prepared_inputs(model, processor, prompt, images):
+ """The processor's inputs for *prompt* and *images*, as mlx-vlm's ``stream_generate`` prepares them."""
+ from mlx_vlm.utils import prepare_inputs
+
+ config = getattr(model, "config", None)
+ read = config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)
+ return prepare_inputs(
+ processor,
+ images = images,
+ prompts = prompt,
+ image_token_index = read("image_token_index"),
+ add_special_tokens = _vlm_add_special_tokens(read("model_type"), processor),
+ )
+
+
+def _vlm_generate_kwargs(inputs):
+ """Prepared inputs as ``stream_generate`` kwargs, which then skips its own preparation."""
+ inputs = dict(inputs)
+ return {
+ "input_ids": inputs.pop("input_ids"),
+ "pixel_values": inputs.pop("pixel_values", None),
+ "mask": inputs.pop("attention_mask", None),
+ **inputs,
+ }
+
+
class _VLMMediaBlock:
"""A prompt's rows through its last vision token, prefilled by Studio: models whose vision
tokens attend to each other prefill in one forward that lands off the grid, and this is that
forward cut where the grid can continue."""
def __init__(self, model, processor, prompt, images, make_cache):
- from mlx_vlm.utils import prepare_inputs
-
self._model = model
self._make_cache = make_cache
- config = getattr(model, "config", None)
- read = config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)
- self._inputs = prepare_inputs(
- processor,
- images = images,
- prompts = prompt,
- image_token_index = read("image_token_index"),
- add_special_tokens = _vlm_add_special_tokens(read("model_type"), processor),
- )
+ self._inputs = _vlm_prepared_inputs(model, processor, prompt, images)
def generate_kwargs(self):
- inputs = dict(self._inputs)
- return {
- "input_ids": inputs.pop("input_ids"),
- "pixel_values": inputs.pop("pixel_values", None),
- "mask": inputs.pop("attention_mask", None),
- **inputs,
- }
+ return _vlm_generate_kwargs(self._inputs)
def rows(self, token_ids):
ids = self._inputs["input_ids"]
@@ -3623,7 +3686,7 @@ class _TextBatchSession:
)
return plan.think_prefix
- def step(self):
+ def step(self, waiting = None):
"""Report what the batch produced, retiring every reply that ended."""
began = time.perf_counter()
prompt_events, events = self.generator.next()
@@ -3771,6 +3834,7 @@ class _VisionBatchRow:
"admitted_at",
"ready_at",
"cancelled",
+ "drafts",
)
def __init__(self, *, handle, plan, row):
@@ -3784,6 +3848,7 @@ class _VisionBatchRow:
self.admitted_at = time.perf_counter()
self.ready_at = None
self.cancelled = False
+ self.drafts = None
def _row_prompt_cache_gap():
@@ -3827,6 +3892,7 @@ class _VisionBatchSession:
_temporary_mlx_adapter_state(backend._model, adapter_state)
)
self._held.enter_context(backend._int8_prefill_scope(zoo_generation = True))
+ self._speculative = getattr(backend, "_speculative_draft", None)
self.stream = BatchStream(
backend._model,
backend._processor,
@@ -3836,6 +3902,7 @@ class _VisionBatchSession:
# Rows quantize their own caches where the single path's decode does.
**backend._kv_runtime_quant_kwargs(),
),
+ **({} if self._speculative is None else {"speculative": self._speculative}),
)
except BaseException:
self._held.close()
@@ -3901,9 +3968,15 @@ class _VisionBatchSession:
)
return plan.think_prefix
- def step(self):
- now = None
- for event in self.stream.step():
+ def step(self, waiting = None):
+ now, stopped = None, {}
+ # Only a speculative step spans many tokens, reported as they are decoded; only its unsloth-zoo has the call.
+ # The batch cannot change mid-step, so a row that stopped itself ends the step and leaves after it.
+ for event in (
+ self.stream.iter_step(lambda: bool(stopped) or (waiting is not None and waiting()))
+ if self._speculative is not None
+ else self.stream.step()
+ ):
row = self._by_row.get(event.index)
if row is None:
continue
@@ -3922,11 +3995,12 @@ class _VisionBatchSession:
result = event.result
if not row.cancelled:
row.reason = result.finish_reason
- row.prompt_tokens = result.prompt_token_count
- row.cached_tokens = getattr(result, "cached_token_count", 0)
- row.generated = len(result.token_ids) + (result.finish_reason == "stop")
+ self._record(row, result, len(result.token_ids) + (result.finish_reason == "stop"))
yield from self._retire(row, cancelled = row.cancelled)
elif row.cancelled:
+ stopped[row.handle] = row
+ for row in stopped.values():
+ if row.handle in self._rows:
yield from self._retake(row)
def withdraw(self, handles):
@@ -3941,11 +4015,15 @@ class _VisionBatchSession:
def _retake(self, row):
managed = self.stream.withdraw(row.row)
if managed is not None:
- row.prompt_tokens = managed.prompt_token_count
- row.cached_tokens = getattr(managed, "cached_token_count", 0)
- row.generated = len(managed.token_ids)
+ self._record(row, managed, len(managed.token_ids))
yield from self._retire(row, cancelled = True)
+ def _record(self, row, result, generated):
+ row.prompt_tokens, row.generated = result.prompt_token_count, generated
+ row.cached_tokens = getattr(result, "cached_token_count", 0)
+ if self._speculative is not None:
+ row.drafts = (result.draft_tokens, result.accepted_draft_tokens)
+
def _retire(self, row, *, cancelled):
self._rows.pop(row.handle, None)
self._by_row.pop(row.row, None)
@@ -3963,6 +4041,7 @@ class _VisionBatchSession:
row.generated / max(decoded, 1e-9),
cached_n = row.cached_tokens,
finish_reason = row.reason or "stop",
+ drafts = row.drafts,
)
yield row.handle, None
@@ -3995,6 +4074,7 @@ class MLXInferenceBackend:
self._is_vlm = False
self._reads_vision = None
self._turboquant_refusal = ""
+ self._speculative_draft = None
self._multi_image_marker = None
self._config = {}
self._distributed_group = None
@@ -4555,9 +4635,15 @@ class MLXInferenceBackend:
kv_quant = None,
chat_template_override = None,
int8_prefill = False,
+ speculative_type = None,
+ spec_draft_n_max = None,
+ spec_draft_model = None,
+ spec_drafters_allowed = None,
) -> bool:
import mlx.core as mx
+ from core.inference import mlx_speculative
+
# Keep the token so the native-template fallback can fetch a gated model's repo template during generation.
self._hf_token = hf_token
model_name = config.identifier if hasattr(config, "identifier") else str(config)
@@ -4579,14 +4665,33 @@ class MLXInferenceBackend:
if self._turboquant
else ""
)
+ spec_mode = mlx_speculative.mlx_spec_mode(speculative_type)
+ spec_reason = None
+ speculates = mlx_speculative.speculates_on_route(
+ spec_mode, bool(is_vision), spec_draft_model
+ )
+ if speculates:
+ spec_reason = mlx_speculative.speculation_refusal(
+ kv_quant = kv_bits is not None or self._turboquant,
+ distributed = is_distributed,
+ lora = is_lora,
+ )
+ speculates = spec_reason is None
+ if (
+ spec_mode == "auto" and not spec_draft_model
+ ): # not a request, so no refusal to report
+ spec_reason = None
+ self._speculative_draft = None
# mlx-lm cannot batch a quantized cache; mlx-vlm, which also ships text architectures, can.
batches_quantized_text = (
kv_bits is not None
and not (is_distributed or is_lora)
and _row_quantized_cache_gap() is None
)
+ # Speculation runs on mlx-vlm models and caches.
use_vlm = (
is_vision
+ or speculates
or (self._turboquant and not self._turboquant_refusal)
or batches_quantized_text
)
@@ -4669,8 +4774,9 @@ class MLXInferenceBackend:
if is_vision or not use_vlm or is_metal_queue_dead(exc):
raise
logger.warning(
- "Load of %s through mlx-vlm for its KV quantization failed (%s); serving through mlx-lm",
+ "Load of %s through mlx-vlm for %s failed (%s); serving through mlx-lm",
model_name,
+ "speculative decoding" if speculates else "its KV quantization",
exc,
)
_eligibility = None
@@ -4692,7 +4798,14 @@ class MLXInferenceBackend:
gc.collect()
_drain_generation_streams(mx)
mx.clear_cache()
- if self._turboquant:
+ if speculates:
+ speculates = False
+ spec_reason = (
+ None
+ if spec_mode == "auto" and not spec_draft_model
+ else mlx_speculative.RUNTIME_ERROR
+ )
+ elif self._turboquant:
self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD
use_vlm = False
load_kwargs["text_only"] = True
@@ -4727,11 +4840,63 @@ class MLXInferenceBackend:
self._model, max_seq_length
)
_priceable = not (is_distributed or is_lora or dtype is not None)
+ spec_kind = _drafter_ctx = None
+ _drafter_fitted = False
+ _vlm_route = speculates and not is_vision
+ if speculates and _vlm_generation_is_diffusion(self._model):
+ # Neither the single path nor the resident batch may decode a diffusion model speculatively.
+ speculates = False
+ spec_reason = (
+ None
+ if spec_mode == "auto" and not spec_draft_model
+ else mlx_speculative.RUNTIME_ERROR
+ )
+ if speculates:
+ model_dir = _snapshot_dir(self._model, model_name)
+ resolution = mlx_speculative.resolve_speculation(
+ spec_mode,
+ spec_draft_model,
+ model_dir = model_dir,
+ target_name = model_name,
+ allowed = spec_drafters_allowed,
+ )
+
+ def _fits(source):
+ if not _priceable:
+ return None, None
+ return mlx_drafter_fit(
+ model_dir,
+ _served_ctx,
+ _positive_int(max_seq_length),
+ source,
+ load_in_4bit = load_in_4bit,
+ costless = spec_mode == "auto",
+ )
+
+ (
+ self._speculative_draft,
+ spec_kind,
+ spec_reason,
+ _drafter_ctx,
+ ) = mlx_speculative.build_draft(
+ self._model, resolution, fits = _fits, draft_n_max = _positive_int(spec_draft_n_max)
+ )
+ _drafter_fitted = (
+ self._speculative_draft is not None and self._speculative_draft.drafter is not None
+ )
+ logger.info(
+ "MLX speculative decoding: %s (mode %s, reason %s)",
+ spec_kind or "off",
+ spec_mode,
+ spec_reason,
+ )
# TurboQuant's layout is not one the planner prices, and a refused one stays full width:
# fit either at full width, which only ever under-promises.
_requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)
_fitted_ctx, _eligibility = (
- (None, _eligibility)
+ (_drafter_ctx, _eligibility)
+ if _drafter_fitted
+ else (None, _eligibility)
if not _priceable or _positive_int(max_seq_length) is not None
else _fitted_context(
self._model,
@@ -4742,6 +4907,7 @@ class MLXInferenceBackend:
kv_bits = _requested_bits,
is_vlm = use_vlm,
eligibility = _eligibility,
+ **({"vision": True} if _vlm_route else {}),
)
)
if _fitted_ctx:
@@ -4858,6 +5024,11 @@ class MLXInferenceBackend:
"mlx_int8_prefill_reason": self._int8_prefill["reason"],
"chat_template_override_requested": self._template_override["requested"],
"chat_template_override_reason": self._template_override["reason"],
+ "speculative_type": spec_mode,
+ "spec_draft_n_max": _positive_int(spec_draft_n_max),
+ "spec_draft_model": spec_draft_model or None,
+ "spec_drafter_kind": spec_kind,
+ "spec_fallback_reason": spec_reason,
}
# Capture chat_template_info for the worker IPC reply and route capability classification.
self._populate_chat_template_info(model_name, native_template)
@@ -4966,6 +5137,7 @@ class MLXInferenceBackend:
self._model_fusion.close()
if model_name in self.models:
del self.models[model_name]
+ self._speculative_draft = None
self._model = None
self._tokenizer = None
self._processor = None
@@ -6075,7 +6247,14 @@ class MLXInferenceBackend:
"constrain the next token. Load an autoregressive model to use "
"guided decoding."
)
- if presence_penalty or frequency_penalty or logit_bias or constraint is not None:
+ draft = self._speculative_draft
+ if (
+ presence_penalty
+ or frequency_penalty
+ or logit_bias
+ or constraint is not None
+ or (_rep_active and draft is not None)
+ ):
# These need custom processors: pass the full list (repetition + the rest) instead of the
# repetition_penalty shortcut so all apply.
vlm_kwargs["logits_processors"] = _mlx_sampling_processors(
@@ -6142,6 +6321,24 @@ class MLXInferenceBackend:
else:
vlm_kwargs.update(self._kv_quant_generate_kwargs())
session_scope = session if session is not None else nullcontext()
+ processors = vlm_kwargs.get("logits_processors") or ()
+ # The draft is told these processors, and takes only ones that read just the history (a grammar's
+ # matcher advances per call); tool turns decode as before; a clip is nothing a drafter was built for.
+ if (
+ draft is None
+ or tools
+ or tool_protocol_active
+ or video is not None
+ or not all(getattr(processor, "history_only", False) for processor in processors)
+ or _vlm_generation_is_diffusion(self._model)
+ ):
+ draft = None
+ elif "input_ids" not in vlm_kwargs:
+ vlm_kwargs.update(
+ _vlm_generate_kwargs(
+ _vlm_prepared_inputs(self._model, self._processor, prompt, images)
+ )
+ )
def _stream_vlm_snapshots():
nonlocal stopped
@@ -6168,6 +6365,22 @@ class MLXInferenceBackend:
generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))
final_response = None
clip_path = None
+ if draft is not None:
+ from unsloth_zoo.mlx.generate import SamplingParams
+
+ # Under the lock: the load's one draft serves one generation at a time.
+ draft.prepare(
+ vlm_kwargs["input_ids"][0].tolist(),
+ SamplingParams(
+ temperature = temperature,
+ top_p = top_p,
+ top_k = max(int(top_k or 0), 0),
+ min_p = float(min_p or 0.0),
+ seed = seed,
+ ),
+ processors,
+ )
+ vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)
try:
if video is not None:
clip_path = _write_video_clip(video)
@@ -6264,6 +6477,9 @@ class MLXInferenceBackend:
getattr(final_response, "generation_tokens", 0),
max_new_tokens,
),
+ drafts = None
+ if draft is None
+ else (draft.draft_n, draft.draft_n_accepted),
)
if document_only:
@@ -6305,6 +6521,8 @@ class MLXInferenceBackend:
return "fewer than two replies were requested"
if self._is_vlm:
reason = self._vlm_batch_unavailable_reason(requests)
+ if reason is None and self._speculative_draft is not None:
+ reason = self._vlm_resident_unavailable_reason({})
if reason is not None:
return reason
if any(_mlx_stop_sequences(request.get("stop")) for request in requests):
@@ -6398,12 +6616,18 @@ class MLXInferenceBackend:
self.last_batch_generation_stats = [None] * len(requests)
self.last_generation_stats = None
- run = self._generate_vlm_batch if self._is_vlm else self._generate_text_batch
- yield from run(requests, cancel_event = cancel_event, _adapter_state = _adapter_state)
+ kwargs = {"cancel_event": cancel_event, "_adapter_state": _adapter_state}
+ if not self._is_vlm:
+ yield from self._generate_session_batch(requests, _TextBatchSession, **kwargs)
+ elif self._speculative_draft is not None: # stream_batch has no speculative mode
+ yield from self._generate_session_batch(requests, _VisionBatchSession, **kwargs)
+ else:
+ yield from self._generate_vlm_batch(requests, **kwargs)
- def _generate_text_batch(
+ def _generate_session_batch(
self,
requests,
+ session_type,
*,
cancel_event = None,
_adapter_state = None,
@@ -6415,7 +6639,7 @@ class MLXInferenceBackend:
yield row, snapshot
with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):
- session = _TextBatchSession(
+ session = session_type(
self,
width = len(requests),
adapter_state = _adapter_state,
@@ -6428,7 +6652,7 @@ class MLXInferenceBackend:
if prefix:
yield row, prefix
while session.rows_in_flight:
- yield from report(session.step())
+ yield from report(session.step(cancel_event and cancel_event.is_set))
if cancel_event is not None and cancel_event.is_set():
yield from report(session.withdraw(list(session.handles)))
finally:
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
index 35665079252..5ad29f8938f 100644
--- a/studio/backend/core/inference/mlx_memory.py
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import contextvars
import glob
import functools
import json
@@ -25,6 +26,7 @@ __all__ = [
"MLX_PREFILL_CHUNK",
"MlxMemoryBreakdown",
"mlx_fit_context",
+ "mlx_fit_outcome",
"mlx_memory_breakdown",
"mlx_shard_files",
"mlx_weight_bytes",
@@ -34,6 +36,8 @@ MLX_KV_BLOCK = 256
_PROBE_SHORT = 8
_PROBE_LONG = 40
+# Above mlx-vlm's decode block (8 tokens), inside which some targets evaluate their forward.
+_DRAFTER_PROBE = (16, 48)
MLX_PREFILL_CHUNK = 2048
@@ -295,8 +299,16 @@ def _generation_settings(config: dict) -> tuple:
)
+# The route a sizing prices when the load overrides the config's own: a speculative text load goes through mlx-vlm.
+_ROUTE_VISION: contextvars.ContextVar = contextvars.ContextVar("mlx_route_vision", default = None)
+
+
def _loads_as_vision(config: dict) -> bool:
+ forced = _ROUTE_VISION.get()
+ if forced is not None:
+ return forced
from types import SimpleNamespace
+
try:
from utils.models.model_config import _is_vlm
except Exception:
@@ -586,6 +598,24 @@ def _config_widths(config: dict):
)
+def _entries(cache, quantized = None) -> list:
+ entries = []
+ for index, entry in enumerate(cache):
+ converts = quantized is not None and quantized[index] is not entry
+ for leaf in _sub_caches(entry):
+ entries.append(
+ {
+ "bytes": _entry_bytes(leaf),
+ "quant_bytes": _entry_bytes(leaf),
+ "bound_spec": _bound_spec(leaf),
+ "conv_width": _conv_width(leaf),
+ "block": _block(leaf),
+ "converts": converts,
+ }
+ )
+ return entries
+
+
def _probe(config: dict, dtype, n_tokens: int, kv_bits):
import mlx.core as mx
@@ -609,20 +639,7 @@ def _probe(config: dict, dtype, n_tokens: int, kv_bits):
if cache is None:
raise failure
quantized = _quantize_like_runtime(cache, kv_bits)
- entries = []
- for index, entry in enumerate(cache):
- converts = quantized is not None and quantized[index] is not entry
- for leaf in _sub_caches(entry):
- entries.append(
- {
- "bytes": _entry_bytes(leaf),
- "quant_bytes": _entry_bytes(leaf),
- "bound_spec": _bound_spec(leaf),
- "conv_width": _conv_width(leaf),
- "block": _block(leaf),
- "converts": converts,
- }
- )
+ entries = _entries(cache, quantized)
if quantized is not None:
converted = [leaf for entry in quantized for leaf in _sub_caches(entry)]
if len(converted) == len(entries):
@@ -643,30 +660,7 @@ def _cache_plan(config: dict, dtype, kv_bits):
far = _probe(config, dtype, _PROBE_LONG, kv_bits)
if not near["entries"] or len(near["entries"]) != len(far["entries"]):
raise ValueError("cache probe returned no comparable entries")
- span = _PROBE_LONG - _PROBE_SHORT
-
- def solve(key):
- return [
- (n[key] - (f[key] - n[key]) / span * _PROBE_SHORT, (f[key] - n[key]) / span)
- for n, f in zip(near["entries"], far["entries"])
- ]
-
- plan = []
- for slot, (const, slope), (q_const, q_slope) in zip(
- near["entries"], solve("bytes"), solve("quant_bytes")
- ):
- plan.append(
- {
- "const": const,
- "slope": slope,
- "quant_const": q_const,
- "quant_slope": q_slope,
- "bound_spec": slot["bound_spec"],
- "conv_width": slot["conv_width"],
- "block": slot["block"],
- "converts": slot["converts"] and near["quantized"],
- }
- )
+ plan = _solved(near["entries"], far["entries"], (_PROBE_SHORT, _PROBE_LONG), near["quantized"])
return (
plan,
0 if near["quantized"] else None,
@@ -678,6 +672,58 @@ def _cache_plan(config: dict, dtype, kv_bits):
)
+def _solved(near: list, far: list, lengths: tuple, quantized: bool) -> list:
+ """Per-entry ``const + slope * T`` from the entries of caches filled to two lengths."""
+ short, span = lengths[0], lengths[1] - lengths[0]
+
+ def solve(key):
+ return [
+ (n[key] - (f[key] - n[key]) / span * short, (f[key] - n[key]) / span)
+ for n, f in zip(near, far)
+ ]
+
+ return [
+ {
+ "const": const,
+ "slope": slope,
+ "quant_const": q_const,
+ "quant_slope": q_slope,
+ "bound_spec": slot["bound_spec"],
+ "conv_width": slot["conv_width"],
+ "block": slot["block"],
+ "converts": slot["converts"] and quantized,
+ }
+ for slot, (const, slope), (q_const, q_slope) in zip(
+ near, solve("bytes"), solve("quant_bytes")
+ )
+ ]
+
+
+def _drafter_sizing(config: dict, dtype, drafter: tuple) -> tuple:
+ """``(weight bytes, cache plan, capture bytes per prompt token)`` of the drafter ``(path, builtin)``
+ attaches, from a weightless target and a lazily built drafter."""
+ from unsloth_zoo.mlx.speculative import companion_drafter, native_mtp_drafter, probe_drafter
+
+ path, builtin = drafter
+ if not builtin and (_snapshot_config(path) or {}).get("model_file"):
+ raise ValueError("this drafter carries its own model module")
+ target = _whole_model(config, dtype)
+ built = (native_mtp_drafter if builtin else companion_drafter)(path, target, lazy = True)
+ if built is None:
+ raise ValueError("no MTP splitter knows this checkpoint")
+ near, far = (_entries(probe_drafter(target, built, length)) for length in _DRAFTER_PROBE)
+ if len(near) != len(far):
+ raise ValueError("drafter cache probe returned no comparable entries")
+ # A dflash or eagle3 prefill keeps every chunk's captured layers, then a concatenated copy.
+ captured = len(getattr(built, "capture", {}).get("capture_layer_ids", ()))
+ hidden = _config_width((config.get("text_config") or config).get("hidden_size"))
+ return (
+ built.weight_bytes,
+ _solved(near, far, _DRAFTER_PROBE, False),
+ 2 * captured * hidden * dtype.size,
+ )
+
+
@functools.lru_cache(maxsize = 256)
def _bounded_peak(cache_type, attribute: str, value: int, n_ctx: int, prefill_chunk: int) -> int:
"""Largest slot count a bounded cache reaches, by driving the real class."""
@@ -981,6 +1027,9 @@ class _MlxSizing:
chunk: int
kv_bits: Optional[int]
kv_group_size: int
+ drafter_weights: int = 0
+ drafter_plan: tuple = ()
+ capture_per_token: int = 0
_SIZING_CACHE_MAX = 16
@@ -1003,7 +1052,12 @@ def _checkpoint_fingerprint(model_dir: str) -> tuple:
def _size_load(
- model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool
+ model_dir: str,
+ kv_bits: Optional[int],
+ prefill_chunk: Optional[int],
+ load_in_4bit: bool,
+ vision: Optional[bool] = None,
+ drafter: Optional[tuple] = None,
) -> Optional[_MlxSizing]:
"""Cached ``_size_load_uncached``: every estimate prices the fit and the breakdown, and every
slider step prices again, each otherwise rebuilding the same weightless architecture."""
@@ -1015,14 +1069,17 @@ def _size_load(
bool(load_in_4bit),
os.environ.get("UNSLOTH_MLX_PROMPT_CACHE_BYTES"),
_checkpoint_fingerprint(model_dir),
+ vision,
+ drafter,
+ drafter and _checkpoint_fingerprint(drafter[0]),
)
except Exception:
- return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
with _sizing_lock:
if key in _sizing_cache:
_sizing_cache.move_to_end(key)
return _sizing_cache[key]
- sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
with _sizing_lock:
_sizing_cache[key] = sizing
while len(_sizing_cache) > _SIZING_CACHE_MAX:
@@ -1036,7 +1093,26 @@ def _clear_sizing_cache() -> None:
def _size_load_uncached(
- model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool
+ model_dir: str,
+ kv_bits: Optional[int],
+ prefill_chunk: Optional[int],
+ load_in_4bit: bool,
+ vision: Optional[bool] = None,
+ drafter: Optional[tuple] = None,
+) -> Optional[_MlxSizing]:
+ token = _ROUTE_VISION.set(vision)
+ try:
+ return _sized(model_dir, kv_bits, prefill_chunk, load_in_4bit, drafter)
+ finally:
+ _ROUTE_VISION.reset(token)
+
+
+def _sized(
+ model_dir: str,
+ kv_bits: Optional[int],
+ prefill_chunk: Optional[int],
+ load_in_4bit: bool,
+ drafter: Optional[tuple],
) -> Optional[_MlxSizing]:
"""Everything about a load that does not move with the context, or None if it cannot be sized."""
config = _snapshot_config(model_dir)
@@ -1062,6 +1138,7 @@ def _size_load_uncached(
for tower, checkpoint in zip(facts["widths"], _config_widths(config))
)
chunk = prefill_chunk or loaded_chunk
+ drafted = (0, (), 0) if drafter is None else _drafter_sizing(config, dtype, drafter)
except Exception as exc:
logger.debug(
"MLX estimate could not size %s (%s): %s", model_dir, config.get("model_type"), exc
@@ -1077,6 +1154,9 @@ def _size_load_uncached(
chunk = chunk,
kv_bits = kv_bits,
kv_group_size = loaded_group,
+ drafter_weights = drafted[0],
+ drafter_plan = tuple(drafted[1]),
+ capture_per_token = drafted[2],
)
@@ -1096,14 +1176,16 @@ def _priced_at(sizing: _MlxSizing, n_ctx: int) -> Optional[MlxMemoryBreakdown]:
sizing.kv_bits,
sizing.kv_group_size,
)
+ drafter_kv = _line_bytes(list(sizing.drafter_plan), context, chunk, quantized = False)
except Exception as exc:
logger.debug("MLX estimate could not price %s tokens: %s", n_ctx, exc)
return None
- total = sizing.weights + kv + compute
+ capture = sizing.capture_per_token * context
+ total = sizing.weights + sizing.drafter_weights + kv + drafter_kv + compute + capture
return MlxMemoryBreakdown(
- weights_bytes = sizing.weights,
- kv_bytes = kv,
- compute_bytes = compute,
+ weights_bytes = sizing.weights + sizing.drafter_weights,
+ kv_bytes = kv + drafter_kv,
+ compute_bytes = compute + capture,
total_bytes = total,
gpu_bytes = total,
n_ctx = context,
@@ -1126,13 +1208,16 @@ def mlx_memory_breakdown(
kv_bits: Optional[int] = None,
prefill_chunk: Optional[int] = None,
load_in_4bit: bool = False,
+ vision: Optional[bool] = None,
+ drafter: Optional[tuple] = None,
) -> Optional[MlxMemoryBreakdown]:
- """Price an MLX load, or None when it cannot honestly be sized."""
- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ """Price an MLX load, or None when it cannot honestly be sized. ``vision`` overrides the loader route
+ the config implies; ``drafter`` is the ``(path, builtin)`` drafter a speculative load attaches."""
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
return None if sizing is None else _priced_at(sizing, n_ctx)
-def mlx_fit_context(
+def mlx_fit_outcome(
model_dir: str,
*,
budget_bytes: int,
@@ -1141,25 +1226,36 @@ def mlx_fit_context(
kv_bits: Optional[int] = None,
prefill_chunk: Optional[int] = None,
load_in_4bit: bool = False,
-) -> Optional[int]:
- """Largest context whose estimated footprint stays inside ``budget_bytes``."""
- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ vision: Optional[bool] = None,
+ drafter: Optional[tuple] = None,
+) -> tuple:
+ """``("unsizable" | "fits" | "fitted" | "no_fit", context)``: the largest context inside
+ ``budget_bytes`` for ``"fitted"``, else None."""
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
if sizing is None:
- return None
+ return "unsizable", None
ceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)
at_ceiling = _priced_at(sizing, ceiling)
- if at_ceiling is None or at_ceiling.total_bytes <= budget_bytes:
- return None
+ if at_ceiling is None:
+ return "unsizable", None
+ if at_ceiling.total_bytes <= budget_bytes:
+ return "fits", None
floor = max(-(-int(min_ctx or 0) // MLX_KV_BLOCK) * MLX_KV_BLOCK, MLX_KV_BLOCK)
low, high, best = floor, ceiling, None
while low <= high:
middle = (low + high) // 2
priced = _priced_at(sizing, middle)
if priced is None:
- return None
+ return "unsizable", None
if priced.total_bytes <= budget_bytes:
best = middle
low = middle + 1
else:
high = middle - 1
- return None if best is None else (best // MLX_KV_BLOCK) * MLX_KV_BLOCK
+ return ("no_fit", None) if best is None else ("fitted", (best // MLX_KV_BLOCK) * MLX_KV_BLOCK)
+
+
+def mlx_fit_context(model_dir: str, **kwargs) -> Optional[int]:
+ """Largest context whose estimated footprint stays inside ``budget_bytes``; None when the ceiling
+ already fits or nothing can be said."""
+ return mlx_fit_outcome(model_dir, **kwargs)[1]
diff --git a/studio/backend/core/inference/mlx_speculative.py b/studio/backend/core/inference/mlx_speculative.py
new file mode 100644
index 00000000000..b8318eb4e52
--- /dev/null
+++ b/studio/backend/core/inference/mlx_speculative.py
@@ -0,0 +1,400 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Which drafter an MLX load speculates with: ``speculative_type`` and ``spec_draft_model`` resolved
+against the target's own MTP head and the companion drafters already in the local Hugging Face cache."""
+
+from __future__ import annotations
+
+import inspect
+import json
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+MLX_DRAFTER_KINDS = ("mtp", "dflash", "dspark", "eagle3")
+MLX_SPEC_MODES = frozenset({"auto", "off", "ngram", *MLX_DRAFTER_KINDS})
+_LEGACY_MODES = {
+ "mtp+ngram": "mtp",
+ "default": "auto",
+ "draft-mtp": "mtp",
+ "draft-dspark": "dspark",
+ "draft-dflash": "dflash",
+ "ngram-mod": "ngram",
+ "ngram-simple": "ngram",
+ "none": "off",
+ "disable": "off",
+ "disabled": "off",
+}
+_COMPANION_ORDER = ("mtp", "dflash2", "dflash", "dspark", "eagle3")
+
+DRAFTER_NOT_FOUND = "drafter_not_found"
+DRAFTER_INCOMPATIBLE = "drafter_incompatible"
+DRAFTER_NO_MEMORY = "drafter_no_memory"
+RUNTIME_ERROR = "runtime_error"
+KV_QUANT = "kv_quant"
+AUTO_CONTEXT_COST = "auto_context_cost"
+
+
+def mlx_spec_mode(value) -> str:
+ """``speculative_type`` as an MLX load reads it: absent or unknown is ``auto``, as on llama.cpp."""
+ if not isinstance(value, str) or not value.strip():
+ return "auto"
+ mode = value.strip().lower()
+ mode = _LEGACY_MODES.get(mode, mode)
+ return mode if mode in MLX_SPEC_MODES else "auto"
+
+
+@dataclass(frozen = True)
+class DrafterSource:
+ kind: str
+ path: str
+ builtin: bool
+
+
+@dataclass(frozen = True)
+class SpecResolution:
+ mode: str
+ sources: tuple = ()
+ copies: bool = False # n-gram copies alongside a drafter, or alone when an explicit kind's drafter does not attach
+ reason: Optional[str] = None
+
+ @property
+ def speculative(self) -> bool:
+ return bool(self.sources) or self.copies
+
+
+def _read_config(path) -> dict:
+ try:
+ with open(os.path.join(path, "config.json"), encoding = "utf-8") as handle:
+ config = json.load(handle)
+ return config if isinstance(config, dict) else {}
+ except (OSError, ValueError):
+ return {}
+
+
+def companion_kind(config: dict) -> Optional[str]:
+ """``dflash2``, ``dflash``, ``dspark``, ``eagle3`` or ``mtp`` (a separate MTP head or a Gemma assistant) for a companion drafter config."""
+ # mlx-vlm executes a checkpoint's own model_file on load; a drafter never gets the target's remote-code grant.
+ if config.get("model_file"):
+ return None
+ architectures = " ".join(str(name) for name in config.get("architectures") or ())
+ # mlx-vlm loads EAGLE-3 only in the speculators format, not e.g. llama-typed Eagle3 exports.
+ if "eagle3" in (config.get("model_type"), config.get("speculators_model_type")):
+ return "eagle3"
+ model_type = str(config.get("model_type", ""))
+ if model_type.endswith("_mtp") or (
+ model_type.endswith("_assistant") and "Assistant" in architectures
+ ):
+ return "mtp"
+ if "dflash_config" in config:
+ return (
+ "dspark"
+ if "DSpark" in architectures
+ else "dflash2"
+ if "DFlash2" in architectures
+ else "dflash"
+ )
+ return None
+
+
+def _reported(kind: str) -> str:
+ return "dflash" if kind == "dflash2" else kind
+
+
+_DRAFTER_WORDS = frozenset(
+ {"dflash", "dflash2", "dspark", "eagle3", "speculator", "assistant", "mtp", "drafter"}
+)
+_WEIGHT_WORDS = re.compile(r"^(mlx|bf16|fp16|fp8|mxfp4|nvfp4|qat|unquantized|\d+bit|dwq)$")
+_GGUF_QUANT = re.compile(r"(?<![a-z0-9])q\d(_[a-z0-9]+)+(?![a-z0-9])")
+
+
+def _alnum(name: str) -> str:
+ return re.sub(r"[^a-z0-9]", "", os.path.basename(str(name).rstrip("/")).lower())
+
+
+def _stem(name: str) -> str:
+ """A drafter repo's name without its organization, drafter and weight-format words, as alphanumerics."""
+ base = _GGUF_QUANT.sub("", os.path.basename(str(name).rstrip("/")).lower())
+ words = re.split(r"[-_]", base.replace("speculator.eagle3", ""))
+ return _alnum(
+ "".join(w for w in words if w not in _DRAFTER_WORDS and not _WEIGHT_WORDS.match(w))
+ )
+
+
+def _text(config: dict) -> dict:
+ scope = config.get("text_config")
+ return scope if isinstance(scope, dict) else config
+
+
+def _vocab_size(config: dict) -> Optional[int]:
+ for scope in (
+ config,
+ config.get("text_config") or {},
+ config.get("transformer_layer_config") or {},
+ ):
+ if isinstance(scope, dict) and isinstance(scope.get("vocab_size"), int):
+ return scope["vocab_size"]
+ return None
+
+
+def _fits(kind: str, config: dict, target: dict) -> bool:
+ """Whether a drafter's config is built for a target of this shape. A dimension either side does not state cannot refuse."""
+ text, own = _text(target), _text(config)
+ layers, flash = text.get("num_hidden_layers"), config.get("dflash_config") or {}
+ if kind == "eagle3":
+ wants = config.get("target_hidden_size") or (
+ config.get("transformer_layer_config") or {}
+ ).get("hidden_size")
+ taps = config.get("eagle_aux_hidden_state_layer_ids") or flash.get("target_layer_ids")
+ elif kind == "mtp":
+ wants = config.get("backbone_hidden_size") or own.get("hidden_size")
+ taps = None
+ if "backbone_hidden_size" not in config:
+ pairs = [
+ (own.get("num_hidden_layers"), layers),
+ (own.get("model_type"), text.get("model_type")),
+ ]
+ if any(a is not None and b is not None and a != b for a, b in pairs):
+ return False
+ else:
+ wants = config.get("target_hidden_size") or config.get("hidden_size")
+ taps = flash.get("target_layer_ids") or config.get("target_layer_ids")
+ stated = config.get("num_target_layers") or flash.get("num_target_layers")
+ if stated is not None and layers is not None and stated != layers:
+ return False
+ vocab = _vocab_size(target)
+ return not (
+ (wants is not None and text.get("hidden_size") not in (None, wants))
+ # Families number tapped layers from 0 or from 1; only an id past both is out of range.
+ or (taps and layers is not None and max(taps) > layers)
+ or (vocab is not None and _vocab_size(config) not in (None, vocab))
+ )
+
+
+def _has_weights(snapshot) -> bool:
+ try:
+ return any(name.endswith(".safetensors") for name in os.listdir(snapshot))
+ except OSError:
+ return False
+
+
+def _cached_repos():
+ from utils.utils import _hf_cache_roots
+ seen = set()
+ for root in _hf_cache_roots():
+ try:
+ entries = sorted(os.listdir(root))
+ except OSError:
+ continue
+ for entry in entries:
+ if entry.startswith("models--") and entry not in seen:
+ seen.add(entry)
+ yield entry[len("models--") :].replace("--", "/")
+
+
+def cached_drafters(target_name: str, target_config: dict) -> list:
+ """``(repo, source, named)`` for each cached drafter with weights whose config fits the target's shape.
+
+ ``named`` drafters carry the target's model name and come first, in auto's order; the rest fit
+ by shape alone (another generation or fine-tune of the same architecture) and follow.
+ """
+ from utils.utils import hf_cache_snapshot_dir_for_repo
+
+ target, found = _alnum(target_name), []
+ for repo in _cached_repos():
+ if repo.lower() == str(target_name).lower():
+ continue
+ snapshot = hf_cache_snapshot_dir_for_repo(repo)
+ config = _read_config(snapshot) if snapshot is not None else {}
+ kind = companion_kind(config)
+ if kind is None or not _fits(kind, config, target_config) or not _has_weights(snapshot):
+ continue
+ stem = _stem(repo)
+ named = bool(stem) and target.startswith(stem)
+ source = DrafterSource(_reported(kind), str(snapshot), False)
+ found.append((not named, _COMPANION_ORDER.index(kind), repo, source))
+ return [(repo, source, not unnamed) for unnamed, _, repo, source in sorted(found)]
+
+
+def discover_companions(
+ target_name: str,
+ target_config: dict,
+ allowed = None,
+) -> list:
+ """The cached drafters auto attaches: those named for the target, within ``allowed`` repos when given."""
+ found = cached_drafters(target_name, target_config)
+ return [
+ source for repo, source, named in found if named and (allowed is None or repo in allowed)
+ ]
+
+
+def _named_companion(spec_draft_model: str) -> Optional[DrafterSource]:
+ from utils.utils import hf_cache_snapshot_dir_for_repo
+
+ path = Path(os.path.expanduser(spec_draft_model))
+ snapshot = path if path.is_dir() else hf_cache_snapshot_dir_for_repo(spec_draft_model)
+ kind = companion_kind(_read_config(snapshot)) if snapshot is not None else None
+ return None if kind is None else DrafterSource(_reported(kind), str(snapshot), False)
+
+
+def has_builtin_head(model_dir: Optional[str]) -> bool:
+ if not model_dir:
+ return False
+ try:
+ from mlx_vlm.speculative.drafters.mtp_split import detect_mtp_splitter
+ return detect_mtp_splitter(Path(model_dir)) is not None
+ except Exception as exc:
+ logger.debug("MLX speculative: no MTP splitter check for %s: %s", model_dir, exc)
+ return False
+
+
+def resolve_speculation(
+ speculative_type,
+ spec_draft_model: Optional[str],
+ *,
+ model_dir: Optional[str],
+ target_name: str,
+ allowed = None,
+) -> SpecResolution:
+ """The drafters an MLX load tries: a named ``spec_draft_model``, then cached companions (any kind under auto) and the built-in head, in ``_COMPANION_ORDER``."""
+ mode = mlx_spec_mode(speculative_type)
+ if mode == "off":
+ return SpecResolution(mode)
+ if mode == "ngram":
+ return SpecResolution(mode, copies = True)
+ auto = mode == "auto"
+ kind = "mtp" if auto else mode
+ sources, reason = [], None
+ if spec_draft_model:
+ named = _named_companion(spec_draft_model)
+ if named is None:
+ reason = DRAFTER_NOT_FOUND
+ elif named.kind != kind and not auto:
+ reason = DRAFTER_INCOMPATIBLE
+ else:
+ sources.append(named)
+ companions = discover_companions(
+ target_name, _read_config(model_dir) if model_dir else {}, allowed
+ )
+ companions = [
+ source for source in companions if (auto or source.kind == kind) and source not in sources
+ ]
+ if kind == "mtp" and has_builtin_head(model_dir):
+ companions.insert(0, DrafterSource("mtp", str(model_dir), True))
+ sources += companions
+ if not sources and auto:
+ return SpecResolution(mode, reason = reason)
+ if not sources and reason is None:
+ reason = DRAFTER_NOT_FOUND
+ return SpecResolution(mode, tuple(sources), copies = True, reason = reason)
+
+
+def speculates_on_route(
+ mode: str,
+ vision: bool,
+ named = None,
+) -> bool:
+ """Whether a load in ``mode`` looks for a drafter: auto only where mlx-vlm serves it anyway, or for a drafter the request names."""
+ return mode != "off" and (mode != "auto" or vision or bool(named))
+
+
+_MAX_DEPTH = (
+ 8 # a drafter without its own block size; the controller picks each round's depth below it
+)
+_MAX_COPY = 16
+_EXACT_HEAD_DEPTH = 3 # MTP heads and assistants chain a forward per drafted token
+
+
+def speculation_refusal(*, kv_quant: bool, distributed: bool, lora: bool) -> Optional[str]:
+ """The reason an MLX load cannot speculate at all, or None."""
+ if kv_quant:
+ return KV_QUANT
+ if distributed or lora:
+ return RUNTIME_ERROR
+ try:
+ from unsloth_zoo.mlx.speculative import speculative_unavailable_reason
+ except ImportError as exc:
+ logger.info("MLX speculative decoding unavailable: %s", exc)
+ return RUNTIME_ERROR
+ unavailable = speculative_unavailable_reason()
+ if unavailable is not None:
+ logger.info("MLX speculative decoding unavailable: %s", unavailable)
+ return RUNTIME_ERROR
+ return None
+
+
+def _draft(drafter, copies: bool, draft_n_max: Optional[int]):
+ from unsloth_zoo.mlx.speculative import (
+ DraftController,
+ install_speculative_seam,
+ SpeculativeDraft,
+ )
+
+ depth = 0 if drafter is None else int(getattr(drafter, "max_depth", _MAX_DEPTH))
+ copy, exact = _MAX_COPY, False
+ if draft_n_max:
+ exact = draft_n_max <= getattr(drafter, "max_depth", _EXACT_HEAD_DEPTH)
+ depth, copy = min(depth, draft_n_max), min(copy, draft_n_max)
+ install_speculative_seam()
+ return SpeculativeDraft(
+ DraftController(max_depth = depth, max_copy = copy, can_copy = copies, fixed_depth = exact), drafter
+ )
+
+
+def _carries_encoder_state(target) -> bool:
+ # generate_step threads these from forward to forward (Mllama, Florence2, Nemotron Parse); the
+ # speculative rounds refuse them only after prefill, too late to decode the reply unspeculated.
+ for module in (target, getattr(target, "language_model", None)):
+ try:
+ parameters = inspect.signature(module).parameters
+ except (TypeError, ValueError):
+ continue
+ if "cross_attention_states" in parameters or "encoder_outputs" in parameters:
+ return True
+ return False
+
+
+def build_draft(
+ target,
+ resolution: SpecResolution,
+ *,
+ fits,
+ draft_n_max: Optional[int] = None,
+) -> tuple:
+ """``(draft, kind, reason, context)`` for the first source ``fits`` accepts that builds, else copies
+ alone when the mode allows. ``reason`` names a passed-over choice, so a substitute never attaches silently."""
+ from unsloth_zoo.mlx.speculative import companion_drafter, native_mtp_drafter
+
+ if _carries_encoder_state(target):
+ logger.info("MLX speculative decoding: %s carries encoder state", type(target).__name__)
+ return None, None, None if resolution.mode == "auto" else RUNTIME_ERROR, None
+ reason = resolution.reason
+ for source in resolution.sources:
+ ok, context = fits(source)
+ if not ok:
+ if resolution.mode != "auto":
+ reason = DRAFTER_NO_MEMORY
+ elif ok is not None: # priced, and the drafter would shrink it
+ reason = AUTO_CONTEXT_COST
+ continue
+ try:
+ drafter = (native_mtp_drafter if source.builtin else companion_drafter)(
+ source.path, target
+ )
+ if drafter is None:
+ raise ValueError("no MTP splitter knows this checkpoint")
+ except Exception as exc:
+ logger.warning("MLX drafter %s (%s) not attached: %s", source.path, source.kind, exc)
+ reason = DRAFTER_INCOMPATIBLE
+ continue
+ return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, context
+ if resolution.copies and resolution.mode != "auto": # auto copies only alongside a drafter
+ return _draft(None, True, draft_n_max), "ngram", reason, None
+ return None, None, reason, None
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 72b6fdab7fb..894ec5f3777 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -257,6 +257,11 @@ _MLX_RUNTIME_MIRROR_FIELDS = (
"mlx_int8_prefill_reason",
"chat_template_override_requested",
"chat_template_override_reason",
+ "speculative_type",
+ "spec_draft_n_max",
+ "spec_draft_model",
+ "spec_drafter_kind",
+ "spec_fallback_reason",
)
@@ -2006,6 +2011,10 @@ class InferenceOrchestrator:
mlx_distributed: bool = False,
mlx_kv_quant: Optional[str] = None,
mlx_int8_prefill: bool = False,
+ speculative_type: Optional[str] = None,
+ spec_draft_n_max: Optional[int] = None,
+ spec_draft_model: Optional[str] = None,
+ spec_drafters_allowed: Optional[list] = None,
chat_template_override: Optional[str] = None,
load_cancel_event: Optional[threading.Event] = None,
post_handoff_expected_free_gb: Optional[dict[int, float]] = None,
@@ -2079,6 +2088,10 @@ class InferenceOrchestrator:
else None,
"mlx_kv_quant": mlx_kv_quant,
"mlx_int8_prefill": bool(mlx_int8_prefill),
+ "speculative_type": speculative_type,
+ "spec_draft_n_max": spec_draft_n_max,
+ "spec_draft_model": spec_draft_model,
+ "spec_drafters_allowed": spec_drafters_allowed,
"chat_template_override": chat_template_override,
# Read in the worker, which hides the accelerators before detection.
"audio_device": audio_device,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 39fb949b4dc..8e6e0f54dcf 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -741,6 +741,10 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")
load_kwargs["kv_quant"] = config.get("mlx_kv_quant")
load_kwargs["int8_prefill"] = bool(config.get("mlx_int8_prefill"))
+ load_kwargs["speculative_type"] = config.get("speculative_type")
+ load_kwargs["spec_draft_n_max"] = config.get("spec_draft_n_max")
+ load_kwargs["spec_draft_model"] = config.get("spec_draft_model")
+ load_kwargs["spec_drafters_allowed"] = config.get("spec_drafters_allowed")
load_kwargs["chat_template_override"] = config.get("chat_template_override")
success = backend.load_model(**load_kwargs)
finally:
@@ -829,6 +833,11 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"mlx_int8_prefill_reason",
"chat_template_override_requested",
"chat_template_override_reason",
+ "speculative_type",
+ "spec_draft_n_max",
+ "spec_draft_model",
+ "spec_drafter_kind",
+ "spec_fallback_reason",
)
if k in _entry
}
@@ -953,6 +962,9 @@ class _Stops:
def __contains__(self, request_id) -> bool:
return request_id in self._stopped
+ def unread(self) -> bool:
+ return self._ledger is not None and self._ledger.snapshot(self._written)[1] is not None
+
class _StopWhileItRuns:
def __init__(self, cancel_event, stops, request_id: str):
@@ -1474,11 +1486,11 @@ class _ResidentBatch:
if request_id in stopped:
self.cancel(request_id)
- def step(self) -> None:
+ def step(self, waiting = None) -> None:
if self.session is None or not self.session.rows_in_flight:
return
try:
- for handle, snapshot in self.session.step():
+ for handle, snapshot in self.session.step(waiting):
self._report(handle, snapshot)
except Exception as exc:
logger.error("Batched generation error: %s", exc, exc_info = True)
@@ -2065,6 +2077,7 @@ def run_inference_process(
warmth = _MLXIdleWarmth()
deferred: list[dict] = []
stops = _Stops(stop_ledger, resp_queue, batch, deferred)
+ waiting = lambda: stops.unread() or not cmd_queue.empty()
if stop_ledger is not None:
stop_ledger.worker_reads_this()
while True:
@@ -2073,7 +2086,7 @@ def run_inference_process(
if not tearing_down:
if batch.rows_in_flight:
warmth.active()
- batch.step()
+ batch.step(waiting)
from_deferred = False
if _held_head_leaves_the_hold(batch, deferred):
cmd = deferred.pop(0)
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index b7c8c32b076..70f2ef69016 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -215,7 +215,11 @@ class LoadRequest(BaseModel):
"Legacy values 'default' (-> auto), 'draft-mtp' (-> mtp), "
"'draft-dspark' (-> dspark), 'draft-dflash' (-> dflash), "
"'ngram-mod' (-> ngram), and 'ngram-simple' (kept as-is) are "
- "still accepted. Ignored for non-GGUF models."
+ "still accepted. MLX models read 'auto', 'mtp', 'dflash', 'dspark', 'eagle3', "
+ "'ngram' and 'off'; every drafter kind also copies repeated text (n-gram), "
+ "so 'mtp+ngram' reads as 'mtp'. On MLX, 'auto' attaches the first "
+ "cached drafter (an MTP head or assistant, then DFlash2, DFlash, DSpark, EAGLE-3) that "
+ "costs no context, on loads served through mlx-vlm. Ignored for other non-GGUF models."
),
)
spec_draft_n_max: Optional[int] = Field(
@@ -228,7 +232,21 @@ class LoadRequest(BaseModel):
"CPU/Mac when unset (upstream-bench sweet spot for dense Qwen3.6 "
"MTP quants, and the measured sweet spot for DFlash too). Only "
"applied when speculative_type resolves to 'mtp', 'mtp+ngram', "
- "'dspark' or 'dflash'."
+ "'dspark' or 'dflash'. On MLX, up to the drafter's trained depth (3 for "
+ "MTP heads and assistants) every step drafts exactly this many (or takes an "
+ "n-gram copy expected to yield more), even when "
+ "plain decoding would be faster; above it, or for n-gram copies alone, it "
+ "is a ceiling and the controller decides when to draft."
+ ),
+ )
+ spec_draft_model: Optional[str] = Field(
+ None,
+ max_length = 1024,
+ description = (
+ "MLX only: a companion drafter (DFlash, DFlash2, DSpark, EAGLE-3 or Gemma "
+ "assistant) to speculate with, as a local directory or an already-cached "
+ "repo id; never downloaded. Tried before the model's own MTP head and the "
+ "cached companions discovery would pick."
),
)
n_parallel: Optional[int] = Field(
@@ -994,6 +1012,11 @@ class EstimateMemoryRequest(BaseModel):
speculative_type: Optional[str] = Field(
None, description = "Speculative mode; decides which drafter's weights are charged."
)
+ spec_draft_model: Optional[str] = Field(
+ None,
+ max_length = 1024,
+ description = "MLX companion drafter intended for the follow-up load, as on /load.",
+ )
spec_draft_n_max: Optional[int] = Field(
None,
ge = 1,
@@ -1084,6 +1107,28 @@ class Int8PrefillAvailabilityResponse(BaseModel):
)
+class MlxDraftersRequest(BaseModel):
+ model_path: str = Field(..., description = "Model identifier or local path of the target")
+
+ _resolve_the_handle = field_validator("model_path")(resolve_inventory_handle)
+
+
+class MlxDrafter(BaseModel):
+ repo_id: str = Field(..., description = "Cached Hugging Face repo, usable as spec_draft_model")
+ kind: str = Field(..., description = "'mtp', 'dflash', 'dspark' or 'eagle3'")
+ named: bool = Field(
+ True,
+ description = "Whether the repo is named for this model. False: it fits the model's "
+ "architecture only (another generation or a fine-tune), so Auto does not use it.",
+ )
+
+
+class MlxDraftersResponse(BaseModel):
+ """Cached drafters an MLX load of the target could name: those named for it first, in Auto's order."""
+
+ drafters: list[MlxDrafter] = Field(default_factory = list)
+
+
class EstimateMemoryResponse(BaseModel):
"""Itemized memory an inference load would occupy, or why it could not be sized."""
@@ -1665,6 +1710,55 @@ class _InferenceRuntimeFields(BaseModel):
"None when the platform default is in effect."
),
)
+ spec_draft_model: Optional[str] = Field(
+ None,
+ description = "The companion drafter an MLX load was asked for; None elsewhere.",
+ )
+ spec_drafter_kind: Optional[str] = Field(
+ None,
+ description = (
+ "Which drafter the resolution was about: 'mtp', 'dspark' or "
+ "'dflash' ('eagle3' or 'ngram' too on MLX, where 'mtp' also names a Gemma "
+ "assistant). Needed "
+ "because Auto resolves the kind itself, so speculative_type still "
+ "reads 'auto', and a fallback leaves the engaged type at 'default': "
+ "neither still says which file the UI should tell the user to fix."
+ ),
+ )
+ spec_fallback_reason: Optional[str] = Field(
+ None,
+ description = (
+ "Why a speculative drafter was disabled despite being requested. "
+ "'binary_no_mtp' / 'binary_outdated' -> a newer prebuilt would "
+ "re-enable it (show the update affordance); 'runtime_error' -> the "
+ "current build could not run it; 'drafter_not_found' -> the model's "
+ "separate MTP or DSpark drafter could not be resolved; "
+ "'drafter_no_vram' -> an Auto-mode fit downgrade: the model pins on "
+ "GPU but the drafter's reserve does not, and Auto keeps the context "
+ "rather than shrink it; select the drafter in Settings to force it. "
+ "'mla_mtp_disabled' -> "
+ "an Auto-mode policy downgrade: the model is MLA (GLM-5.2 et al.) "
+ "whose llama.cpp MTP path runs slower than no speculation, so Auto "
+ "used ngram-mod or spec-off instead -- updating won't help; choose "
+ "MTP in Settings (or set UNSLOTH_MLA_MTP_ENABLED=1) to force it. "
+ "'mtp_partial_offload' -> an Auto-mode policy downgrade: the model "
+ "has an embedded Hybrid Mamba MTP head and the placement offloads "
+ "only part of it, where the recurrent rollback copies cost more "
+ "layers than the drafting wins back -- updating won't help; choose "
+ "MTP in Settings to force it. "
+ "MLX loads report 'drafter_not_found' (no drafter of the requested kind "
+ "is cached), 'drafter_incompatible' (the named or found drafter does not "
+ "fit this model), 'drafter_no_memory' (its weights and cache leave no "
+ "context that fits), 'kv_quant' (KV cache quantization is on) or "
+ "'runtime_error' (the installed MLX packages cannot speculate, or cannot "
+ "load this model the way speculation needs), also when a later drafter "
+ "or n-gram copies stood in for the one asked for. Under auto an MLX load "
+ "reports no refusal; it reports 'auto_context_cost' (a found drafter would "
+ "shrink the fitted context) or the drafter codes above for a named or "
+ "unbuildable drafter. "
+ "None when the requested strategy engaged or was not requested."
+ ),
+ )
tensor_parallel: bool = Field(
False,
description = "Whether tensor-parallel split (--split-mode tensor) is active.",
@@ -2045,40 +2139,6 @@ class InferenceStatusResponse(_InferenceRuntimeFields):
"False -> recommend `unsloth studio update`."
),
)
- spec_drafter_kind: Optional[str] = Field(
- None,
- description = (
- "Which drafter the resolution was about: 'mtp', 'dspark' or "
- "'dflash'. Needed "
- "because Auto resolves the kind itself, so speculative_type still "
- "reads 'auto', and a fallback leaves the engaged type at 'default': "
- "neither still says which file the UI should tell the user to fix."
- ),
- )
- spec_fallback_reason: Optional[str] = Field(
- None,
- description = (
- "Why a speculative drafter was disabled despite being requested. "
- "'binary_no_mtp' / 'binary_outdated' -> a newer prebuilt would "
- "re-enable it (show the update affordance); 'runtime_error' -> the "
- "current build could not run it; 'drafter_not_found' -> the model's "
- "separate MTP or DSpark drafter could not be resolved; "
- "'drafter_no_vram' -> an Auto-mode fit downgrade: the model pins on "
- "GPU but the drafter's reserve does not, and Auto keeps the context "
- "rather than shrink it; select the drafter in Settings to force it. "
- "'mla_mtp_disabled' -> "
- "an Auto-mode policy downgrade: the model is MLA (GLM-5.2 et al.) "
- "whose llama.cpp MTP path runs slower than no speculation, so Auto "
- "used ngram-mod or spec-off instead -- updating won't help; choose "
- "MTP in Settings (or set UNSLOTH_MLA_MTP_ENABLED=1) to force it. "
- "'mtp_partial_offload' -> an Auto-mode policy downgrade: the model "
- "has an embedded Hybrid Mamba MTP head and the placement offloads "
- "only part of it, where the recurrent rollback copies cost more "
- "layers than the drafting wins back -- updating won't help; choose "
- "MTP in Settings to force it. "
- "None when the requested strategy engaged or was not requested."
- ),
- )
spec_fallback_binary_changed: Optional[bool] = Field(
None,
description = (
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 2f2f9d198ca..6d3ee5d50b4 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3740,6 +3740,8 @@ from models.inference import (
EstimateMemoryResponse,
Int8PrefillAvailabilityRequest,
Int8PrefillAvailabilityResponse,
+ MlxDraftersRequest,
+ MlxDraftersResponse,
TransformersUpgradeInfo,
TransformersUpgradeCheckRequest,
TransformersUpgradeCheckResponse,
@@ -8374,6 +8376,11 @@ def _unsloth_serving_fields(model_info: dict) -> dict:
"context_length_enforced": model_info.get("context_length_enforced"),
"context_length_fitted": _positive_int_or_none(model_info.get("context_length_fitted")),
"context_unbounded_when_batched": bool(model_info.get("context_unbounded_when_batched")),
+ "speculative_type": model_info.get("speculative_type"),
+ "spec_draft_n_max": model_info.get("spec_draft_n_max"),
+ "spec_draft_model": model_info.get("spec_draft_model"),
+ "spec_drafter_kind": model_info.get("spec_drafter_kind"),
+ "spec_fallback_reason": model_info.get("spec_fallback_reason"),
}
@@ -8402,6 +8409,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
mlx_int8_prefill_requested = None,
mlx_int8_prefill_reason = None,
mlx_context_budget = None,
+ spec_draft_model = None,
chat_template_override_reason = None,
context_length_enforced = True,
context_length_fitted = None,
@@ -11928,6 +11936,7 @@ async def load_model_for_preview(
) -> None:
if account_access.managed_account():
await asyncio.to_thread(account_access.require_model_access, request.model_path)
+ await _require_drafter_access(request)
account_access.require_idle_other_accounts()
from core.inference.llama_keepwarm import (
inference_lifecycle_gate,
@@ -14988,7 +14997,7 @@ def _mlx_estimate_ceiling(model_dir: str) -> Optional[int]:
return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)
-def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits):
+def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits, **route):
"""The window a load naming no Context Length would be fitted to."""
from core.inference.mlx_inference import (
mlx_fit_to_memory,
@@ -15006,9 +15015,48 @@ def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_
not getattr(config, "is_vision", False) or mlx_vlm_snapshot_store_available()
),
kv_bits = kv_bits,
+ **route,
)
+def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_in_4bit: bool):
+ """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it
+ would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""
+ from core.inference import mlx_speculative
+ from core.inference.mlx_inference import mlx_drafter_fit, parse_mlx_kv_quant
+ from core.inference.mlx_memory import _loads_as_vision, _snapshot_config
+
+ mode = mlx_speculative.mlx_spec_mode(request.speculative_type)
+ if not mlx_speculative.speculates_on_route(
+ mode, _loads_as_vision(_snapshot_config(model_dir) or {}), request.spec_draft_model
+ ):
+ return None, None, False
+ resolution = mlx_speculative.resolve_speculation(
+ request.speculative_type,
+ request.spec_draft_model,
+ model_dir = model_dir,
+ target_name = model_identifier,
+ allowed = _discoverable_drafters(model_identifier),
+ )
+ bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)
+ legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None
+ kv_quant = bits is not None or turboquant or legacy_bits
+ if mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False):
+ return None, None, False
+ for source in resolution.sources:
+ attaches, fitted = mlx_drafter_fit(
+ model_dir,
+ ceiling,
+ request.max_seq_length or None,
+ source,
+ load_in_4bit = load_in_4bit,
+ costless = mode == "auto",
+ )
+ if attaches:
+ return (source.path, source.builtin), fitted, True
+ return None, None, True
+
+
def _mlx_estimate_available() -> bool:
"""Whether this host would actually run the load through MLX."""
try:
@@ -17177,6 +17225,8 @@ def _mlx_runtime_settings_match(backend, request) -> bool:
return True
from core.inference.mlx_inference import encode_mlx_kv_quant, parse_mlx_kv_quant
+ from core.inference.mlx_speculative import mlx_spec_mode
+
requested = encode_mlx_kv_quant(*parse_mlx_kv_quant(getattr(request, "mlx_kv_quant", None)))
return (
entry["mlx_kv_quant_requested"] == requested
@@ -17184,6 +17234,9 @@ def _mlx_runtime_settings_match(backend, request) -> bool:
== (request.chat_template_override or None)
and bool(entry.get("mlx_int8_prefill_requested"))
== bool(getattr(request, "mlx_int8_prefill", False))
+ and entry.get("speculative_type", "auto") == mlx_spec_mode(request.speculative_type)
+ and entry.get("spec_draft_model") == (getattr(request, "spec_draft_model", None) or None)
+ and entry.get("spec_draft_n_max") == _positive_int_or_none(request.spec_draft_n_max)
)
@@ -17917,6 +17970,20 @@ async def _run_gguf_load_attempt(
return False
+async def _require_drafter_access(request) -> None:
+ """A named MLX drafter is a second model reference, so the caller needs access to it too."""
+ drafter = getattr(request, "spec_draft_model", None)
+ if account_access.managed_account() and isinstance(drafter, str) and drafter.strip():
+ await asyncio.to_thread(account_access.require_model_access, drafter.strip())
+
+
+def _discoverable_drafters(model_path: str) -> Optional[list]:
+ """The cached drafters a load may attach on its own: any for the owner, else those the account may see."""
+ if not account_access.managed_account():
+ return None
+ return [row["repo_id"] for row in _mlx_cached_drafters(model_path)]
+
+
def _require_resolved_base_access(config) -> None:
"""Grants apply to the base in an adapter's config, so it cannot pull a foreign cached base."""
base = getattr(config, "base_model", None)
@@ -18133,6 +18200,7 @@ async def _load_model_impl(
native_access_deferred = _defers_access_to_native_grant(request)
if account_access.managed_account() and not native_access_deferred:
await asyncio.to_thread(account_access.require_model_access, request.model_path)
+ await _require_drafter_access(request)
request = await asyncio.to_thread(_as_ollama_manifest_request, request)
_requested_model_id = request.model_path
request = await asyncio.to_thread(
@@ -19217,6 +19285,12 @@ async def _load_model_impl(
subject = current_subject,
mlx_kv_quant = request.mlx_kv_quant,
mlx_int8_prefill = request.mlx_int8_prefill,
+ speculative_type = request.speculative_type,
+ spec_draft_n_max = request.spec_draft_n_max,
+ spec_draft_model = request.spec_draft_model,
+ spec_drafters_allowed = await asyncio.to_thread(
+ _discoverable_drafters, request.model_path
+ ),
chat_template_override = request.chat_template_override,
load_cancel_event = load_cancel_event,
on_prior_worker_released = _release_chat_after_teardown,
@@ -20786,6 +20860,7 @@ async def estimate_memory(
native_access_deferred = _defers_access_to_native_grant(request)
if account_access.managed_account() and not native_access_deferred:
await asyncio.to_thread(account_access.require_model_access, request.model_path)
+ await _require_drafter_access(request)
from core.inference.llama_cpp import _args_place_tensors_on_cpu
from core.inference.llama_server_args import (
_effective_tensor_parallel,
@@ -20864,10 +20939,17 @@ async def estimate_memory(
if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):
mlx_kv_bits = None
mlx_named_ctx = request.max_seq_length or 0
- mlx_fitted_ctx = None
- if not mlx_named_ctx:
+ mlx_drafter, mlx_fitted_ctx, mlx_speculates = _mlx_estimate_drafter(
+ request,
+ model_identifier,
+ model_dir,
+ _mlx_estimate_ceiling(model_dir),
+ mlx_load_in_4bit,
+ )
+ mlx_route = {"vision": True} if mlx_speculates else {}
+ if not mlx_named_ctx and mlx_drafter is None:
mlx_fitted_ctx = _mlx_estimate_fitted_context(
- config, model_dir, mlx_load_in_4bit, mlx_kv_bits
+ config, model_dir, mlx_load_in_4bit, mlx_kv_bits, **mlx_route
)
mlx_priced_ctx = mlx_named_ctx or mlx_fitted_ctx or _mlx_estimate_ceiling(model_dir)
if not mlx_priced_ctx:
@@ -20877,6 +20959,8 @@ async def estimate_memory(
n_ctx = mlx_priced_ctx,
kv_bits = mlx_kv_bits,
load_in_4bit = mlx_load_in_4bit,
+ **mlx_route,
+ **({} if mlx_drafter is None else {"drafter": mlx_drafter}),
)
if mlx_breakdown is None:
return EstimateMemoryResponse(available = False, reason = "unsizable")
@@ -21019,6 +21103,35 @@ async def estimate_memory(
return await asyncio.to_thread(_estimate)
+def _mlx_cached_drafters(model_path: str) -> list:
+ from core.inference import mlx_speculative
+ from utils.utils import hf_cache_snapshot_dir
+
+ model_dir = model_path if os.path.isdir(model_path) else hf_cache_snapshot_dir(model_path)
+ if model_dir is None:
+ return []
+ config = mlx_speculative._read_config(model_dir)
+ # The cache is shared between accounts: list only what this one may see.
+ return account_access.filter_model_rows(
+ [
+ {"repo_id": repo, "kind": source.kind, "named": named}
+ for repo, source, named in mlx_speculative.cached_drafters(model_path, config)
+ ]
+ )
+
+
[email protected]("/mlx-drafters", response_model = MlxDraftersResponse)
+async def mlx_drafters(
+ request: MlxDraftersRequest, current_subject: str = Depends(get_current_subject)
+):
+ """Cached drafters an MLX load of this model could name. Reads only local config files."""
+ if account_access.managed_account():
+ await asyncio.to_thread(account_access.require_model_access, request.model_path)
+ return MlxDraftersResponse(
+ drafters = await asyncio.to_thread(_mlx_cached_drafters, request.model_path)
+ )
+
+
@router.post("/unload", response_model = UnloadResponse)
@account_access.gpu_busy_route
async def unload_model(request: UnloadRequest, current_subject: str = Depends(get_current_subject)):
@@ -22028,7 +22141,6 @@ async def _slot_status(current_subject: str):
**_runtime_fields,
requested_context_length = llama_backend.requested_n_ctx,
llama_cpp_supports_mtp = _supports_mtp,
- spec_fallback_reason = llama_backend.spec_fallback_reason,
spec_fallback_binary_changed = _spec_fallback_binary_changed(llama_backend),
spec_probe_retry_pending = _spec_probe_retry_pending(llama_backend),
spec_dflash_retry_pending = _spec_dflash_retry_pending(llama_backend),
@@ -22039,7 +22151,6 @@ async def _slot_status(current_subject: str):
tensor_parallel_dropped_by_arch_gate = _arch_gate_dropped_tensor_parallel(
llama_backend
),
- spec_drafter_kind = llama_backend.spec_drafter_kind,
llama_cpp_prebuilt_stale = _stale,
llama_cpp_installed_tag = _installed_tag,
llama_cpp_latest_tag = _latest_tag,
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 428d9726ac2..a186fa7d9b9 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -998,6 +998,7 @@ class ModelOverridePayload(BaseModel):
speculative_type: Optional[str] = Field(default = None, max_length = 32)
spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16)
+ spec_draft_model: Optional[str] = Field(default = None, max_length = 1024)
# Parallel decode slots (llama-server --parallel), GGUF-only; None follows the server default.
n_parallel: Optional[int] = Field(default = None, ge = PARALLEL_SLOTS_MIN, le = PARALLEL_SLOTS_MAX)
reasoning_budget: Optional[int] = Field(default = None, ge = -1, le = 2_147_483_647)
@@ -1020,6 +1021,7 @@ class ModelOverridePayload(BaseModel):
# The reasoning pair came later than the four, so a build that mirrors them can still
# predate it: its own flag, same contract.
mirrors_reasoning_budget: bool = False
+ mirrors_spec_draft_model: bool = False
tensor_parallel: bool = False
disable_vision: bool = False
mlx_int8_prefill: bool = False
@@ -2496,6 +2498,7 @@ def update_openai_auto_switch_override(
"fill_absent_fields",
"mirrors_server_tuning",
"mirrors_reasoning_budget",
+ "mirrors_spec_draft_model",
},
exclude_none = True,
)
@@ -2571,10 +2574,16 @@ def update_openai_auto_switch_override(
# legacy contract is a payload carrying only model_id, which leaves remove None.
_tuning_fields = ("load_mode", "spec_draft_cache_type", "ctx_checkpoints", "cache_ram")
_reasoning_fields = ("reasoning_budget", "reasoning_budget_message")
- _kept_tuning = {name: getattr(payload, name) for name in _tuning_fields + _reasoning_fields}
+ _drafter_fields = ("spec_draft_model",)
+ _kept_tuning = {
+ name: getattr(payload, name)
+ for name in _tuning_fields + _reasoning_fields + _drafter_fields
+ }
# Each group is carried only for a client that does not mirror it.
- _carried_fields = (() if payload.mirrors_server_tuning else _tuning_fields) + (
- () if payload.mirrors_reasoning_budget else _reasoning_fields
+ _carried_fields = (
+ (() if payload.mirrors_server_tuning else _tuning_fields)
+ + (() if payload.mirrors_reasoning_budget else _reasoning_fields)
+ + (() if payload.mirrors_spec_draft_model else _drafter_fields)
)
if _carried_fields and not is_removal:
# The same spellings the extra-args carry-over walks: a cached repo is not an ordinary folded match,
@@ -2705,6 +2714,7 @@ def update_openai_auto_switch_override(
mlx_kv_quant = payload.mlx_kv_quant,
speculative_type = payload.speculative_type,
spec_draft_n_max = payload.spec_draft_n_max,
+ spec_draft_model = _kept_tuning["spec_draft_model"],
n_parallel = payload.n_parallel,
reasoning_budget = (
None
diff --git a/studio/backend/tests/test_account_inference_isolation.py b/studio/backend/tests/test_account_inference_isolation.py
index 6bcd6563c5c..34b59582618 100644
--- a/studio/backend/tests/test_account_inference_isolation.py
+++ b/studio/backend/tests/test_account_inference_isolation.py
@@ -284,6 +284,16 @@ def test_a_resolved_adapter_base_needs_its_own_grant(monkeypatch, base, expected
inference._require_resolved_base_access(config)
+def test_a_named_mlx_drafter_needs_its_own_grant(monkeypatch):
+ monkeypatch.setattr(access, "repo_is_public", lambda repo, *a, **k: repo == "org/public")
+ monkeypatch.setattr(inference, "_mlx_cached_drafters", lambda path: [{"repo_id": "org/d"}])
+ assert run_as(BOB, inference._discoverable_drafters, "org/public") == ["org/d"]
+ assert inference._discoverable_drafters("org/public") is None
+ body = {"model_path": "org/public", "spec_draft_model": "org/private"}
+ with client_for(BOB) as client:
+ assert client.post("/api/inference/estimate-memory", json = body).status_code == 404
+
+
def test_preview_load_refuses_private_foreign_target_before_gpu_work():
with pytest.raises(HTTPException) as exc:
asyncio.run(
diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.py
index 102a848b904..4b4ba5feffd 100644
--- a/studio/backend/tests/test_active_generations.py
+++ b/studio/backend/tests/test_active_generations.py
@@ -18,7 +18,7 @@ import pytest
_backend = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, _backend)
-from core.inference.worker import _SLOTS, StopLedger
+from core.inference.worker import _SLOTS, StopLedger, _Stops
from state import active_generations
@@ -2600,6 +2600,8 @@ def test_a_request_reads_as_stopped_once_it_is():
assert _stopped(ledger) == set()
assert ledger.stop(mine)
+ stops = _Stops(ledger, None, None, [])
+ assert stops.unread() and stops._refresh() is None and mine in stops and not stops.unread()
assert _stopped(ledger) == {mine}, "a stop names one request and no other"
assert theirs not in _stopped(ledger)
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 820b9b375fb..51da3829967 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1195,6 +1195,7 @@ class TestLoadHubDownloadExclusion:
"mlx_int8_prefill_requested",
"mlx_int8_prefill_reason",
"mlx_context_budget",
+ "spec_draft_model",
"chat_template_override_reason",
# Constant True: llama.cpp allocates the window it reports.
"context_length_enforced",
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index b22cb7bd66c..b3be5d2094f 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -1016,6 +1016,89 @@ class TestEstimateMemoryRoute:
monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", _breakdown)
return seen
+ def test_an_explicit_drafter_is_priced_as_the_load_would_attach_it(self, monkeypatch):
+ import core.inference.mlx_inference as mlx_inference
+ import core.inference.mlx_memory as mlx_memory
+ from core.inference import mlx_speculative as spec
+
+ self._mlx_target(monkeypatch, "/models/thing")
+ sources = (spec.DrafterSource("mtp", "/big", True), spec.DrafterSource("mtp", "/ok", False))
+ monkeypatch.setattr(
+ spec,
+ "resolve_speculation",
+ lambda mode, *a, **k: spec.SpecResolution(mode, sources if mode == "mtp" else ()),
+ )
+ monkeypatch.setattr(
+ spec, "speculation_refusal", lambda kv_quant, **_: spec.KV_QUANT if kv_quant else None
+ )
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_drafter_fit",
+ lambda d, ceiling, pinned, source, **_: (
+ source.path == "/ok" and (pinned or 0) <= 8192,
+ None if pinned else 12288,
+ ),
+ )
+ monkeypatch.setattr(ri, "_mlx_estimate_ceiling", lambda model_dir: 32768)
+ monkeypatch.setattr(
+ ri, "_mlx_estimate_fitted_context", lambda *a, **kw: 24576 if kw else 20480
+ )
+ priced = []
+ monkeypatch.setattr(
+ mlx_memory,
+ "mlx_memory_breakdown",
+ lambda d, *, n_ctx, **kw: priced.append((n_ctx, kw.get("drafter"), kw.get("vision")))
+ or mlx_memory.MlxMemoryBreakdown(1, 0, 0, 1, 1, n_ctx = n_ctx),
+ )
+ assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288
+ for pinned in (4096, 65536):
+ _estimate(model_path = "org/model", speculative_type = "mtp", max_seq_length = pinned)
+ _estimate(model_path = "org/model", speculative_type = "mtp", mlx_kv_quant = "8")
+ _estimate(model_path = "org/model")
+ _estimate(model_path = "o/m", speculative_type = "ngram", mlx_kv_quant = "auto", mlx_kv_bits = 8)
+ _estimate(model_path = "org/model", spec_draft_model = "org/d") # named under Auto
+ assert [route for *_, route in priced] == [True, True, True, None, None, True, True]
+ assert [row[:2] for row in priced[:6]] == [
+ (12288, ("/ok", False)),
+ (4096, ("/ok", False)),
+ (65536, None),
+ (20480, None),
+ (20480, None),
+ (24576, None),
+ ]
+
+ def test_a_drafter_attaches_where_its_priced_fit_holds_and_a_pin_fits_whole(self, monkeypatch):
+ import core.inference.mlx_inference as mlx_inference
+ import core.inference.mlx_memory as mlx_memory
+ from core.inference.mlx_speculative import DrafterSource
+
+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 << 30)
+ priced = []
+
+ def fit(pinned, outcome, *bare):
+ monkeypatch.setattr(
+ mlx_memory,
+ "mlx_fit_outcome",
+ lambda d, *, max_ctx, vision, drafter = None, **_: priced.append(
+ (max_ctx, drafter, vision)
+ )
+ or ((outcome, 4096) if drafter else ("fits", bare[0])),
+ )
+ source = DrafterSource("dflash", "/drafter", False)
+ return mlx_inference.mlx_drafter_fit(
+ "/m", 32768, pinned, source, load_in_4bit = False, costless = bool(bare)
+ )
+
+ attaches = [fit(None, o)[0] for o in ("fits", "fitted", "unsizable", "no_fit")]
+ assert attaches == [True, True, None, False]
+ assert (fit(8192, "fits"), fit(8192, "fitted")) == ((True, 4096), (False, 4096))
+ assert (
+ priced
+ == [(32768, ("/drafter", False), True)] * 4 + [(8192, ("/drafter", False), True)] * 2
+ )
+ costless = [fit(None, *c) for c in (("fits", 4096), ("fits", 8192), ("unsizable", 4096))]
+ assert costless == [(True, 4096), (False, 4096), (None, 4096)]
+
def test_mlx_model_not_on_disk_is_not_downloaded(self, monkeypatch):
self._mlx_target(monkeypatch, None)
resp = _estimate(model_path = "org/model")
@@ -4030,6 +4113,11 @@ def test_a_load_is_sized_once_until_its_files_change(monkeypatch, tmp_path):
assert mm._size_load(str(tmp_path), None, None, True) != first and len(built) == 2
shard.write_bytes(b"xy")
assert mm._size_load(str(tmp_path), 4, None, True) != first and len(built) == 3
+ (drafter := tmp_path / "drafter").mkdir()
+ sized = lambda: mm._size_load(str(tmp_path), 4, None, True, True, (str(drafter), False))
+ first = sized()
+ (drafter / "config.json").write_text("{}")
+ assert sized() != first # a changed local drafter is re-sized too
try:
@@ -5025,6 +5113,17 @@ class TestTheContextSearch:
monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)
assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None
+ def test_the_outcome_says_why_no_context_came_back(self, monkeypatch):
+ self._linear(monkeypatch)
+ assert mm.mlx_fit_outcome("x", budget_bytes = 9000, max_ctx = 8192) == ("fits", None)
+ assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("fitted", 6912)
+ assert mm.mlx_fit_outcome("x", budget_bytes = 4200, max_ctx = 8192, min_ctx = 4100) == (
+ "no_fit",
+ None,
+ )
+ monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)
+ assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("unsizable", None)
+
@_NEEDS_MLX
@pytest.mark.parametrize("budget_gib, fitted", [(6, 18_432), (12, 61_952), (24, 149_504)])
@@ -5042,6 +5141,23 @@ def test_a_real_checkpoint_is_fitted_to_the_byte(budget_gib, fitted):
assert mm.mlx_memory_breakdown(snapshot, n_ctx = float("nan"), load_in_4bit = True) is None
+@_NEEDS_MLX
+def test_a_companion_drafter_adds_its_stored_weights_and_a_cache_to_the_load():
+ _on_bfloat16_chip()
+ target = _local_snapshot("mlx-community/Qwen3.5-4B-4bit")
+ drafter = _local_snapshot("z-lab/Qwen3.5-4B-DFlash")
+ stored = 0
+ for shard in glob.glob(os.path.join(drafter, "*.safetensors")):
+ with open(shard, "rb") as handle:
+ header = json.loads(handle.read(int.from_bytes(handle.read(8), "little")))
+ header.pop("__metadata__", None)
+ stored += sum(end - start for start, end in (t["data_offsets"] for t in header.values()))
+ alone = mm.mlx_memory_breakdown(target, n_ctx = 8192, vision = True)
+ paired = mm.mlx_memory_breakdown(target, n_ctx = 8192, vision = True, drafter = (drafter, False))
+ assert paired.weights_bytes - alone.weights_bytes == stored
+ assert paired.kv_bytes > alone.kv_bytes and paired.compute_bytes > alone.compute_bytes
+
+
@_NEEDS_MLX
@pytest.mark.parametrize(
"repo, kv_bits, whole_prompt",
@@ -5061,3 +5177,28 @@ def test_the_total_never_falls_as_the_context_grows(repo, kv_bits, whole_prompt)
steps = range(mm.MLX_KV_BLOCK, 262_145, mm.MLX_KV_BLOCK)
totals = [mm._priced_at(sizing, n).total_bytes for n in steps]
assert totals == sorted(totals)
+
+
+def test_the_drafter_picker_lists_cached_drafters_of_a_cached_target(tmp_path, monkeypatch):
+ from core.inference import mlx_speculative
+
+ (tmp_path / "config.json").write_text('{"vocab_size": 7}')
+ found = [
+ ("o/T-DFlash", SimpleNamespace(kind = "dflash"), True),
+ ("o/T-MTP", SimpleNamespace(kind = "mtp"), False),
+ ]
+ monkeypatch.setattr(
+ mlx_speculative,
+ "cached_drafters",
+ lambda name, config: found if (name, config) == ("o/T", {"vocab_size": 7}) else [],
+ )
+ monkeypatch.setattr(
+ "utils.utils.hf_cache_snapshot_dir", lambda name: tmp_path if name == "o/T" else None
+ )
+ assert ri._mlx_cached_drafters("o/T") == [
+ {"repo_id": "o/T-DFlash", "kind": "dflash", "named": True},
+ {"repo_id": "o/T-MTP", "kind": "mtp", "named": False},
+ ]
+ assert ri._mlx_cached_drafters("o/absent") == []
+ monkeypatch.setattr(ri.account_access, "filter_model_rows", lambda rows: rows[1:])
+ assert [row["repo_id"] for row in ri._mlx_cached_drafters("o/T")] == ["o/T-MTP"]
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index a4ccb2c4660..b5712df569e 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -3280,6 +3280,91 @@ def _tiny_lm(
return _LM()
+def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fits(monkeypatch):
+ pytest.importorskip("mlx.core")
+ drafters = pytest.importorskip("unsloth_zoo.mlx.speculative")
+
+ from core.inference import mlx_inference, mlx_speculative
+
+ attempts, fits = [], []
+ failing = []
+
+ class _Loader:
+ @staticmethod
+ def from_pretrained(name, **kwargs):
+ attempts.append(kwargs["text_only"])
+ if not kwargs["text_only"] and failing:
+ raise ValueError("mlx-vlm has no such architecture")
+ return SimpleNamespace(config = {}), SimpleNamespace()
+
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader.FastMLXModel = _Loader
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)
+ monkeypatch.setattr(
+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (8192, 8192, 8192)
+ )
+ source = mlx_speculative.DrafterSource("dflash", "/drafter", False)
+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)
+ monkeypatch.setattr(
+ mlx_speculative,
+ "resolve_speculation",
+ lambda mode, *_a, **_k: mlx_speculative.SpecResolution(mode, (source,)),
+ )
+ monkeypatch.setattr(
+ mlx_inference, "mlx_drafter_fit", lambda *a, **k: fits.append(a[1:4]) or (True, 4096)
+ )
+ monkeypatch.setattr(drafters, "companion_drafter", lambda path, target: SimpleNamespace())
+ config = SimpleNamespace(identifier = "org/text", is_vision = False, is_lora = False)
+
+ backend = mlx_inference.MLXInferenceBackend()
+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash", spec_draft_n_max = 5)
+ entry = backend.models["org/text"]
+ assert attempts == [False] and fits == [(8192, None, source)]
+ assert (entry["spec_drafter_kind"], entry["spec_fallback_reason"]) == ("dflash", None)
+ assert entry["context_length"] == 4096 and entry["spec_draft_n_max"] == 5
+ assert backend._speculative_draft.controller.max_depth == 5
+
+ attempts.clear(), failing.append(True)
+ backend.load_model(config, speculative_type = "dflash")
+ entry = backend.models["org/text"]
+ assert attempts == [False, True] and backend._speculative_draft is None
+ assert entry["spec_fallback_reason"] == mlx_speculative.RUNTIME_ERROR
+ attempts.clear()
+ backend.load_model(config)
+ assert attempts == [True] and backend.models["org/text"]["speculative_type"] == "auto"
+ failing.clear()
+ fit = lambda *a, **kw: fits.append(kw.get("vision")) or (None, None)
+ monkeypatch.setattr(mlx_inference, "_fitted_context", fit)
+ monkeypatch.setattr(mlx_inference, "mlx_drafter_fit", lambda *a, **k: (False, None))
+ backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")
+ assert fits[-1] is True # copies alone still load through mlx-vlm, so the fit prices that route
+ backend.load_model(config, max_seq_length = 0, spec_draft_model = "org/d")
+ assert fits[-2:] == [True, True] # a named drafter enters the speculative route under Auto
+ # ... and is a request, so a load that cannot honour it says why.
+ failing.append(True)
+ backend.load_model(config, spec_draft_model = "org/d")
+ monkeypatch.setattr(
+ mlx_speculative, "speculation_refusal", lambda **_k: mlx_speculative.KV_QUANT
+ )
+ reasons = [
+ entry["spec_fallback_reason"]
+ for entry in (
+ dict(backend.models["org/text"]),
+ backend.load_model(config, spec_draft_model = "org/d") and backend.models["org/text"],
+ )
+ ]
+ assert reasons == [mlx_speculative.RUNTIME_ERROR, mlx_speculative.KV_QUANT]
+ # A diffusion model gets no draft at load, so the resident batch cannot speculate on it either.
+ failing.clear()
+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)
+ monkeypatch.setattr(mlx_inference, "_vlm_generation_is_diffusion", lambda model: True)
+ backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")
+ entry = backend.models["org/text"]
+ assert backend._speculative_draft is None and entry["spec_drafter_kind"] is None
+ assert entry["spec_fallback_reason"] == mlx_speculative.RUNTIME_ERROR
+
+
def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypatch):
pytest.importorskip("mlx.core")
from core.inference import mlx_inference
@@ -3435,6 +3520,16 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():
"chat_template_override_reason": "it could not render a conversation",
}
assert _mlx_runtime_settings_match(be, req(chat_template_override = "{{ refused }}"))
+ # Speculation compares canonically: an absent mode is the resident auto.
+ be.models["m"] = {"mlx_kv_quant_requested": "auto", "speculative_type": "auto"}
+ assert _mlx_runtime_settings_match(be, req(speculative_type = "default"))
+ assert not _mlx_runtime_settings_match(be, req(speculative_type = "mtp"))
+ be.models["m"].update(speculative_type = "mtp", spec_draft_n_max = 4)
+ assert _mlx_runtime_settings_match(be, req(speculative_type = "draft-mtp", spec_draft_n_max = 4))
+ assert not _mlx_runtime_settings_match(be, req(speculative_type = "mtp"))
+ assert not _mlx_runtime_settings_match(
+ be, req(speculative_type = "mtp", spec_draft_n_max = 4, spec_draft_model = "org/drafter")
+ )
resp = LoadResponse(
status = "loaded",
@@ -5261,6 +5356,7 @@ def _drive_vlm_generation(
max_new_tokens = 1,
session = None,
video = None,
+ **request,
):
import sys
import types
@@ -5296,11 +5392,73 @@ def _drive_vlm_generation(
image = image,
video = video,
max_new_tokens = max_new_tokens,
+ **request,
)
)
return seen
+def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):
+ pytest.importorskip("mlx_lm")
+ import sys
+ import types
+
+ from core.inference import mlx_inference
+
+ class _Ids(list):
+ def tolist(self):
+ return list(self)
+
+ utils = types.ModuleType("mlx_vlm.utils")
+ utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7, 8, 9])], "extra": 1}
+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)
+ prepared = []
+ draft = SimpleNamespace(draft_kind = "mtp", draft_n = 6, draft_n_accepted = 4)
+ draft.prepare = lambda ids, sampling, given: prepared.append(
+ (ids, sampling.temperature, sampling.top_k)
+ ) or setattr(draft, "given", given)
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._speculative_draft = draft
+
+ # top_k -1 is the API's "disabled", which the drafter's sampling settings spell 0.
+ seen = _drive_vlm_generation(
+ backend, monkeypatch, temperature = 0.5, top_k = -1, repetition_penalty = 1.1
+ )
+ assert seen["draft_model"] is draft and seen["extra"] == 1 and prepared == [([7, 8, 9], 0.5, 0)]
+ # The draft is told every processor the reply runs: none reaches mlx-vlm as a shortcut.
+ assert "repetition_penalty" not in seen and draft.given == seen["logits_processors"] != []
+ monkeypatch.setattr(
+ mlx_inference, "_mlx_sampling_processors", lambda **_: [lambda tokens, logits: logits]
+ )
+ timings = backend.last_generation_stats["timings"]
+ assert (timings["draft_n"], timings["draft_n_accepted"]) == (6, 4)
+ for request in (
+ {"repetition_penalty": 1.1},
+ {"tools": [{"type": "function"}]},
+ {"tool_protocol_active": True},
+ ):
+ seen = _drive_vlm_generation(backend, monkeypatch, **request)
+ assert "max_tokens" in seen and "draft_model" not in seen
+ assert "draft_n" not in backend.last_generation_stats["timings"]
+ assert len(prepared) == 1
+ opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r: None
+ backend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)
+ assert backend.batch_unavailable_reason([{}, {}]) == "no stream"
+ monkeypatch.setattr(
+ _batch_engine(monkeypatch), "BatchStream", lambda *a, **k: opened.append(k["speculative"])
+ )
+ backend._generate_session_batch = lambda r, kind, **_: opened.append(kind) or ()
+ mlx_inference._VisionBatchSession(backend, width = 2)
+ list(backend.generate_chat_batch([{}, {}]))
+ assert opened == [draft, mlx_inference._VisionBatchSession]
+ del backend._generate_session_batch
+ steps, cancel = [], threading.Event()
+ step = lambda waiting: steps.append(waiting) or setattr(session, "rows_in_flight", 0) or ()
+ session = SimpleNamespace(admit = lambda *a: None, rows_in_flight = 1, step = step, close = list)
+ list(backend._generate_session_batch([{}], lambda *a, **k: session, cancel_event = cancel))
+ assert steps == [cancel.is_set] # a cancelled reply ends its speculative step early
+
+
def _stub_prepare_inputs(monkeypatch, per_medium = 520):
import sys
import types
@@ -5910,6 +6068,7 @@ def test_what_the_fit_is_asked_and_when_it_is_asked_at_all(monkeypatch, tmp_path
"dir": str(tmp_path),
}
]
+ assert fit(vision = True) and priced[0]["vision"] is True
# The width the cache will take is the width the fit is priced at.
for answer, bits in ((("full", "", True), 4), (("partial", "w", True), 4)):
verdict["answer"] = answer
@@ -6214,14 +6373,19 @@ class _TwoRowStream:
):
self._events, self._retired, self.withdrawn = events, set(retired), []
- def step(self):
- return iter(self._events)
+ def iter_step(self, waiting):
+ yield from self._events
+ self.exhausted = True
def withdraw(self, row):
+ assert self.exhausted, "the batch cannot change mid-step"
self.withdrawn.append(row)
if row in self._retired:
return None
- return SimpleNamespace(prompt_token_count = 3, token_ids = [1, 2])
+ return SimpleNamespace(prompt_token_count = 3, token_ids = [1, 2], **_DRAFTS)
+
+
+_DRAFTS = {"draft_tokens": 8, "accepted_draft_tokens": 3}
def _open_vision_session(rows):
@@ -6231,6 +6395,7 @@ def _open_vision_session(rows):
session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)
session._rows = {row.handle: row for row in rows}
session._by_row, session._settled = {row.row: row for row in rows}, {}
+ session._speculative = None
return session
@@ -6252,8 +6417,9 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto
for number, handle in enumerate(handles)
]
)
- other = 1 - stopping
+ other, session._speculative = 1 - stopping, "draft"
result = SimpleNamespace(finish_reason = "length", prompt_token_count = 3, token_ids = [1, 2])
+ vars(result).update(_DRAFTS)
session.stream = _TwoRowStream(
[
SimpleNamespace(
@@ -6268,6 +6434,8 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto
assert (handles[stopping], None) in reported, "the stopped row is reported finished"
assert session._settled[handles[stopping]]["finish_reason"] == "stop"
+ timings = session._settled[handles[stopping]]["timings"]
+ assert (timings["draft_n"], timings["draft_n_accepted"]) == (8, 3)
assert handles[stopping] not in session._rows, "and it has left the batch"
assert handles[other] in session._rows, "its neighbour is still in the batch"
assert (handles[other], "still going") in reported
diff --git a/studio/backend/tests/test_mlx_resident_batch.py b/studio/backend/tests/test_mlx_resident_batch.py
index 2dfa88027a1..0a7055d8bc6 100644
--- a/studio/backend/tests/test_mlx_resident_batch.py
+++ b/studio/backend/tests/test_mlx_resident_batch.py
@@ -79,7 +79,7 @@ class _Session:
self.rows.remove(handle)
self.settled[handle] = self.stats_at_retire.get(handle, {})
- def step(self):
+ def step(self, waiting):
for handle in list(self.rows):
events = self.script.get(handle) or []
if not events:
diff --git a/studio/backend/tests/test_mlx_speculative.py b/studio/backend/tests/test_mlx_speculative.py
new file mode 100644
index 00000000000..3087ac9f8ea
--- /dev/null
+++ b/studio/backend/tests/test_mlx_speculative.py
@@ -0,0 +1,186 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import json
+import os
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference import mlx_speculative as spec
+
+_DFLASH = {"architectures": ["DFlashDraftModel"], "dflash_config": {}, "vocab_size": 10}
+_EAGLE3 = {"architectures": ["Eagle3Speculator"], "speculators_model_type": "eagle3"}
+_ASSISTANT = {"architectures": ["Gemma4AssistantForCausalLM"], "model_type": "gemma4_assistant"}
+
+
+_TARGET = "org/Qwen3.5-4B-4bit"
+
+
+def cache(tmp_path, monkeypatch):
+ """Fills a Hugging Face cache with ``{repo: config}`` beside the target, and returns a resolver
+ for the target."""
+ snapshots = {}
+
+ def fill(repos, builtin = False):
+ for repo, config in {_TARGET: {"vocab_size": 10}, **repos}.items():
+ snapshots[repo] = tmp_path / repo.replace("/", "--")
+ snapshots[repo].mkdir(parents = True, exist_ok = True)
+ (snapshots[repo] / "config.json").write_text(json.dumps(config))
+ (snapshots[repo] / "model.safetensors").touch()
+ monkeypatch.setattr(spec, "has_builtin_head", lambda model_dir: builtin)
+ return lambda mode, named = None: spec.resolve_speculation(
+ mode, named, model_dir = str(snapshots[_TARGET]), target_name = _TARGET
+ )
+
+ monkeypatch.setattr(spec, "_cached_repos", lambda: iter(list(snapshots)))
+ monkeypatch.setattr(
+ "utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: snapshots.get(repo)
+ )
+ return fill
+
+
+def test_an_explicit_kind_tries_the_named_drafter_then_the_head_then_cached_companions(cache):
+ _resolve = cache(
+ {
+ "a/Qwen3.5-4B-DFlash": _DFLASH,
+ "b/Qwen3.5-4B-Eagle3": _EAGLE3,
+ "e/Qwen3.5-4B-Eagle3": {
+ "architectures": ["LlamaForCausalLMEagle3"],
+ "model_type": "llama",
+ },
+ "c/Qwen3.5-9B-DFlash": _DFLASH, # another model
+ "d/Qwen3.5-4B-DFlash-bf16": {**_DFLASH, "vocab_size": 11}, # another vocabulary
+ "g/Qwen3.5-4B-assistant": _ASSISTANT,
+ },
+ builtin = True,
+ )
+ assert [os.path.basename(s.path) for s in _resolve("dflash").sources] == [
+ "a--Qwen3.5-4B-DFlash"
+ ]
+ assert _resolve("dflash").copies and spec.mlx_spec_mode("mtp+ngram") == "mtp"
+ assert spec.mlx_spec_mode("eagle3+ngram") == "auto"
+ assert [os.path.basename(s.path) for s in _resolve("eagle3").sources] == [
+ "b--Qwen3.5-4B-Eagle3"
+ ]
+ named = _resolve("eagle3", "b/Qwen3.5-4B-Eagle3")
+ assert [s.kind for s in named.sources] == ["eagle3"] and named.copies and named.speculative
+ mtp = [(os.path.basename(s.path), s.builtin) for s in _resolve("draft-mtp").sources]
+ assert mtp == [("org--Qwen3.5-4B-4bit", True), ("g--Qwen3.5-4B-assistant", False)]
+ assert _resolve("mtp", "g/Qwen3.5-4B-assistant").sources[0].builtin is False
+ # A named drafter of another kind is refused with a reason, never substituted silently.
+ assert _resolve("dflash", "b/Qwen3.5-4B-Eagle3").reason == spec.DRAFTER_INCOMPATIBLE
+ assert _resolve("dspark").reason == spec.DRAFTER_NOT_FOUND
+ assert _resolve("dspark").speculative
+
+
+def test_auto_takes_every_cached_kind_in_preference_order(cache):
+ _resolve = cache({"b/Qwen3.5-4B-Eagle3": _EAGLE3, "a/Qwen3.5-4B-DFlash": _DFLASH}, True)
+ assert [s.kind for s in _resolve("auto").sources] == ["mtp", "dflash", "eagle3"] and not (
+ _resolve("auto").reason or _resolve("off").speculative
+ )
+ assert [spec.speculates_on_route("auto", vision) for vision in (False, True)] == [False, True]
+ listed = [(repo, s.kind) for repo, s, _ in spec.cached_drafters(_TARGET, {"vocab_size": 10})]
+ assert listed == [("a/Qwen3.5-4B-DFlash", "dflash"), ("b/Qwen3.5-4B-Eagle3", "eagle3")]
+ mtp_head = {"h/Qwen3.5-4B-MTP-bf16": {"model_type": "qwen3_5_mtp"}}
+ _resolve = cache({"g/Qwen3.5-4B-assistant": _ASSISTANT, **mtp_head}, True)
+ for mode in (None, "bogus", "auto"):
+ assert [(s.builtin, s.kind) for s in _resolve(mode).sources] == [(True, "mtp")] + [
+ (False, "mtp")
+ ] * 2 + [
+ (False, "dflash"),
+ (False, "eagle3"),
+ ]
+ assert _resolve("ngram-mod").speculative
+
+
+def test_cached_drafters_are_matched_by_shape_and_weights_then_marked_by_name(cache, tmp_path):
+ shaped = {**_DFLASH, "hidden_size": 8, "num_target_layers": 4}
+ narrow = {"target_hidden_size": 8, "transformer_layer_config": {"hidden_size": 2}}
+ cache(
+ {
+ "a/Qwen3.5-4B-DFlash": shaped,
+ "n/Other-DFlash": shaped, # fits, but is not named for the target
+ "h/Qwen3.5-4B-DFlash": {**shaped, "hidden_size": 9},
+ "t/Qwen3.5-4B-DFlash": {**shaped, "num_target_layers": 5},
+ "l/Qwen3.5-4B-DFlash": {**shaped, "dflash_config": {"target_layer_ids": [1, 5]}},
+ "w/Qwen3.5-4B-DFlash": shaped,
+ "b/Qwen3.5-4B-assistant": {**_ASSISTANT, "backbone_hidden_size": 9},
+ "m/Qwen3.5-4B-MTP": {"model_type": "x_mtp", "text_config": {"num_hidden_layers": 5}},
+ "e/Qwen3.5-4B-Eagle3": {
+ **_EAGLE3,
+ **narrow,
+ "eagle_aux_hidden_state_layer_ids": [1, 4],
+ },
+ }
+ )
+ (tmp_path / "w--Qwen3.5-4B-DFlash" / "model.safetensors").unlink()
+ target = {"vocab_size": 10, "text_config": {"hidden_size": 8, "num_hidden_layers": 4}}
+ found = spec.cached_drafters("unsloth/Qwen3.5-4B-UD-MLX-4bit", target)
+ assert [(repo, named) for repo, _, named in found] == [
+ ("a/Qwen3.5-4B-DFlash", True),
+ ("e/Qwen3.5-4B-Eagle3", True),
+ ("n/Other-DFlash", False),
+ ]
+ assert len(spec.discover_companions("unsloth/Qwen3.5-4B-UD-MLX-4bit", target)) == 2
+ assert len(spec.discover_companions("unsloth/Qwen3.5-4B-UD-MLX-4bit", target, found[1])) == 1
+ assert spec._stem("g/gemma-4-31B-it-qat-q4_0-unquantized-assistant") == "gemma431bit"
+
+
+def test_a_drafter_passed_over_keeps_its_reason_on_the_one_that_attaches(monkeypatch):
+ drafters = pytest.importorskip("unsloth_zoo.mlx.speculative")
+
+ built = []
+
+ def factory(path, target):
+ if path == "broken":
+ raise ValueError("mismatched vocabulary")
+ built.append(path)
+ return SimpleNamespace(kind = "dflash", max_depth = 15)
+
+ monkeypatch.setattr(drafters, "companion_drafter", factory)
+ monkeypatch.setattr(drafters, "native_mtp_drafter", factory)
+ sources = tuple(spec.DrafterSource("dflash", path, False) for path in ("big", "broken", "ok"))
+ fits = lambda source: (source.path != "big", 4096)
+
+ draft, kind, reason, context = spec.build_draft(
+ None, spec.SpecResolution("dflash", sources), fits = fits, draft_n_max = 6
+ )
+ assert (kind, reason, context, built) == ("dflash", spec.DRAFTER_INCOMPATIBLE, 4096, ["ok"])
+ assert (draft.controller.max_depth, draft.controller.max_copy) == (6, 0)
+ for d, n, e in ((object(), 3, True), (object(), 4, False), (draft.drafter, 15, True)):
+ assert spec._draft(d, False, n).controller.fixed_depth is e
+ draft, kind, reason, _ = spec.build_draft(
+ None, spec.SpecResolution("dflash", sources[:1], copies = True), fits = fits
+ )
+ assert (draft.drafter, kind, reason) == (None, "ngram", spec.DRAFTER_NO_MEMORY)
+ assert (draft.controller.max_depth, draft.controller.max_copy) == (0, 16)
+ auto = spec.SpecResolution("auto", sources[:1], copies = True)
+ unfit = [spec.build_draft(None, auto, fits = lambda _: (ok, None)) for ok in (False, None)]
+ assert unfit == [(None, None, spec.AUTO_CONTEXT_COST, None), (None,) * 4]
+ capped = spec.build_draft(
+ None, spec.SpecResolution("ngram", copies = True), fits = fits, draft_n_max = 5
+ )
+ assert capped[0].controller.max_copy == 5
+
+ def encoder_decoder(inputs, encoder_outputs = None):
+ pass
+
+ def cross_attending(inputs, cross_attention_states = None):
+ pass
+
+ for target in (encoder_decoder, SimpleNamespace(language_model = cross_attending)):
+ refused = spec.build_draft(
+ target, spec.SpecResolution("dflash", sources[2:], copies = True), fits = fits
+ )
+ assert refused == (None, None, spec.RUNTIME_ERROR, None)
+
+
+def test_a_drafter_carrying_its_own_model_module_is_never_found_or_attached(cache):
+ # mlx-vlm imports a config's model_file on load, so such a checkpoint would run its own code.
+ custom = {**_DFLASH, "model_file": "model.py"}
+ _resolve = cache({"a/Qwen3.5-4B-DFlash": custom})
+ assert spec.cached_drafters(_TARGET, {"vocab_size": 10}) == []
+ assert not _resolve("auto").sources and not _resolve("dflash").sources
+ assert _resolve("dflash", "a/Qwen3.5-4B-DFlash").reason == spec.DRAFTER_NOT_FOUND
diff --git a/studio/backend/tests/test_mlx_worker_audio_commands.py b/studio/backend/tests/test_mlx_worker_audio_commands.py
index 610abdcfa70..21186d1ca28 100644
--- a/studio/backend/tests/test_mlx_worker_audio_commands.py
+++ b/studio/backend/tests/test_mlx_worker_audio_commands.py
@@ -203,7 +203,7 @@ def test_mlx_loop_ticks_only_when_loaded_recent_and_idle(monkeypatch, blocked, e
def rows_in_flight(self):
return int(busy[0])
- def step(self):
+ def step(self, waiting = None):
pass
monkeypatch.setattr(worker, "_ResidentBatch", Batch)
diff --git a/studio/backend/tests/test_model_override_schema_compatibility.py b/studio/backend/tests/test_model_override_schema_compatibility.py
index 10a07abe1d5..c97b864a423 100644
--- a/studio/backend/tests/test_model_override_schema_compatibility.py
+++ b/studio/backend/tests/test_model_override_schema_compatibility.py
@@ -410,6 +410,12 @@ def test_a_client_that_does_know_the_reasoning_pair_still_clears_by_omission(ove
assert field not in after, f"{field} survived an explicit clear"
+def test_a_client_that_does_not_know_the_drafter_cannot_erase_it(override_store):
+ _put("org/m", speculative_type = "eagle3", spec_draft_model = "d", mirrors_spec_draft_model = True)
+ _put("org/m", speculative_type = "eagle3")
+ assert settings.get_model_override("org/m")["spec_draft_model"] == "d"
+
+
def test_the_reasoning_flag_is_not_itself_a_saved_field(override_store):
_put(MODEL, **PRE_TUNING_PAYLOAD, mirrors_reasoning_budget = True)
assert settings.get_model_override(MODEL)
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 39b7ac42598..ef3db688089 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -7317,6 +7317,17 @@ def test_spec_draft_n_max_only_stored_for_mtp_modes():
assert "spec_draft_n_max" not in ngram
+def test_mlx_speculative_modes_and_drafter_survive_to_the_load():
+ stored = {"spec_draft_n_max": 4, "spec_draft_model": " d "}
+ for mode, is_gguf, kept in (
+ ("eagle3", False, {"spec_draft_n_max": 4, "spec_draft_model": "d"}),
+ ("eagle3", True, {}),
+ ):
+ entry = settings.normalize_model_override({"speculative_type": mode, **stored})
+ kwargs = settings.model_override_load_kwargs(entry, is_gguf = is_gguf)
+ assert kwargs == kept | ({} if is_gguf else {"speculative_type": mode})
+
+
def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers():
# Manual GPU memory with Auto layers hands the context to llama.cpp --fit, so
# the load sends the context pin (or 0), not the stored max seq length.
diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.py
index e2bae9045e3..be92e800514 100644
--- a/studio/backend/tests/test_parallel_slots_per_load.py
+++ b/studio/backend/tests/test_parallel_slots_per_load.py
@@ -615,4 +615,5 @@ def test_a_load_reply_carries_the_two_window_facts_and_the_width_that_selects_th
body["parallel_slots"],
) == (True, True, 4)
assert reply(can_batch = False, context_unbounded_when_batched = False)["parallel_slots"] == 1
+ assert reply(spec_draft_model = "org/drafter")["spec_draft_model"] == "org/drafter"
assert _InferenceRuntimeFields.model_fields["context_unbounded_when_batched"].default is False
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index ecfeb4c01c2..ad79892aa8d 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -336,7 +336,9 @@ def set_openai_auto_switch(
VALID_KV_CACHE_DTYPES = frozenset(
{"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"}
)
-VALID_SPECULATIVE_TYPES = frozenset(
+# The GGUF control never offers these, and a GGUF load drops them.
+MLX_ONLY_SPEC_TYPES = frozenset({"eagle3"})
+VALID_SPECULATIVE_TYPES = MLX_ONLY_SPEC_TYPES | frozenset(
{
"auto",
"mtp",
@@ -360,7 +362,9 @@ VALID_SPECULATIVE_TYPES = frozenset(
# Only these consume spec_draft_n_max (mirrors DRAFT_N_MAX_SPEC_TYPES in the UI).
DRAFT_N_MAX_SPEC_TYPES = frozenset(
{"mtp", "mtp+ngram", "draft-mtp", "dspark", "draft-dspark", "dflash", "draft-dflash"}
+ | MLX_ONLY_SPEC_TYPES
)
+DRAFTER_MODEL_SPEC_TYPES = DRAFT_N_MAX_SPEC_TYPES | {"auto", "default"}
# Only these load a separate draft model, and so a draft context for the dtype to apply to. Mirrors SEPARATE_DRAFT_MODEL_SPEC_TYPES in the UI.
SEPARATE_DRAFT_MODEL_SPEC_TYPES = frozenset({"dspark", "draft-dspark", "dflash", "draft-dflash"})
# Mirrors _LOAD_MODE_VALUES in llama_server_args.py. "auto" is the llama.cpp default and is not stored: an entry holding it would pin what a build may redefine.
@@ -470,6 +474,10 @@ def normalize_model_override(
spec_draft_n_max = _bounded_int(payload.get("spec_draft_n_max"), minimum = 1, maximum = 16)
if spec_draft_n_max:
entry["spec_draft_n_max"] = spec_draft_n_max
+ spec_draft_model = payload.get("spec_draft_model")
+ if speculative_type in DRAFTER_MODEL_SPEC_TYPES and isinstance(spec_draft_model, str):
+ if 0 < len(spec_draft_model.strip()) <= 1024:
+ entry["spec_draft_model"] = spec_draft_model.strip()
# Same rule, narrower set: the dtype needs a separate draft model, and only the sidecar modes always load one.
if speculative_type in SEPARATE_DRAFT_MODEL_SPEC_TYPES:
spec_draft_cache_type = _clean_str(
@@ -648,12 +656,19 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
)
override = {**override, "llama_extra_args": kept}
+ # MLX drafter settings; a GGUF load reads an MLX-only mode as no stored mode, keeping its launch flags.
+ gguf_drops = set()
+ if is_gguf:
+ gguf_drops.add("spec_draft_model")
+ if override.get("speculative_type") in MLX_ONLY_SPEC_TYPES:
+ gguf_drops.update(("speculative_type", "spec_draft_n_max"))
for source, target in (
("llama_extra_args", "llama_extra_args"),
("kv_cache_dtype", "cache_type_kv"),
("n_parallel", "n_parallel"),
("speculative_type", "speculative_type"),
("spec_draft_n_max", "spec_draft_n_max"),
+ ("spec_draft_model", "spec_draft_model"),
("reasoning_budget", "reasoning_budget"),
("reasoning_budget_message", "reasoning_budget_message"),
("tensor_parallel", "tensor_parallel"),
@@ -661,7 +676,7 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
("chat_template_override", "chat_template_override"),
("mlx_int8_prefill", "mlx_int8_prefill"),
):
- if override.get(source) is not None:
+ if override.get(source) is not None and source not in gguf_drops:
kwargs[target] = override[source]
mlx_kv_quant = _mlx_kv_quant_of(override)
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index a22c43abf51..aacb4b30ae5 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -48,6 +48,7 @@ import { loadManagedLlamaFlags } from "@/features/model-picker/api/llama-flags";
import { fetchLoadExtraArgs } from "@/features/model-picker/api/model-overrides";
import { sanitizeStoredExtraArgs } from "@/features/model-picker/model-config/llama-extra-args";
import { usePlatformStore } from "@/config/env";
+import { resolveSpeculativeType } from "@/lib/speculative-modes";
import { getSkillsSnapshot, settleSkillsForText } from "./skills-api";
@@ -2693,6 +2694,8 @@ const VISIBLE_MODEL_RUNTIME_KEYS = [
"specFallbackReason",
"specDraftNMax",
"loadedSpecDraftNMax",
+ "specDraftModel",
+ "loadedSpecDraftModel",
] as const satisfies readonly (keyof ChatRuntimeState)[];
type VisibleModelRuntimeState = Pick<
@@ -3657,10 +3660,25 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
config.customContextLength ?? null,
effectiveMaxSeqLength,
);
- const effectiveSpeculativeType =
- config.speculativeType ?? specSettings.speculativeType;
+ const candidateIsMlx = isServedByMlx(
+ candidate.kind === "gguf",
+ platform.deviceType,
+ platform.chatOnlyReason,
+ );
+ // MLX falls back to the standing preference: the live store may hold another model's settings.
+ const standingSpec = candidateIsMlx
+ ? resolveSpeculativeSettingsForLoad({ usePersistedPreference: true })
+ : specSettings;
+ const effectiveSpeculativeType = resolveSpeculativeType(
+ config.speculativeType,
+ standingSpec.speculativeType ?? "auto",
+ candidateIsMlx,
+ );
const effectiveSpecDraftNMax =
- config.specDraftNMax ?? specSettings.specDraftNMax;
+ config.specDraftNMax ?? standingSpec.specDraftNMax;
+ const mlxDrafter = candidateIsMlx
+ ? { spec_draft_model: config.specDraftModel ?? null }
+ : {};
const effectiveChatTemplateOverride = config.chatTemplateOverride?.trim()
? config.chatTemplateOverride
: null;
@@ -3735,6 +3753,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
mlx_int8_prefill: config.mlxInt8Prefill ?? false,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
+ ...mlxDrafter,
reasoning_budget:
candidate.kind === "gguf" && !isDiffusion ? config.reasoningBudget : -1,
reasoning_budget_message:
@@ -3773,8 +3792,8 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
options?.abortSignal?.throwIfAborted();
applyAutoLoadRuntimeState(options, () => {
// Persist the global preference only when the value came from global settings, or autoloading
- // a remembered model rewrites the default.
- if (config.speculativeType == null) {
+ // a remembered model rewrites the default. MLX only reads it.
+ if (config.speculativeType == null && !candidateIsMlx) {
saveSpeculativeType(effectiveSpeculativeType);
}
// Self-gates on is_gguf (skips diffusion), so persists only for a real GGUF load.
diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
index e8420da53a1..45b50113f17 100644
--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx
+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
@@ -128,6 +128,7 @@ import {
} from "./types/runtime";
import { effectiveMinPMode, isMinPMode } from "./lib/min-p-policy";
+import { mlxSpecFallbackMessage } from "./lib/mlx-spec-fallback";
export { defaultInferenceParams, type Preset } from "./presets/preset-policy";
export type { InferenceParams } from "./types/runtime";
@@ -642,13 +643,19 @@ export function ChatSettingsPanel({
// ngram-mod runs no drafter, so only the binary stand-down reaches it. Without
// this the panel shows ngram selected, no speculation running, and no reason.
speculativeType === "ngram");
+ const loadedIsMlx = useChatRuntimeStore((s) => s.loadedIsMlx);
+ const mlxSpecFallback =
+ !isExternalModel && loadedIsMlx && specFallbackReason != null
+ ? mlxSpecFallbackMessage(specFallbackReason, specDrafterKind)
+ : null;
const showContextVramWarning =
!isExternalModel &&
isGguf &&
maxContextLength != null &&
loadedEffectiveContext != null &&
loadedEffectiveContext > maxContextLength;
- const showLoadedDiagnostics = showSpecFallback || showContextVramWarning;
+ const showLoadedDiagnostics =
+ showSpecFallback || mlxSpecFallback != null || showContextVramWarning;
const hasModelContent = showLoadedDiagnostics;
const setActivePresetSource = useChatRuntimeStore(
(s) => s.setActivePresetSource,
@@ -1179,6 +1186,11 @@ export function ChatSettingsPanel({
)}
</div>
)}
+ {mlxSpecFallback && (
+ <div className="rounded-lg bg-amber-500/[0.08] px-3 py-2 text-ui-12 leading-[1.4] text-nav-fg/80">
+ <p>{mlxSpecFallback}</p>
+ </div>
+ )}
{showContextVramWarning && (
<p className="text-ui-11 text-amber-500">
{isUnifiedMemory ? (
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 9427ef5c1b7..85a3ae8d24e 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -21,6 +21,7 @@ import {
isSilencedDesktopUpdateFailure,
} from "@/lib/desktop-update-activity";
import { subscribeModelLifecycle } from "@/lib/model-lifecycle-events";
+import { resolveSpeculativeType } from "@/lib/speculative-modes";
import {
type TransferSample,
appendSample,
@@ -2245,6 +2246,7 @@ export function useChatModelRuntime() {
: stateBeforeUnload.speculativeType;
let loadSpecDraftNMax =
pendingLoadConfig?.specDraftNMax ?? stateBeforeUnload.specDraftNMax;
+ let loadSpecDraftModel = stateBeforeUnload.specDraftModel;
let loadNParallel =
pendingLoadConfig?.nParallel ?? stateBeforeUnload.nParallel;
let loadReasoningBudget =
@@ -2506,6 +2508,8 @@ export function useChatModelRuntime() {
loadedSpeculativeType: persistedSpeculativeType,
specDraftNMax: null,
loadedSpecDraftNMax: null,
+ specDraftModel: null,
+ loadedSpecDraftModel: null,
// Per-model too: a different model follows the server default unless its staged config overrides it.
nParallel: null,
loadedNParallel: null,
@@ -2577,6 +2581,14 @@ export function useChatModelRuntime() {
platform.deviceType,
platform.chatOnlyReason,
);
+ // Only a same-model reload without a staged config reuses the resident's settings; otherwise the
+ // drafter is this model's own and a standing mode goes through the resolver.
+ if (targetIsMlx && (pendingLoadConfig != null || switchingModelOrVariant)) {
+ loadSpecDraftModel = pendingLoadConfig?.specDraftModel ?? null;
+ if (loadSpeculativeType && pendingLoadConfig?.speculativeType == null) {
+ loadSpeculativeType = resolveSpeculativeType(null, loadSpeculativeType, true);
+ }
+ }
const explicitCtxPin = loadRequestContextPin(
loadCustomContextLength,
targetIsMlx,
@@ -2658,6 +2670,7 @@ export function useChatModelRuntime() {
mlx_int8_prefill: loadMlxInt8Prefill,
speculative_type: loadSpeculativeType,
spec_draft_n_max: loadSpecDraftNMax,
+ ...(targetIsMlx ? { spec_draft_model: loadSpecDraftModel } : {}),
n_parallel: loadNParallel,
reasoning_budget:
isGguf && !targetIsDiffusion ? loadReasoningBudget : -1,
@@ -2707,8 +2720,9 @@ export function useChatModelRuntime() {
// The load applied this spec mode, so persist the user's standing preference now: the requested
// intent, not the resolved echo, since saveSpeculativeType keeps only the universal auto/ngram/off.
- // Skipped for a per-model config (keepSpeculative), whose choice must not overwrite the global.
- if (!keepSpeculative) {
+ // Skipped for a per-model config (keepSpeculative), whose choice must not overwrite the global,
+ // and for MLX, which only reads it.
+ if (!(keepSpeculative || targetIsMlx)) {
saveSpeculativeType(loadSpeculativeType);
}
// Persist the GPU Memory mode only on a successful load, so an abandoned selection does not stick.
@@ -2884,6 +2898,8 @@ export function useChatModelRuntime() {
loadedSpeculativeType: loadedSpec,
specDraftNMax: loadResponse.spec_draft_n_max ?? null,
loadedSpecDraftNMax: loadResponse.spec_draft_n_max ?? null,
+ specDraftModel: loadResponse.spec_draft_model ?? null,
+ loadedSpecDraftModel: loadResponse.spec_draft_model ?? null,
// Keep the click-time value: the echo is the resolved count, and adopting it would pin a blank
// "server default" control.
nParallel: committedSlots,
@@ -3053,6 +3069,7 @@ export function useChatModelRuntime() {
rollbackState.loadedSpeculativeType,
spec_draft_n_max:
rollbackState.loadedSpecDraftNMax,
+ spec_draft_model: rollbackState.loadedSpecDraftModel,
n_parallel: rollbackState.loadedNParallel,
reasoning_budget:
rollbackState.loadedReasoningBudgetRequested ?? -1,
@@ -3112,6 +3129,7 @@ export function useChatModelRuntime() {
// from its reload echo.
speculativeType: rollbackState.loadedSpeculativeType ?? null,
specDraftNMax: rollbackState.loadedSpecDraftNMax ?? null,
+ specDraftModel: rollbackState.loadedSpecDraftModel ?? null,
// Control keeps its intent; only the baseline takes the echo.
nParallel: previousNParallel,
loadedNParallel: rollbackState.loadedNParallel ?? null,
@@ -3149,6 +3167,7 @@ export function useChatModelRuntime() {
loadedSpeculativeType: rollbackSpeculativeType,
loadedSpecDraftNMax:
rollbackResponse.spec_draft_n_max ?? null,
+ loadedSpecDraftModel: rollbackResponse.spec_draft_model ?? null,
loadedKvCacheDtype: rollbackResponse.cache_type_kv ?? null,
...mlxRuntimeStateFrom(rollbackResponse),
// After the spread, which seeds the control from the echo; the control keeps its intent, like
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index 03dd97b25b3..48bdb82937a 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -496,6 +496,14 @@ export function applyActiveModelStatusToStore(
specDraftNMax: status.spec_draft_n_max ?? null,
}),
}),
+ ...(seedLoadParams &&
+ status.spec_draft_model !== undefined && {
+ loadedSpecDraftModel: status.spec_draft_model ?? null,
+ ...((hydratingExistingModel ||
+ prevState.specDraftModel === prevState.loadedSpecDraftModel) && {
+ specDraftModel: status.spec_draft_model ?? null,
+ }),
+ }),
...(seedLoadParams &&
status.cache_type_kv !== undefined && {
loadedKvCacheDtype: status.cache_type_kv,
diff --git a/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts
new file mode 100644
index 00000000000..c8ecfb4dee7
--- /dev/null
+++ b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+const MLX_DRAFTER_LABELS: Record<string, string> = {
+ mtp: "MTP",
+ dflash: "DFlash",
+ dspark: "DSpark",
+ eagle3: "EAGLE-3",
+};
+
+/** Copy for an MLX load's spec_fallback_reason, or null for a code MLX does not report. Several
+ * backend paths share each code, so the copy states only what every one of them implies, and the
+ * outcome follows what runs: the reason can name a drafter passed over while another attached. */
+export function mlxSpecFallbackMessage(
+ reason: string,
+ drafterKind: string | null,
+): string | null {
+ const outcome =
+ drafterKind == null
+ ? "so it is running without speculative decoding"
+ : drafterKind === "ngram"
+ ? "so it is drafting with n-gram copies only"
+ : `so it is drafting with ${MLX_DRAFTER_LABELS[drafterKind] ?? drafterKind} instead`;
+ switch (reason) {
+ case "drafter_not_found":
+ return `A drafter this load asked for was not found locally, or is not a drafter, ${outcome}. Download it, or check the drafter named in this model's settings, then reload.`;
+ case "drafter_incompatible":
+ return `A drafter was found but could not be used with this model or mode, ${outcome}. Try a drafter made for this model and mode, then reload.`;
+ case "drafter_no_memory":
+ return `The drafter could not be confirmed to fit in memory beside this model, ${outcome}. When memory is the limit, a shorter context length or a smaller drafter can help.`;
+ case "runtime_error":
+ return `Speculative decoding could not start for this load, ${outcome}.`;
+ case "kv_quant":
+ return "Speculative decoding cannot run with KV cache quantization, which this load asked for. Setting KV Cache Dtype to Auto in this model's settings removes that obstacle.";
+ case "auto_context_cost":
+ return `Auto left out a drafter that does not fit beside this model at its full context length, ${outcome}. To try it anyway, choose its mode in this model's settings, with a shorter or automatic context length.`;
+ default:
+ return null;
+ }
+}
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index 2834e824b80..d5bb60b4214 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -11,6 +11,10 @@ import {
} from "./llama-extra-args-normalize";
import { reconcileTensorSplit } from "@/hooks/gpu-tensor-split";
import type { GpuIndexKind } from "@/hooks/gpu-selection";
+import {
+ mlxSpeculativeMode,
+ resolveSpeculativeType,
+} from "@/lib/speculative-modes";
import type { InferenceStatusResponse } from "../types/api";
@@ -27,6 +31,7 @@ type ResidentRuntime = Pick<
| "mlx_int8_prefill_requested"
| "speculative_type"
| "spec_draft_n_max"
+ | "spec_draft_model"
| "requested_parallel_slots"
| "requested_n_batch"
| "requested_n_ubatch"
@@ -48,6 +53,7 @@ type ResidentRuntime = Pick<
| "requested_gpu_ids"
| "gpu_ids"
| "is_gguf"
+ | "is_mlx"
| "is_diffusion"
| "diffusion_requested_ngl"
| "diffusion_split_supported"
@@ -182,12 +188,17 @@ export function residentSpeculativeNeedsRepair(
| "spec_dflash_retry_pending"
| "spec_dspark_sidecar_absent"
| "spec_drafter_kind"
+ | "is_mlx"
>,
resolvedSpeculativeType: string | null,
/** Whether the load carries a `gguf_path`, which the route sets from the identifier alone:
* `source.gguf_path if model_identifier.lower().endswith(".gguf") else None`. */
sendsGgufPath = false,
): boolean {
+ // Every arm below is llama.cpp's; an identical MLX load dedupes.
+ if (status.is_mlx === true) {
+ return false;
+ }
const mode = resolvedSpeculativeType ?? "auto";
// Two arms that record no fallback reason, so the reason check below cannot see them. The probe
// arm is not gated on a mode; the DFlash one is, as the backend gates it.
@@ -279,14 +290,28 @@ const SETTING_CHECKS: SettingCheck[] = [
{
// Always pinned: an unset mode resolves to the standing preference and the load sends it. Reading
// it as silence let a pick asking for "off" adopt a resident MTP runtime.
+ mlxComparable: true,
pinned: () => true,
- agrees: (c, s, standing) =>
- (standing.normalizeSpeculative(c.speculativeType) ??
- standing.speculativeType) ===
- (standing.normalizeSpeculative(s.speculative_type) ??
- standing.speculativeType),
+ agrees: (c, s, standing) => {
+ const unset = resolveSpeculativeType(
+ null,
+ standing.speculativeType ?? "auto",
+ s.is_mlx === true,
+ );
+ const mode = (value: string | null | undefined) => {
+ const resolved = standing.normalizeSpeculative(value) ?? unset;
+ return s.is_mlx === true ? mlxSpeculativeMode(resolved) : resolved;
+ };
+ return mode(c.speculativeType) === mode(s.speculative_type);
+ },
+ },
+ {
+ mlxComparable: true,
+ pinned: () => true,
+ agrees: (c, s) => (c.specDraftModel ?? null) === (s.spec_draft_model ?? null),
},
{
+ mlxComparable: true,
// Pinned like the rest: _runtime_matches_intent reloads for the null-against-explicit
// flip, so an unset limit asks for the default. No `draft_depth_matters` gate here,
// as the status carries a count only when a depth-consuming load recorded an override.
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 0f4a7dd7eef..0b6fddb9432 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -44,7 +44,10 @@ import {
import { NonModalDropdownMenu } from "@/components/ui/non-modal-dropdown-menu";
import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params";
import { FIND_SKIP_ATTRIBUTE } from "@/features/find-in-page";
-import { DRAFT_N_MAX_SPEC_TYPES } from "@/lib/speculative-modes";
+import {
+ DRAFT_N_MAX_SPEC_TYPES,
+ resolveSpeculativeType,
+} from "@/lib/speculative-modes";
import {
StudioDictationAdapter,
isStudioDictationAvailable,
@@ -1622,8 +1625,16 @@ export function SharedComposer({
const effectiveChatTemplateOverride = cleanCompareChatTemplate(
ownConfig.chatTemplateOverride,
);
- const effectiveSpeculativeType =
- ownConfig.speculativeType ?? specSettings.speculativeType;
+ const targetIsMlx = isServedByMlx(
+ targetIsGguf,
+ platform.deviceType,
+ platform.chatOnlyReason,
+ );
+ const effectiveSpeculativeType = resolveSpeculativeType(
+ ownConfig.speculativeType,
+ specSettings.speculativeType ?? "auto",
+ targetIsMlx,
+ );
const effectiveSpecDraftNMax = ownRemembered
? resolveCompareSpecDraftNMax(
effectiveSpeculativeType,
@@ -1823,6 +1834,7 @@ export function SharedComposer({
mlx_int8_prefill: ownConfig.mlxInt8Prefill ?? false,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
+ ...(targetIsMlx ? { spec_draft_model: ownConfig.specDraftModel ?? null } : {}),
reasoning_budget:
targetIsGguf && !resolvedIsDiffusion
? ownConfig.reasoningBudget
@@ -1868,8 +1880,8 @@ export function SharedComposer({
compareRunsRef.current.setLoadingModel(run, null);
throwIfCompareCancelled(compareSignal);
// Keep a compare pane's per-model speculative choice load-local: persist the global preference
- // only when it came from global settings.
- if (ownConfig.speculativeType == null) {
+ // only when it came from global settings, and never from MLX, which only reads it.
+ if (ownConfig.speculativeType == null && !targetIsMlx) {
saveSpeculativeType(effectiveSpeculativeType);
}
// Persist the GPU Memory mode on a non-diffusion GGUF compare-load too, so an applied manual
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 1ba870104b0..06a3d99e575 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -1957,6 +1957,7 @@ export function normalizeSpeculativeType(
return "ngram";
}
if (s === "mtp+ngram") return "mtp+ngram";
+ if (s === "eagle3") return s;
// Comma-chained legacy values (e.g. from older backend echoes).
const parts = s
.split(",")
@@ -1976,21 +1977,27 @@ export function normalizeSpeculativeType(
export function resolveLoadedSpeculativeSettings(response: {
speculative_type?: string | null;
spec_draft_n_max?: number | null;
+ spec_draft_model?: string | null;
}): {
speculativeType: string | null;
loadedSpeculativeType: string | null;
specDraftNMax: number | null;
loadedSpecDraftNMax: number | null;
+ specDraftModel: string | null;
+ loadedSpecDraftModel: string | null;
} {
const loadedSpeculativeType = normalizeSpeculativeType(
response.speculative_type,
);
const loadedSpecDraftNMax = response.spec_draft_n_max ?? null;
+ const loadedSpecDraftModel = response.spec_draft_model ?? null;
return {
speculativeType: loadedSpeculativeType,
loadedSpeculativeType,
specDraftNMax: loadedSpecDraftNMax,
loadedSpecDraftNMax,
+ specDraftModel: loadedSpecDraftModel,
+ loadedSpecDraftModel,
};
}
@@ -2462,6 +2469,9 @@ type ChatRuntimeStore = {
/** User --spec-draft-n-max override (null = platform default). */
specDraftNMax: number | null;
loadedSpecDraftNMax: number | null;
+ /** MLX companion drafter, a repo id or local path (null = discovery). */
+ specDraftModel: string | null;
+ loadedSpecDraftModel: string | null;
/** User reply-width override, for either backend (null = server default). GGUF spends it
* on llama-server's --parallel slots; MLX on how many replies decode together.
* Never re-seeded from an echo: the resolved count would pin a blank control. */
@@ -4244,6 +4254,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
specDrafterKind: null,
specDraftNMax: null,
loadedSpecDraftNMax: null,
+ specDraftModel: null,
+ loadedSpecDraftModel: null,
nParallel: null,
loadedNParallel: null,
reasoningBudget: -1,
@@ -5193,6 +5205,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
specDrafterKind: null,
specDraftNMax: null,
loadedSpecDraftNMax: null,
+ specDraftModel: null,
+ loadedSpecDraftModel: null,
nParallel: null,
loadedNParallel: null,
reasoningBudget: -1,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 6d001e17b9a..da4f0fe96c1 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -81,6 +81,8 @@ export interface LoadModelRequest {
/** Override --spec-draft-n-max for drafter speculative decoding. Applied only when speculative_type
* resolves to "mtp", "mtp+ngram", "dspark" or "dflash". */
spec_draft_n_max?: number | null;
+ /** MLX only: a companion drafter, a local directory or an already-cached repo id. */
+ spec_draft_model?: string | null;
/** Parallel decode slots for llama-server (--parallel), 1..64. Omit/null = the launch default. The
* VRAM fitter may launch fewer to stay on GPU. */
n_parallel?: number | null;
@@ -289,6 +291,7 @@ export interface LoadModelResponse {
/** Canonical UI-facing mode the load request resolved to. See LoadModelRequest. */
speculative_type?: string | null;
spec_draft_n_max?: number | null;
+ spec_draft_model?: string | null;
/** Whether tensor-parallel split (--split-mode tensor) is active. */
tensor_parallel?: boolean;
/** The load ran with the vision projector deliberately left unloaded. Echoes the request, so it
@@ -434,6 +437,7 @@ export interface InferenceStatusResponse {
/** Canonical UI-facing mode currently active. See LoadModelRequest. */
speculative_type?: string | null;
spec_draft_n_max?: number | null;
+ spec_draft_model?: string | null;
/** Whether tensor-parallel split (--split-mode tensor) is active. */
tensor_parallel?: boolean;
/** The load ran with the vision projector deliberately left unloaded. Echoes the request, so it
diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
index b1392953847..f3b823a30af 100644
--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
@@ -81,6 +81,7 @@ export interface MemoryEstimateRequest {
ctxCheckpoints?: number | null;
speculativeType?: string | null;
specDraftNMax?: number | null;
+ specDraftModel?: string | null;
specDraftCacheType?: string | null;
tensorParallel?: boolean;
disableVision?: boolean;
@@ -166,6 +167,7 @@ function estimateRequestBody(
ctx_checkpoints: payload.ctxCheckpoints ?? null,
speculative_type: payload.speculativeType ?? null,
spec_draft_n_max: payload.specDraftNMax ?? null,
+ spec_draft_model: payload.specDraftModel ?? null,
spec_draft_cache_type: payload.specDraftCacheType ?? null,
tensor_parallel: payload.tensorParallel ?? false,
disable_vision: payload.disableVision ?? false,
diff --git a/studio/frontend/src/features/model-picker/api/mlx-drafters.ts b/studio/frontend/src/features/model-picker/api/mlx-drafters.ts
new file mode 100644
index 00000000000..a587b45c49a
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/api/mlx-drafters.ts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { authFetch } from "@/features/auth";
+import type { MlxDrafter } from "@/lib/speculative-modes";
+
+/** Cached drafters an MLX load of `modelPath` could name; empty when the backend cannot say. */
+export async function fetchMlxDrafters(
+ modelPath: string,
+ signal?: AbortSignal,
+): Promise<MlxDrafter[]> {
+ const response = await authFetch("/api/inference/mlx-drafters", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ signal,
+ body: JSON.stringify({ model_path: modelPath }),
+ });
+ if (!response.ok) return [];
+ const body = (await response.json()) as { drafters?: unknown };
+ return Array.isArray(body.drafters)
+ ? body.drafters
+ .filter((d) => typeof d?.repo_id === "string" && typeof d?.kind === "string")
+ .map((d) => ({ repo: d.repo_id, kind: d.kind, named: d.named !== false }))
+ : [];
+}
diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts
index e189f93c732..0c6c69bd01c 100644
--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts
+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts
@@ -47,6 +47,8 @@ export interface ApiModelOverride {
// biome-ignore lint/style/useNamingConvention: API schema
spec_draft_n_max?: number;
// biome-ignore lint/style/useNamingConvention: API schema
+ spec_draft_model?: string;
+ // biome-ignore lint/style/useNamingConvention: API schema
n_parallel?: number;
// biome-ignore lint/style/useNamingConvention: API schema
reasoning_budget?: number;
@@ -347,6 +349,7 @@ export function fromApiOverride(
: (local.mlxKvQuant ?? null),
speculativeType: override.speculative_type ?? local.speculativeType,
specDraftNMax: override.spec_draft_n_max ?? local.specDraftNMax,
+ specDraftModel: override.spec_draft_model ?? local.specDraftModel,
specDraftCacheDtype:
override.spec_draft_cache_type ?? local.specDraftCacheDtype,
nParallel: override.n_parallel ?? local.nParallel,
@@ -417,6 +420,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
if (config.specDraftNMax && config.specDraftNMax > 0) {
payload.spec_draft_n_max = config.specDraftNMax;
}
+ if (config.specDraftModel) {
+ payload.spec_draft_model = config.specDraftModel;
+ }
// Blank follows the server-wide --parallel default, which is the app default here.
if (config.nParallel && config.nParallel > 0) {
payload.n_parallel = config.nParallel;
@@ -568,6 +574,8 @@ async function sendModelOverride(
// can still predate.
// biome-ignore lint/style/useNamingConvention: API schema
mirrors_reasoning_budget: true,
+ // biome-ignore lint/style/useNamingConvention: API schema
+ mirrors_spec_draft_model: true,
// Only sent when set, so an older backend is not handed an unknown key every save.
...(options?.fillAbsentFields
? // biome-ignore lint/style/useNamingConvention: API schema
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 77fc9801c5e..bc4e39c9060 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -67,6 +67,14 @@ import {
resolveMemoryCapacityGb,
} from "@/hooks/gpu-vram";
import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
+import {
+ DRAFTER_MODEL_SPEC_TYPES,
+ MLX_SPECULATIVE_TYPES,
+ type MlxDrafter,
+ mlxDrafterChoices,
+ mlxSpeculativeMode,
+ resolveSpeculativeType,
+} from "@/lib/speculative-modes";
import { toast } from "@/lib/toast";
import {
type ReactNode,
@@ -94,6 +102,7 @@ import {
subscribeLlamaFlagCatalog,
} from "../api/llama-flags";
import { type MemoryEstimate } from "../api/memory-estimate";
+import { fetchMlxDrafters } from "../api/mlx-drafters";
import {
resolveEstimateContext,
resolveMlxEstimateContext,
@@ -186,10 +195,12 @@ import {
normalizeMaxSeqLength,
normalizePerModelConfig,
perModelConfigStorageChanged,
+ pinSpeculativeMode,
readAdvancedSettingsOpen,
resolveInitialConfig,
saveAdvancedSettingsOpen,
savePerModelConfig,
+ storedSpeculativeAuto,
subscribeAdvancedSettingsOpen,
VRAM_BUDGET_PERCENT_STEP,
vramFractionToPercent,
@@ -255,6 +266,18 @@ const SPECULATIVE_TYPE_LABELS: Record<
"mtp+ngram": "MTP+Ngram",
off: "Off",
};
+const MLX_SPECULATIVE_TYPE_LABELS: Record<
+ (typeof MLX_SPECULATIVE_TYPES)[number],
+ string
+> = {
+ auto: "Auto",
+ mtp: "MTP",
+ dflash: "DFlash",
+ dspark: "DSpark",
+ eagle3: "EAGLE-3",
+ ngram: "Ngram",
+ off: "Off",
+};
// Lower-case where the value is the flag's own spelling, so the pick reads the same as the launch command.
const LOAD_MODE_LABELS: Record<(typeof LOAD_MODES)[number], string> = {
@@ -314,6 +337,7 @@ function hasNonDefaultAdvanced(config: PerModelConfig): boolean {
(config.speculativeType ?? "auto") !== "auto" ||
config.specDraftNMax != null ||
config.specDraftCacheDtype != null ||
+ config.specDraftModel != null ||
config.nParallel != null ||
config.reasoningBudget !== -1 ||
config.reasoningBudgetMessage !== "" ||
@@ -1181,6 +1205,157 @@ function ParallelSlotsRow({
);
}
+/** Unset shows the mode the load would send. */
+function MlxSpeculativeRows({
+ config,
+ update,
+ speculativeFallback,
+ modelPath,
+}: {
+ config: PerModelConfig;
+ update: (patch: Partial<PerModelConfig>) => void;
+ speculativeFallback: string;
+ modelPath: string;
+}) {
+ const [drafters, setDrafters] = useState<MlxDrafter[]>([]);
+ useEffect(() => {
+ const controller = new AbortController();
+ setDrafters([]);
+ fetchMlxDrafters(modelPath, controller.signal)
+ .then((found) => !controller.signal.aborted && setDrafters(found))
+ .catch(() => {});
+ return () => controller.abort();
+ }, [modelPath]);
+ const mode = mlxSpeculativeMode(
+ config.speculativeType ??
+ resolveSpeculativeType(null, speculativeFallback, true),
+ );
+ return (
+ <>
+ <div className={ROW_CLASS}>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <span className={LABEL_CLASS_WRAP}>Speculative Decoding</span>
+ <InfoHint>
+ Faster generation. Auto uses the first cached drafter (an MTP head
+ or assistant, then DFlash2, DFlash, DSpark, EAGLE-3), tunes the
+ draft length to this machine and drafts only while that is faster.
+ Choose a kind to force it. Every drafter also copies repeated text;
+ Ngram copies without one. Drafters are read from the local Hugging
+ Face cache, never downloaded.
+ </InfoHint>
+ </div>
+ <Select
+ value={mode}
+ onValueChange={(v) =>
+ update({
+ speculativeType: v,
+ specDraftNMax: DRAFT_N_MAX_SPEC_TYPES.has(v)
+ ? config.specDraftNMax
+ : null,
+ specDraftModel: DRAFTER_MODEL_SPEC_TYPES.has(v)
+ ? config.specDraftModel
+ : null,
+ })
+ }
+ >
+ <SelectTrigger
+ animateRadius={false}
+ icon={ChevronDownStandardIcon}
+ iconClassName="size-3.5"
+ className={SELECT_TRIGGER_CLASS}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
+ {MLX_SPECULATIVE_TYPES.map((type) => (
+ <SelectItem key={type} value={type}>
+ {MLX_SPECULATIVE_TYPE_LABELS[type]}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ {DRAFT_N_MAX_SPEC_TYPES.has(mode) && (
+ <div className={ROW_CLASS}>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <span className={LABEL_CLASS}>Draft Tokens</span>
+ <InfoHint>
+ Tokens per draft: up to the drafter's depth (3 for MTP heads),
+ every step drafts exactly this many, even when plain decoding would
+ be faster (a copy of repeated text replaces a draft when it should
+ yield more). Above that depth, or for Ngram, it is a
+ ceiling and Unsloth decodes plainly when drafting would be slower.
+ Leave blank to let Unsloth tune it for this machine.
+ </InfoHint>
+ </div>
+ <input
+ type="number"
+ min={1}
+ max={16}
+ step={1}
+ value={config.specDraftNMax ?? ""}
+ placeholder="auto"
+ onChange={(event) => {
+ const parsed = Number.parseInt(event.target.value, 10);
+ update(
+ pinSpeculativeMode(config, mode, {
+ specDraftNMax: Number.isFinite(parsed)
+ ? Math.max(1, Math.min(16, parsed))
+ : null,
+ }),
+ );
+ }}
+ aria-label="Speculative decoding draft tokens"
+ className={NUMBER_INPUT_CLASS}
+ />
+ </div>
+ )}
+ {DRAFTER_MODEL_SPEC_TYPES.has(mode) && (
+ <div className={ROW_CLASS}>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <span className={LABEL_CLASS}>Drafter</span>
+ <InfoHint>
+ A drafter from the local Hugging Face cache that fits this
+ model. Auto uses the model's own head or the first cached drafter
+ named for it.
+ </InfoHint>
+ </div>
+ <Select
+ value={config.specDraftModel ?? "auto"}
+ onValueChange={(v) =>
+ update(
+ pinSpeculativeMode(config, mode, {
+ specDraftModel: v === "auto" ? null : v,
+ }),
+ )
+ }
+ >
+ <SelectTrigger
+ animateRadius={false}
+ icon={ChevronDownStandardIcon}
+ iconClassName="size-3.5"
+ aria-label="Speculative decoding drafter"
+ className={SELECT_TRIGGER_CLASS}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
+ <SelectItem value="auto">Auto</SelectItem>
+ {mlxDrafterChoices(drafters, mode, config.specDraftModel ?? null).map(
+ ([repo, label]) => (
+ <SelectItem key={repo} value={repo}>
+ {label}
+ </SelectItem>
+ ),
+ )}
+ </SelectContent>
+ </Select>
+ </div>
+ )}
+ </>
+ );
+}
+
function MlxAdvancedSettings({
config,
update,
@@ -1188,6 +1363,8 @@ function MlxAdvancedSettings({
servedByMlx,
int8PrefillAvailable,
onInt8PrefillChange,
+ speculativeFallback,
+ modelPath,
onEditTemplate,
templateOutcome,
}: {
@@ -1199,6 +1376,8 @@ function MlxAdvancedSettings({
servedByMlx: boolean;
int8PrefillAvailable: boolean;
onInt8PrefillChange: (checked: boolean) => void;
+ speculativeFallback: string;
+ modelPath: string;
onEditTemplate: () => void;
/** Why the loaded model could not take the override it was given. */
templateOutcome: string | null;
@@ -1268,6 +1447,14 @@ function MlxAdvancedSettings({
/>
</div>
)}
+ {servedByMlx && (
+ <MlxSpeculativeRows
+ config={config}
+ update={update}
+ speculativeFallback={speculativeFallback}
+ modelPath={modelPath}
+ />
+ )}
{servedByMlx && (
<ParallelSlotsRow config={config} update={update} hint={MLX_PARALLEL_HINT} />
)}
@@ -2646,7 +2833,7 @@ export function ModelConfigPage({
const hydrationSaved = savePerModelConfig(
configId,
target.ggufVariant,
- rememberedConfig,
+ storedSpeculativeAuto(rememberedConfig, !target.isGguf),
hydrationEvicted,
);
setSavedRemember(hydrationSaved);
@@ -2927,8 +3114,13 @@ export function ModelConfigPage({
// override sends null, which the backend reads as Auto, while the selector has been showing
// the global fallback. With the global Off and an 11 GB DSpark sidecar in the repo, the row
// charged the sidecar for a load that disables it.
- speculativeType: runtimeConfig.speculativeType ?? speculativeFallback ?? null,
+ speculativeType: resolveSpeculativeType(
+ runtimeConfig.speculativeType,
+ speculativeFallback,
+ targetIsMlx,
+ ),
specDraftNMax: runtimeConfig.specDraftNMax,
+ specDraftModel: targetIsMlx ? (runtimeConfig.specDraftModel ?? null) : null,
specDraftCacheType: runtimeConfig.specDraftCacheDtype ?? null,
tensorParallel: runtimeConfig.tensorParallel,
disableVision: runtimeConfig.disableVision,
@@ -3229,7 +3421,7 @@ export function ModelConfigPage({
const persistConfig = (next: PerModelConfig) => {
// Judge what storage keeps: savePerModelConfig normalizes first, so the raw object over-reports.
- const normalized = normalizePerModelConfig(next);
+ const normalized = normalizePerModelConfig(storedSpeculativeAuto(next, targetIsMlx));
const evicted: { modelId: string; ggufVariant: string | null }[] = [];
const saved = remember
? savePerModelConfig(configId, target.ggufVariant, normalized, evicted)
@@ -3610,6 +3802,8 @@ export function ModelConfigPage({
? setInt8PrefillConfirmOpen(true)
: update({ mlxInt8Prefill: false })
}
+ speculativeFallback={speculativeFallback}
+ modelPath={target.id}
onEditTemplate={() => setTemplateOpen(true)}
templateOutcome={chatTemplateOutcome}
/>
diff --git a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
index 532dfb94f95..96915398db7 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
@@ -44,6 +44,7 @@ function estimateKey(request: MemoryEstimateRequest | null): string | null {
request.ctxCheckpoints ?? null,
request.speculativeType ?? null,
request.specDraftNMax ?? null,
+ request.specDraftModel ?? null,
request.specDraftCacheType ?? null,
request.tensorParallel ?? false,
request.disableVision ?? false,
diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
index f13223e8e61..f9d9cba8e59 100644
--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
@@ -65,6 +65,7 @@ export function applyPerModelConfigToRuntime(
readPersistedSpeculativeType(),
specDraftNMax: config.specDraftNMax ?? null,
specDraftCacheDtype: config.specDraftCacheDtype ?? null,
+ specDraftModel: config.specDraftModel ?? null,
nParallel: config.nParallel ?? null,
reasoningBudget: options.isDiffusion ? -1 : config.reasoningBudget,
reasoningBudgetMessage: options.isDiffusion
@@ -133,6 +134,7 @@ export function currentRuntimePerModelConfig(
speculativeType: normalizeSpeculativeType(s.speculativeType),
specDraftNMax: s.specDraftNMax ?? null,
specDraftCacheDtype: s.specDraftCacheDtype ?? null,
+ specDraftModel: s.specDraftModel ?? null,
nParallel: s.nParallel ?? null,
reasoningBudget:
s.reasoningBudget === s.loadedReasoningBudget
@@ -184,6 +186,7 @@ export function perModelConfigsEqual(
speculative(a.speculativeType) === speculative(b.speculativeType) &&
(a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) &&
(a.specDraftCacheDtype ?? null) === (b.specDraftCacheDtype ?? null) &&
+ (a.specDraftModel ?? null) === (b.specDraftModel ?? null) &&
(a.nParallel ?? null) === (b.nParallel ?? null) &&
a.reasoningBudget === b.reasoningBudget &&
a.reasoningBudgetMessage === b.reasoningBudgetMessage &&
diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
index c9f86156b76..2b7645fe82a 100644
--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts
+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
@@ -59,6 +59,7 @@ export function loadedConfigSignature(
config.speculativeType ?? "",
config.specDraftNMax ?? "",
config.specDraftCacheDtype ?? "",
+ config.specDraftModel ?? "",
config.nParallel ?? "",
config.reasoningBudget ?? "",
`${(config.reasoningBudgetMessage ?? "").length}:${hashString(config.reasoningBudgetMessage ?? "")}`,
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 63ff51aea6a..dcea694b9b6 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -16,6 +16,8 @@ import {
import { isExternalModelId } from "@/features/chat/external-providers";
import {
DRAFT_N_MAX_SPEC_TYPES,
+ DRAFTER_MODEL_SPEC_TYPES,
+ MLX_ONLY_SPEC_TYPES,
SEPARATE_DRAFT_MODEL_SPEC_TYPES,
} from "@/lib/speculative-modes";
@@ -34,6 +36,8 @@ export interface PerModelConfig {
/** KV cache dtype for the DRAFT context, sized and quantized independently of kvCacheDtype.
* Optional so older blobs parse. */
specDraftCacheDtype?: string | null;
+ /** MLX companion drafter, a repo id or local path. Optional so older blobs parse. */
+ specDraftModel?: string | null;
nParallel: number | null;
reasoningBudget: number;
reasoningBudgetMessage: string;
@@ -78,6 +82,7 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {
speculativeType: null,
specDraftNMax: null,
specDraftCacheDtype: null,
+ specDraftModel: null,
nParallel: null,
reasoningBudget: -1,
reasoningBudgetMessage: "",
@@ -390,9 +395,10 @@ const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";
// would normalize the unknown field straight back out of the record.
// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group
// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,
-// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split. v8 is skipped: nightly builds stamped it for the reverted custom
+// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split, v11 the MLX drafter. v8 is skipped: nightly builds stamped it for the reverted custom
// llama.cpp config (#12725), so a v8 client must not claim to understand an int8 prefill record.
-const STORAGE_SCHEMA_VERSION = 10;
+const STORAGE_SCHEMA_VERSION = 11;
+const PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;
const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;
const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;
const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;
@@ -440,6 +446,7 @@ const STORED_CONFIG_FIELDS = new Set([
"speculativeType",
"specDraftNMax",
"specDraftCacheDtype",
+ "specDraftModel",
"nParallel",
"reasoningBudget",
"reasoningBudgetMessage",
@@ -530,9 +537,9 @@ function canonicalizeSpeculativeType(value: string): string | null {
if (!s) {
return null;
}
- // "auto"/"default" is the follow-global sentinel; store as null so it is never an override.
+ // Kept: an MLX model's explicit Auto beats a standing "off". GGUF saves fold it via storedSpeculativeAuto.
if (s === "auto" || s === "default") {
- return null;
+ return "auto";
}
// _LEGACY_SPEC_MODE_MAP's four. A stored override arrives raw, and the null below
// is "follow the global preference", which would enable a drafter under Auto.
@@ -551,12 +558,35 @@ function canonicalizeSpeculativeType(value: string): string | null {
if (s === "ngram" || s === "ngram-mod" || s === "ngram-simple") {
return "ngram";
}
- if (s === "mtp+ngram") {
- return "mtp+ngram";
+ if (s === "mtp+ngram" || (MLX_ONLY_SPEC_TYPES as readonly string[]).includes(s)) {
+ return s;
}
return null;
}
+/** The config as storage keeps it: GGUF reads Auto as "follow the standing preference", spelled null,
+ * while an MLX model stores its explicit Auto. */
+export function storedSpeculativeAuto<T extends Pick<PerModelConfig, "speculativeType">>(
+ config: T,
+ isMlx: boolean,
+): T {
+ return !isMlx && config.speculativeType === "auto"
+ ? { ...config, speculativeType: null }
+ : config;
+}
+
+/** An edit to draft tokens or the drafter. Made while the mode is unset, it pins the shown mode:
+ * storage keeps either value only beside a mode that reads it. */
+export function pinSpeculativeMode(
+ config: Pick<PerModelConfig, "speculativeType">,
+ shownMode: string,
+ patch: Partial<PerModelConfig>,
+): Partial<PerModelConfig> {
+ return config.speculativeType == null
+ ? { ...patch, speculativeType: shownMode }
+ : patch;
+}
+
/** Canonicalize a stored --load-mode, or null to follow the llama.cpp default. "auto" folds
* to null: it IS the default, so storing it would pin a value the build may redefine. */
export function canonicalizeLoadMode(value: unknown): string | null {
@@ -835,7 +865,8 @@ function parseLegacyModelKey(
}
function legacyEntryToConfig(raw: Record<string, unknown>): PerModelConfig {
- return normalizeV1({
+ // Written before any MLX control, so its Auto is GGUF's.
+ return storedSpeculativeAuto(normalizeV1({
customContextLength:
typeof raw.contextLength === "number" ? raw.contextLength : null,
maxSeqLength: null,
@@ -870,7 +901,7 @@ function legacyEntryToConfig(raw: Record<string, unknown>): PerModelConfig {
: Array.isArray(raw.selectedGpuIds)
? (raw.selectedGpuIds as number[])
: undefined,
- });
+ }), false);
}
function mergeLegacyEntries(
@@ -1028,6 +1059,14 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
VALID_KV_CACHE_DTYPES.has(partial.specDraftCacheDtype)
? partial.specDraftCacheDtype
: null;
+ const specDraftModel =
+ speculativeType != null &&
+ DRAFTER_MODEL_SPEC_TYPES.has(speculativeType) &&
+ typeof partial.specDraftModel === "string" &&
+ partial.specDraftModel.trim().length > 0 &&
+ partial.specDraftModel.trim().length <= 1024
+ ? partial.specDraftModel.trim()
+ : null;
return {
engineParallelism: partial.engineParallelism === "pipeline" || partial.engineParallelism === "data"
? partial.engineParallelism : "tensor",
@@ -1057,6 +1096,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
speculativeType,
specDraftNMax,
specDraftCacheDtype,
+ specDraftModel,
loadMode: canonicalizeLoadMode(partial.loadMode),
ctxCheckpoints: normalizeCtxCheckpoints(partial.ctxCheckpoints),
cacheRam: normalizeCacheRam(partial.cacheRam),
@@ -1108,8 +1148,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
};
}
-/** A config in the exact shape storage keeps: the UI carries sentinels storage does not
- * (Speculative Decoding "auto" canonicalizes to null), which would read as non-default. */
+/** A config in the exact shape storage keeps, so a UI value storage drops cannot read as non-default. */
export function normalizePerModelConfig(raw: unknown): PerModelConfig {
return normalize(raw);
}
@@ -1132,9 +1171,15 @@ function normalize(raw: unknown): PerModelConfig {
* client reconstructs anyway, and stamping every record v4 would put the whole store out of reach.
* The tuning group and the reasoning pair follow the same rule. */
function storedSchemaVersion(normalized: PerModelConfig): number {
- if (normalized.tensorSplit != null) {
+ const mode = normalized.speculativeType;
+ // A kept Auto and the MLX-only modes are values an older client folds back to unset.
+ const mlxMode = mode === "auto" || (MLX_ONLY_SPEC_TYPES as readonly string[]).includes(mode ?? "");
+ if (normalized.specDraftModel != null || mlxMode) {
return STORAGE_SCHEMA_VERSION;
}
+ if (normalized.tensorSplit != null) {
+ return PRE_MLX_DRAFTER_SCHEMA_VERSION;
+ }
if (normalized.mlxInt8Prefill) {
return PRE_TENSOR_SPLIT_SCHEMA_VERSION;
}
@@ -1329,6 +1374,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean {
// default, so a config changing only these was dropped on the way to storage. Compared against null, not truth:
// 0 checkpoints and a 0 or -1 cache are values.
(config.specDraftCacheDtype ?? null) === null &&
+ (config.specDraftModel ?? null) === null &&
(config.loadMode ?? null) === null &&
config.ctxCheckpoints == null &&
config.cacheRam == null &&
diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.ts
index b8c14f74d31..bf0a9556809 100644
--- a/studio/frontend/src/features/model-picker/sharing/fields.ts
+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts
@@ -50,6 +50,8 @@ export type SharedConfigKey = Exclude<
| "engine"
| "enginePrecision"
| "engineParallelism"
+ // A named drafter can be a local directory.
+ | "specDraftModel"
>;
type Field = { label: string; valid: Validator; error?: string };
diff --git a/studio/frontend/src/lib/speculative-modes.ts b/studio/frontend/src/lib/speculative-modes.ts
index 83b21cd7927..e0af1f7871f 100644
--- a/studio/frontend/src/lib/speculative-modes.ts
+++ b/studio/frontend/src/lib/speculative-modes.ts
@@ -21,6 +21,49 @@ export const SPECULATIVE_TYPES = [
"off",
] as const;
+/** Values only an MLX load reads (studio/backend/core/inference/mlx_speculative.py). */
+export const MLX_ONLY_SPEC_TYPES = ["eagle3"] as const;
+
+export const MLX_SPECULATIVE_TYPES = [
+ "auto",
+ "mtp",
+ "dflash",
+ "dspark",
+ "eagle3",
+ "ngram",
+ "off",
+] as const;
+
+export interface MlxDrafter {
+ repo: string;
+ kind: string;
+ /** False when the drafter only fits the model's architecture and is not named for it. */
+ named: boolean;
+}
+
+/** Drafters the Drafter picker offers in `mode`, as `[repo, label]`: every cached kind under Auto,
+ * else that kind, plus a saved choice that is not cached (or is a local folder) so it stays visible. */
+export function mlxDrafterChoices(
+ drafters: readonly MlxDrafter[],
+ mode: string,
+ selected: string | null,
+): [string, string][] {
+ const choices = drafters
+ .filter((drafter) => mode === "auto" || drafter.kind === mode)
+ .map((drafter): [string, string] => [
+ drafter.repo,
+ drafter.named ? drafter.repo : `${drafter.repo} (same architecture)`,
+ ]);
+ return selected != null && !choices.some(([repo]) => repo === selected)
+ ? [[selected, selected], ...choices]
+ : choices;
+}
+
+/** The mode an MLX load runs: every drafter kind also copies repeated text, so llama.cpp's `mtp+ngram` is `mtp`. */
+export function mlxSpeculativeMode(mode: string): string {
+ return mode === "mtp+ngram" ? "mtp" : mode;
+}
+
/**
* The modes that consume spec_draft_n_max, i.e. the ones that launch a drafter
* with a configurable depth. Named for the setting rather than for MTP: DSpark
@@ -32,8 +75,24 @@ export const DRAFT_N_MAX_SPEC_TYPES: ReadonlySet<string> = new Set([
"mtp+ngram",
"dspark",
"dflash",
+ ...MLX_ONLY_SPEC_TYPES,
]);
+export const DRAFTER_MODEL_SPEC_TYPES: ReadonlySet<string> = new Set([
+ ...DRAFT_N_MAX_SPEC_TYPES,
+ "auto",
+]);
+
+/** The mode a load sends: the model's own choice, else the standing preference, which GGUF loads write.
+ * On MLX its ngram reads as auto: n-gram copying alone would move a text model onto the vision runtime. */
+export function resolveSpeculativeType(
+ chosen: string | null,
+ standing: string,
+ isMlx: boolean,
+): string {
+ return chosen ?? (isMlx && standing === "ngram" ? "auto" : standing);
+}
+
/**
* The modes that always launch a SEPARATE draft model, and so a second context
* with its own KV cache for the draft cache dtype to apply to.
diff --git a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
index 707efd9c722..5126b8666cb 100644
--- a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
+++ b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
@@ -156,7 +156,7 @@ test("hydration does not mark itself saved when the write failed", () => {
assert.match(
src,
- /const hydrationSaved = savePerModelConfig\( configId, target\.ggufVariant, rememberedConfig, hydrationEvicted, \); setSavedRemember\(hydrationSaved\);/,
+ /const hydrationSaved = savePerModelConfig\( configId, target\.ggufVariant, storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\), hydrationEvicted, \); setSavedRemember\(hydrationSaved\);/,
);
});
@@ -171,7 +171,7 @@ test("hydration propagates what its own write evicted", () => {
// The write hands savePerModelConfig somewhere to report evictions.
assert.match(
src,
- /savePerModelConfig\( configId, target\.ggufVariant, rememberedConfig, hydrationEvicted, \)/,
+ /savePerModelConfig\( configId, target\.ggufVariant, storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\), hydrationEvicted, \)/,
);
// And they are cleared the way the save path clears them: mirrored fields only.
assert.match(
diff --git a/studio/frontend/tests/llama-extra-args-panel-hydration.test.ts b/studio/frontend/tests/llama-extra-args-panel-hydration.test.ts
index c7e145b35c3..cd5b639682c 100644
--- a/studio/frontend/tests/llama-extra-args-panel-hydration.test.ts
+++ b/studio/frontend/tests/llama-extra-args-panel-hydration.test.ts
@@ -186,7 +186,7 @@ test("the panel adopts a shared server config without overwriting a live edit",
// Whitespace-tolerant: the call carries an eviction list now, so it spans lines.
assert.match(
PANEL,
- /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*rememberedConfig,/,
+ /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\),/,
);
assert.match(PANEL, /configRef\.current === configAtStart/);
assert.match(PANEL, /rememberRef\.current === rememberAtStart/);
@@ -279,7 +279,7 @@ test("the hydration write-back rejects a stale server response", () => {
const writeBack = PANEL.slice(adoptionStart, writeBackEnd);
assert.match(
writeBack,
- /const storedConfig = resolveInitialConfig\(\s*configId,\s*target\.ggufVariant,\s*\);\s*if \(perModelConfigStorageChanged\(storedAtStart, storedConfig\)\) \{\s*return;\s*\}[\s\S]*const rememberedConfig = fromApiOverride\(\s*resolvedRow,\s*storedConfig\.config,\s*\);[\s\S]*savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*rememberedConfig,/,
+ /const storedConfig = resolveInitialConfig\(\s*configId,\s*target\.ggufVariant,\s*\);\s*if \(perModelConfigStorageChanged\(storedAtStart, storedConfig\)\) \{\s*return;\s*\}[\s\S]*const rememberedConfig = fromApiOverride\(\s*resolvedRow,\s*storedConfig\.config,\s*\);[\s\S]*savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\),/,
);
});
diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
index 35f71c60b34..f14ae33b2fd 100644
--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
@@ -208,8 +208,8 @@ const ROWS: Row[] = [
"and reads normally.",
},
{
- name: "version 11 (genuinely future)",
- raw: { version: 11, customContextLength: 32768 },
+ name: "version 12 (genuinely future)",
+ raw: { version: 12, customContextLength: 32768 },
normalizedPin: null,
rawPin: 32768,
isDefault: true,
@@ -357,7 +357,7 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by ve
true,
);
assert.equal(
- stage({ version: 11, customContextLength: 32768 }).remembered,
+ stage({ version: 12, customContextLength: 32768 }).remembered,
false,
);
});
@@ -365,6 +365,8 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by ve
test("a stored mlxKvQuant stamps a version older builds refuse", () => {
assert.equal(stampedVersion({ mlxKvQuant: "tq-4" }), 7);
+ // An older client folds a kept Auto and an MLX-only mode back to unset, so both lock it out.
+ assert.deepEqual(["auto", "eagle3", "mtp"].map((speculativeType) => stampedVersion({ speculativeType })), [11, 11, 1]);
assert.equal(stampedVersion({ mlxKvQuant: "4" }), 7);
assert.equal(stampedVersion({ loadMode: "mmap" }), 5);
assert.equal(stampedVersion({ mlxKvQuant: "tq-4", loadMode: "mmap" }), 7);
diff --git a/studio/frontend/tests/mlx-spec-fallback-copy.test.ts b/studio/frontend/tests/mlx-spec-fallback-copy.test.ts
new file mode 100644
index 00000000000..1a941c0a1c4
--- /dev/null
+++ b/studio/frontend/tests/mlx-spec-fallback-copy.test.ts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { mlxSpecFallbackMessage } from "../src/features/chat/lib/mlx-spec-fallback.ts";
+
+test("an MLX fallback notice says what runs in the passed-over drafter's place", () => {
+ assert.match(mlxSpecFallbackMessage("drafter_not_found", "mtp") ?? "", /with MTP instead/);
+ assert.match(mlxSpecFallbackMessage("drafter_no_memory", "ngram") ?? "", /n-gram copies only/);
+ assert.match(mlxSpecFallbackMessage("auto_context_cost", null) ?? "", /without speculative/);
+ // llama.cpp's codes are not MLX's, so an unknown one shows nothing rather than a wrong remedy.
+ assert.equal(mlxSpecFallbackMessage("binary_no_mtp", null), null);
+});
+
+test("an MLX fallback notice claims no more than every path behind its code", () => {
+ // No code guarantees its remedy works, and drafter_no_memory also covers an unpriced fit.
+ for (const code of ["drafter_not_found", "drafter_incompatible", "drafter_no_memory", "runtime_error", "kv_quant", "auto_context_cost"]) {
+ assert.doesNotMatch(mlxSpecFallbackMessage(code, null) ?? "missing", /to use it|missing/);
+ }
+ assert.doesNotMatch(mlxSpecFallbackMessage("drafter_no_memory", null) ?? "", /does not fit/);
+});
diff --git a/studio/frontend/tests/per-model-spec-disable-aliases.test.ts b/studio/frontend/tests/per-model-spec-disable-aliases.test.ts
index 704d177d565..638952fbc7a 100644
--- a/studio/frontend/tests/per-model-spec-disable-aliases.test.ts
+++ b/studio/frontend/tests/per-model-spec-disable-aliases.test.ts
@@ -8,9 +8,22 @@ import { registerBundlerResolver } from "./helpers/kit.ts";
registerBundlerResolver();
-const { normalizePerModelConfig } = await import(
+const { normalizePerModelConfig, pinSpeculativeMode, storedSpeculativeAuto } = await import(
"../src/features/model-picker/model-config/per-model-config.ts"
);
+const { loadedConfigSignature } = await import(
+ "../src/features/model-picker/model-config/config-signature.ts"
+);
+
+const { mlxDrafterChoices } = await import("../src/lib/speculative-modes.ts");
+
+test("the drafter picker offers the mode's cached drafters and keeps a saved one visible", () => {
+ const cached = [{ repo: "o/m-MTP", kind: "mtp", named: true }, { repo: "o/n-DFlash", kind: "dflash", named: false }];
+ const pick = (mode: string, saved: string | null) => mlxDrafterChoices(cached, mode, saved);
+ assert.deepEqual(pick("auto", null).map(([repo]: [string, string]) => repo), ["o/m-MTP", "o/n-DFlash"]);
+ assert.deepEqual(pick("dflash", "o/n-DFlash"), [["o/n-DFlash", "o/n-DFlash (same architecture)"]]);
+ assert.deepEqual(pick("mtp", "/local/drafter")[0], ["/local/drafter", "/local/drafter"]);
+});
const specOf = (value: string) =>
normalizePerModelConfig({ speculativeType: value }).speculativeType;
@@ -38,13 +51,32 @@ test("a stored disable alias is an override, not a fall-through to the global de
});
test("the rest of the mapping still resolves as before", () => {
- // "auto" and an unknown value are the follow-global sentinel, which is what the
- // aliases above must not be confused with.
- assert.equal(specOf("auto"), null);
- assert.equal(specOf("default"), null);
+ // An unknown value is the follow-global sentinel the aliases must not become; Auto is kept.
+ assert.equal(specOf("auto"), "auto");
+ assert.equal(specOf("default"), "auto");
assert.equal(specOf("bogus"), null);
assert.equal(specOf("mtp"), "mtp");
assert.equal(specOf("draft-mtp"), "mtp");
assert.equal(specOf("ngram-mod"), "ngram");
assert.equal(specOf("mtp+ngram"), "mtp+ngram");
});
+
+test("MLX modes and an explicit Auto survive storage; the drafter only with a drafter mode", () => {
+ assert.deepEqual([specOf("eagle3"), specOf("dflash+ngram")], ["eagle3", null]);
+ const drafterOf = (speculativeType: string, specDraftModel: string) =>
+ normalizePerModelConfig({ speculativeType, specDraftModel }).specDraftModel;
+ assert.deepEqual(
+ [drafterOf("auto", " o/d "), drafterOf("ngram", "o/d"), drafterOf("mtp", " ")],
+ ["o/d", null, null],
+ );
+ // GGUF's Auto is "follow the standing preference", which storage spells null.
+ assert.equal(storedSpeculativeAuto({ speculativeType: "auto" }, false).speculativeType, null);
+ assert.equal(storedSpeculativeAuto({ speculativeType: "auto" }, true).speculativeType, "auto");
+ // A drafter typed while the mode is unset pins the shown one, or storage would drop it.
+ const pinned = pinSpeculativeMode({ speculativeType: null }, "auto", { specDraftModel: "o/d" });
+ assert.equal(normalizePerModelConfig(pinned).specDraftModel, "o/d");
+ assert.equal(pinSpeculativeMode({ speculativeType: "mtp" }, "auto", {}).speculativeType, undefined);
+ // A new live drafter re-seeds an open editor.
+ const auto = normalizePerModelConfig({ speculativeType: "auto" });
+ assert.notEqual(loadedConfigSignature(auto), loadedConfigSignature({ ...auto, specDraftModel: "o/d" }));
+});
diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
index 5d20fb28c19..d603d2708d4 100644
--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
@@ -173,6 +173,7 @@ const FIELDS: FieldCase[] = [
same: 16,
different: 8,
},
+ { key: "specDraftModel", statusKey: "spec_draft_model", same: "org/d", different: "org/e" },
{
key: "nParallel",
statusKey: "requested_parallel_slots",
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index f1788aa6fe7..e584400749f 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -720,6 +720,27 @@ test("a binary stand-down that cannot repair does not decline the shortcut", ()
);
});
+test("an MLX resident is judged on its speculative settings, a standing ngram reading as auto", () => {
+ const mlx = { ...DEFAULTS, is_gguf: false, is_mlx: true, speculative_type: "auto" };
+ const ngram = { ...STANDING, speculativeType: "ngram" };
+ assert.equal(matches(mlx, BLANK, ngram), true);
+ assert.equal(matches({ ...mlx, speculative_type: "ngram" }, BLANK, ngram), false);
+ assert.equal(matches(mlx, { ...BLANK, speculativeType: "eagle3" }), false);
+ const drafted = { ...BLANK, speculativeType: "auto", specDraftModel: "o/d" };
+ assert.equal(matches({ ...mlx, spec_draft_model: "o/d" }, drafted), true);
+ assert.equal(matches(mlx, drafted), false);
+ const deep = { ...BLANK, speculativeType: "mtp", specDraftNMax: 6 };
+ assert.equal(matches({ ...mlx, speculative_type: "mtp", spec_draft_n_max: 6 }, deep), true);
+ assert.equal(matches({ ...mlx, speculative_type: "mtp", spec_draft_n_max: 4 }, deep), false);
+ // An MLX load reads "+ngram" as its kind, which already copies; GGUF keeps the two apart.
+ const copying = { ...BLANK, speculativeType: "mtp+ngram" };
+ assert.equal(matches({ ...mlx, speculative_type: "mtp" }, copying), true);
+ assert.equal(matches({ ...DEFAULTS, speculative_type: "mtp" }, copying), false);
+ // llama.cpp's retry arms: an identical MLX load dedupes.
+ const notFound = { ...mlx, spec_fallback_reason: "drafter_not_found" };
+ assert.equal(residentSpeculativeNeedsRepair(notFound, "auto"), false);
+});
+
test("a non-GGUF resident is not judged on a GGUF invocation field", () => {
// requested_context_length is set only by the llama.cpp path. A safetensors or MLX
// status never carries it, and the resolver answers the generation length for a
diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.ts
index 204ffad93df..f27dae0a843 100644
--- a/studio/frontend/tests/server-tuning-settings.test.ts
+++ b/studio/frontend/tests/server-tuning-settings.test.ts
@@ -113,7 +113,9 @@ test("a record only claims the new schema version when it carries one", () => {
// toStoredConfig stamps the OLDEST version that understands every field
// present, so an older client can still rewrite a record it fully knows.
const source = readSrc("features/model-picker/model-config/per-model-config.ts");
- assert.match(source, /const STORAGE_SCHEMA_VERSION = 10;/);
+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 11;\nconst PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;/);
+ assert.match(source, /specDraftModel != null \|\| mlxMode\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);
+ assert.match(source, /tensorSplit != null\)\s*\{\s*return PRE_MLX_DRAFTER_SCHEMA_VERSION;/);
assert.match(source, /const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;/);
assert.match(source, /const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;/);
assert.match(source, /const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;/);
diff --git a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts
index 3c451ee82ef..37caaba342f 100644
--- a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts
+++ b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts
@@ -724,7 +724,7 @@ test("opening the panel ticks Remember for any model with a resolvable row", ()
// the entry, so a merge that comes out default is a clear that has to travel.
assert.match(
PANEL,
- /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*rememberedConfig,/,
+ /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\),/,
);
// Whatever that write evicted is cleared before the block returns, or a dropped
// model keeps applying its server row with nothing able to forget it.
@@ -769,3 +769,18 @@ test("a localStorage that throws degrades on every path instead of propagating",
asWindow.window.localStorage = storage;
}
});
+
+
+test("the MLX drafter mirrors to the server and back", () => {
+ const config = normalizePerModelConfig({ speculativeType: "eagle3", specDraftModel: "o/d" });
+ assert.equal(toApiOverride(config).spec_draft_model, "o/d");
+ // biome-ignore lint/style/useNamingConvention: API schema
+ const row = { speculative_type: "auto", spec_draft_model: "o/d" };
+ assert.equal(fromApiOverride(row).specDraftModel, "o/d");
+ // Without the flag the backend keeps a drafter the save cleared.
+ const overrides = path.join(
+ path.dirname(fileURLToPath(import.meta.url)),
+ "../src/features/model-picker/api/model-overrides.ts",
+ );
+ assert.match(readFileSync(overrides, "utf8"), /mirrors_spec_draft_model: true/);
+});
diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.py
index 4c380b24f80..0231277e779 100644
--- a/tests/studio/test_chat_autoload_failure_gate.py
+++ b/tests/studio/test_chat_autoload_failure_gate.py
@@ -258,6 +258,10 @@ function isExternalModelId(value: unknown) {
function resolvePreserveThinkingOnLoad(resp: any) {
return Boolean(resp?.supports_preserve_thinking && resp?.preserve_thinking_default);
}
+// Verbatim from lib/speculative-modes.ts, so a load asks for the mode production would.
+function resolveSpeculativeType(chosen: string | null, standing: string, isMlx: boolean) {
+ return chosen ?? (isMlx && standing === "ngram" ? "auto" : standing);
+}
function resolveInferenceCheckpointId(status: any) {
return status.active_model
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index d45bcfdb6ff..b00f764762b 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -1125,14 +1125,19 @@ def test_blur_cache_cleared_on_every_settled_render():
def test_auto_defaults_not_persisted_as_overrides():
- """Auto GPU memory mode and Auto/default speculative type are follow-global
- defaults; normalization must not persist them as per-model overrides, else a
- model stops following later changes to the global preference."""
+ """Auto GPU memory mode and a GGUF model's Auto/default speculative type are
+ follow-global defaults; they must not persist as per-model overrides, else a
+ model stops following later changes to the global preference. An MLX model
+ keeps its explicit Auto, which beats a standing "off"."""
src = _read("features/model-picker/model-config/per-model-config.ts")
assert 'if (partial.gpuMemoryMode === "manual") {' in src
assert 'partial.gpuMemoryMode === "auto" || partial.gpuMemoryMode === "manual"' not in src
spec = re.search(r'if \(s === "auto" \|\| s === "default"\) \{\s*return ([^;]+);', src)
- assert spec and spec.group(1).strip() == "null"
+ assert spec and spec.group(1).strip() == '"auto"'
+ fold = " ".join(src.split())
+ assert (
+ '!isMlx && config.speculativeType === "auto" ? { ...config, speculativeType: null }' in fold
+ )
def test_compare_pane_context_from_own_config_only():
@@ -2446,7 +2451,7 @@ def test_parallel_slots_reach_an_api_load_through_the_server_mirror():
store = _read_backend("utils/openai_auto_switch_settings.py")
assert 'entry["n_parallel"] = n_parallel' in store
# Ungated: MLX sizes its batch by the same width llama-server sizes its slots by.
- shared, gguf_block = store.split(" if is_gguf:", 1)
+ shared, gguf_block = store.rsplit(" if is_gguf:", 1)
assert '("n_parallel", "n_parallel"),' in shared
assert "n_parallel" not in gguf_block.split("\n\n", 1)[0]
@@ -3249,10 +3254,15 @@ def test_backfill_splits_a_quant_suffix_the_way_the_backend_does():
def test_the_settings_page_judges_the_config_storage_actually_keeps():
"""savePerModelConfig normalizes before deciding, and the runtime hands this page
- Speculative Decoding "auto", which canonicalizes to null."""
+ Speculative Decoding "auto", which a GGUF model stores as null."""
src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split())
# Load and Save (#10216) both go through persistConfig.
- assert "const normalized = normalizePerModelConfig(next);" in src
+ assert (
+ "const normalized = normalizePerModelConfig(storedSpeculativeAuto(next, targetIsMlx));"
+ in src
+ )
+ # Server hydration writes the same record, so a GGUF row's stored "auto" folds there too.
+ assert "storedSpeculativeAuto(rememberedConfig, !target.isGguf)" in src
assert "defaultConfig: isDefaultConfig(normalized)" in src
# The same object goes to storage and to the server, or they disagree again.
assert "savePerModelConfig(configId, target.ggufVariant, normalized, evicted)" in src