Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: speculative decoding for the MLX inference backend (#13014)
Long Yixing6bd164faafbc
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 87e6c0525e5..d90294a907c 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -1722,9 +1722,10 @@ def _build_generation_stats(gen_tps,cached_n = 0,finish_reason = None,+ drafts = None,):"""Map mlx stream stats onto the usage/timings shape llama-server emits, plus the reason- generation ended."""+ generation ended. ``drafts`` is a speculative reply's (drafted, accepted) token counts."""prompt_n = int(prompt_n or 0)gen_n = int(gen_n or 0)cached_n = int(cached_n or 0)@@ -1752,6 +1753,7 @@ def _build_generation_stats("predicted_per_token_ms": (predicted_ms / gen_n) if gen_n > 0 else 0.0,"predicted_per_second": gen_tps,"cache_n": cached_n,+ **({} if drafts is None else {"draft_n": drafts[0], "draft_n_accepted": drafts[1]}),},# Latched where generation exits, so a cancel arriving afterwards cannot rewrite the reason the completion# actually ended for.@@ -2568,8 +2570,10 @@ def mlx_fit_to_memory(load_in_4bit: bool,retains_history: bool,kv_bits = None,+ **route,) -> Optional[int]:- """The context a load of *model_dir* is held to, priced at the KV width its cache takes."""+ """The context a load of *model_dir* is held to, priced at the KV width its cache takes.+ ``vision=True`` in *route* prices the mlx-vlm route a speculative text load takes."""budget = mlx_memory_budget(retains_history = retains_history)if budget is None or not ceiling or not model_dir:return None@@ -2583,6 +2587,7 @@ def mlx_fit_to_memory(max_ctx = int(ceiling),kv_bits = kv_bits,load_in_4bit = load_in_4bit,+ **route,)logger.debug("MLX fit for %s: %s tokens under %.1f GB", model_dir, fitted, budget / 1e9)return fitted@@ -2601,6 +2606,7 @@ def _fitted_context(kv_bits = None,is_vlm = False,eligibility = None,+ **route,):"""The fit for a resident model and the eligibility verdict it was priced under."""try:@@ -2617,10 +2623,50 @@ def _fitted_context(load_in_4bit = load_in_4bit,retains_history = retains_history,kv_bits = kv_bits if verdict is None or verdict[0] in ("full", "partial") else None,+ **route,)return fitted, verdict+def mlx_drafter_fit(+ model_dir,+ ceiling,+ pinned,+ source,+ *,+ load_in_4bit: bool,+ costless = False,+):+ """``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:+ only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit+ the context the load fits without it. ``attaches`` is None when the fit could not be priced."""+ budget = mlx_memory_budget(retains_history = mlx_vlm_snapshot_store_available())+ ceiling = pinned or ceiling+ if budget is None or not ceiling or not model_dir:+ return None, None+ with mlx_rng_preserved():+ try:+ from core.inference.mlx_memory import mlx_fit_outcome++ fit = functools.partial(+ mlx_fit_outcome,+ model_dir,+ budget_bytes = budget,+ max_ctx = int(ceiling),+ load_in_4bit = load_in_4bit,+ vision = True,+ )+ outcome, fitted = fit(drafter = (source.path, source.builtin))+ if outcome == "unsizable":+ return None, fitted+ if costless and (outcome, fitted) != fit():+ return False, fitted+ except Exception as exc:+ logger.info("MLX drafter %s not priced: %s", source.path, exc)+ return None, None+ return outcome == "fits" or (outcome == "fitted" and not pinned), fitted++def _kv_window_enforced(model, is_vlm, window):"""Whether a cache built at *window* is bounded in every layer, None where none could be builtto judge: both runtimes defer to a model's own make_cache and ignore max_kv_size there."""@@ -3072,6 +3118,7 @@ def _make_mlx_presence_penalty_processor(penalty: float):logits = logits - mask[:vocab]return logits+ _processor.history_only = Truereturn _processor@@ -3116,6 +3163,7 @@ def _make_mlx_frequency_penalty_processor(penalty: float):counts = mx.zeros((vocab + 1,), dtype = mx.float32).at[safe].add(1.0)return logits - (penalty * counts[:vocab]).astype(logits.dtype)+ _processor.history_only = Truereturn _processor@@ -3140,6 +3188,7 @@ def _make_mlx_logit_bias_processor(logit_bias: dict):mask = mx.zeros((vocab + 1,), dtype = mx.float32).at[state["safe"]].add(state["values"])return logits + mask[:vocab].astype(logits.dtype)+ _processor.history_only = Truereturn _processor@@ -3203,7 +3252,11 @@ def _mlx_sampling_processors(processors.append(_make_mlx_logit_bias_processor(logit_bias))if repetition_penalty is not None and float(repetition_penalty) not in (0.0, 1.0):from mlx_lm.sample_utils import make_logits_processors- processors.extend(make_logits_processors(repetition_penalty = float(repetition_penalty)))++ repetition = make_logits_processors(repetition_penalty = float(repetition_penalty))+ for processor in repetition:+ processor.history_only = True+ processors.extend(repetition)if presence_penalty:processors.append(_make_mlx_presence_penalty_processor(float(presence_penalty)))if frequency_penalty:@@ -3232,34 +3285,44 @@ def _vlm_add_special_tokens(model_type, processor):)+def _vlm_prepared_inputs(model, processor, prompt, images):+ """The processor's inputs for *prompt* and *images*, as mlx-vlm's ``stream_generate`` prepares them."""+ from mlx_vlm.utils import prepare_inputs++ config = getattr(model, "config", None)+ read = config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)+ return prepare_inputs(+ processor,+ images = images,+ prompts = prompt,+ image_token_index = read("image_token_index"),+ add_special_tokens = _vlm_add_special_tokens(read("model_type"), processor),+ )+++def _vlm_generate_kwargs(inputs):+ """Prepared inputs as ``stream_generate`` kwargs, which then skips its own preparation."""+ inputs = dict(inputs)+ return {+ "input_ids": inputs.pop("input_ids"),+ "pixel_values": inputs.pop("pixel_values", None),+ "mask": inputs.pop("attention_mask", None),+ **inputs,+ }++class _VLMMediaBlock:"""A prompt's rows through its last vision token, prefilled by Studio: models whose visiontokens attend to each other prefill in one forward that lands off the grid, and this is thatforward cut where the grid can continue."""def __init__(self, model, processor, prompt, images, make_cache):- from mlx_vlm.utils import prepare_inputs-self._model = modelself._make_cache = make_cache- config = getattr(model, "config", None)- read = config.get if isinstance(config, dict) else lambda attr: getattr(config, attr, None)- self._inputs = prepare_inputs(- processor,- images = images,- prompts = prompt,- image_token_index = read("image_token_index"),- add_special_tokens = _vlm_add_special_tokens(read("model_type"), processor),- )+ self._inputs = _vlm_prepared_inputs(model, processor, prompt, images)def generate_kwargs(self):- inputs = dict(self._inputs)- return {- "input_ids": inputs.pop("input_ids"),- "pixel_values": inputs.pop("pixel_values", None),- "mask": inputs.pop("attention_mask", None),- **inputs,- }+ return _vlm_generate_kwargs(self._inputs)def rows(self, token_ids):ids = self._inputs["input_ids"]@@ -3623,7 +3686,7 @@ class _TextBatchSession:)return plan.think_prefix- def step(self):+ def step(self, waiting = None):"""Report what the batch produced, retiring every reply that ended."""began = time.perf_counter()prompt_events, events = self.generator.next()@@ -3771,6 +3834,7 @@ class _VisionBatchRow:"admitted_at","ready_at","cancelled",+ "drafts",)def __init__(self, *, handle, plan, row):@@ -3784,6 +3848,7 @@ class _VisionBatchRow:self.admitted_at = time.perf_counter()self.ready_at = Noneself.cancelled = False+ self.drafts = Nonedef _row_prompt_cache_gap():@@ -3827,6 +3892,7 @@ class _VisionBatchSession:_temporary_mlx_adapter_state(backend._model, adapter_state))self._held.enter_context(backend._int8_prefill_scope(zoo_generation = True))+ self._speculative = getattr(backend, "_speculative_draft", None)self.stream = BatchStream(backend._model,backend._processor,@@ -3836,6 +3902,7 @@ class _VisionBatchSession:# Rows quantize their own caches where the single path's decode does.**backend._kv_runtime_quant_kwargs(),),+ **({} if self._speculative is None else {"speculative": self._speculative}),)except BaseException:self._held.close()@@ -3901,9 +3968,15 @@ class _VisionBatchSession:)return plan.think_prefix- def step(self):- now = None- for event in self.stream.step():+ def step(self, waiting = None):+ now, stopped = None, {}+ # Only a speculative step spans many tokens, reported as they are decoded; only its unsloth-zoo has the call.+ # The batch cannot change mid-step, so a row that stopped itself ends the step and leaves after it.+ for event in (+ self.stream.iter_step(lambda: bool(stopped) or (waiting is not None and waiting()))+ if self._speculative is not None+ else self.stream.step()+ ):row = self._by_row.get(event.index)if row is None:continue@@ -3922,11 +3995,12 @@ class _VisionBatchSession:result = event.resultif not row.cancelled:row.reason = result.finish_reason- row.prompt_tokens = result.prompt_token_count- row.cached_tokens = getattr(result, "cached_token_count", 0)- row.generated = len(result.token_ids) + (result.finish_reason == "stop")+ self._record(row, result, len(result.token_ids) + (result.finish_reason == "stop"))yield from self._retire(row, cancelled = row.cancelled)elif row.cancelled:+ stopped[row.handle] = row+ for row in stopped.values():+ if row.handle in self._rows:yield from self._retake(row)def withdraw(self, handles):@@ -3941,11 +4015,15 @@ class _VisionBatchSession:def _retake(self, row):managed = self.stream.withdraw(row.row)if managed is not None:- row.prompt_tokens = managed.prompt_token_count- row.cached_tokens = getattr(managed, "cached_token_count", 0)- row.generated = len(managed.token_ids)+ self._record(row, managed, len(managed.token_ids))yield from self._retire(row, cancelled = True)+ def _record(self, row, result, generated):+ row.prompt_tokens, row.generated = result.prompt_token_count, generated+ row.cached_tokens = getattr(result, "cached_token_count", 0)+ if self._speculative is not None:+ row.drafts = (result.draft_tokens, result.accepted_draft_tokens)+def _retire(self, row, *, cancelled):self._rows.pop(row.handle, None)self._by_row.pop(row.row, None)@@ -3963,6 +4041,7 @@ class _VisionBatchSession:row.generated / max(decoded, 1e-9),cached_n = row.cached_tokens,finish_reason = row.reason or "stop",+ drafts = row.drafts,)yield row.handle, None@@ -3995,6 +4074,7 @@ class MLXInferenceBackend:self._is_vlm = Falseself._reads_vision = Noneself._turboquant_refusal = ""+ self._speculative_draft = Noneself._multi_image_marker = Noneself._config = {}self._distributed_group = None@@ -4555,9 +4635,15 @@ class MLXInferenceBackend:kv_quant = None,chat_template_override = None,int8_prefill = False,+ speculative_type = None,+ spec_draft_n_max = None,+ spec_draft_model = None,+ spec_drafters_allowed = None,) -> bool:import mlx.core as mx+ from core.inference import mlx_speculative+# Keep the token so the native-template fallback can fetch a gated model's repo template during generation.self._hf_token = hf_tokenmodel_name = config.identifier if hasattr(config, "identifier") else str(config)@@ -4579,14 +4665,33 @@ class MLXInferenceBackend:if self._turboquantelse "")+ spec_mode = mlx_speculative.mlx_spec_mode(speculative_type)+ spec_reason = None+ speculates = mlx_speculative.speculates_on_route(+ spec_mode, bool(is_vision), spec_draft_model+ )+ if speculates:+ spec_reason = mlx_speculative.speculation_refusal(+ kv_quant = kv_bits is not None or self._turboquant,+ distributed = is_distributed,+ lora = is_lora,+ )+ speculates = spec_reason is None+ if (+ spec_mode == "auto" and not spec_draft_model+ ): # not a request, so no refusal to report+ spec_reason = None+ self._speculative_draft = None# mlx-lm cannot batch a quantized cache; mlx-vlm, which also ships text architectures, can.batches_quantized_text = (kv_bits is not Noneand not (is_distributed or is_lora)and _row_quantized_cache_gap() is None)+ # Speculation runs on mlx-vlm models and caches.use_vlm = (is_vision+ or speculatesor (self._turboquant and not self._turboquant_refusal)or batches_quantized_text)@@ -4669,8 +4774,9 @@ class MLXInferenceBackend:if is_vision or not use_vlm or is_metal_queue_dead(exc):raiselogger.warning(- "Load of %s through mlx-vlm for its KV quantization failed (%s); serving through mlx-lm",+ "Load of %s through mlx-vlm for %s failed (%s); serving through mlx-lm",model_name,+ "speculative decoding" if speculates else "its KV quantization",exc,)_eligibility = None@@ -4692,7 +4798,14 @@ class MLXInferenceBackend:gc.collect()_drain_generation_streams(mx)mx.clear_cache()- if self._turboquant:+ if speculates:+ speculates = False+ spec_reason = (+ None+ if spec_mode == "auto" and not spec_draft_model+ else mlx_speculative.RUNTIME_ERROR+ )+ elif self._turboquant:self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOADuse_vlm = Falseload_kwargs["text_only"] = True@@ -4727,11 +4840,63 @@ class MLXInferenceBackend:self._model, max_seq_length)_priceable = not (is_distributed or is_lora or dtype is not None)+ spec_kind = _drafter_ctx = None+ _drafter_fitted = False+ _vlm_route = speculates and not is_vision+ if speculates and _vlm_generation_is_diffusion(self._model):+ # Neither the single path nor the resident batch may decode a diffusion model speculatively.+ speculates = False+ spec_reason = (+ None+ if spec_mode == "auto" and not spec_draft_model+ else mlx_speculative.RUNTIME_ERROR+ )+ if speculates:+ model_dir = _snapshot_dir(self._model, model_name)+ resolution = mlx_speculative.resolve_speculation(+ spec_mode,+ spec_draft_model,+ model_dir = model_dir,+ target_name = model_name,+ allowed = spec_drafters_allowed,+ )++ def _fits(source):+ if not _priceable:+ return None, None+ return mlx_drafter_fit(+ model_dir,+ _served_ctx,+ _positive_int(max_seq_length),+ source,+ load_in_4bit = load_in_4bit,+ costless = spec_mode == "auto",+ )++ (+ self._speculative_draft,+ spec_kind,+ spec_reason,+ _drafter_ctx,+ ) = mlx_speculative.build_draft(+ self._model, resolution, fits = _fits, draft_n_max = _positive_int(spec_draft_n_max)+ )+ _drafter_fitted = (+ self._speculative_draft is not None and self._speculative_draft.drafter is not None+ )+ logger.info(+ "MLX speculative decoding: %s (mode %s, reason %s)",+ spec_kind or "off",+ spec_mode,+ spec_reason,+ )# TurboQuant's layout is not one the planner prices, and a refused one stays full width:# fit either at full width, which only ever under-promises._requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)_fitted_ctx, _eligibility = (- (None, _eligibility)+ (_drafter_ctx, _eligibility)+ if _drafter_fitted+ else (None, _eligibility)if not _priceable or _positive_int(max_seq_length) is not Noneelse _fitted_context(self._model,@@ -4742,6 +4907,7 @@ class MLXInferenceBackend:kv_bits = _requested_bits,is_vlm = use_vlm,eligibility = _eligibility,+ **({"vision": True} if _vlm_route else {}),))if _fitted_ctx:@@ -4858,6 +5024,11 @@ class MLXInferenceBackend:"mlx_int8_prefill_reason": self._int8_prefill["reason"],"chat_template_override_requested": self._template_override["requested"],"chat_template_override_reason": self._template_override["reason"],+ "speculative_type": spec_mode,+ "spec_draft_n_max": _positive_int(spec_draft_n_max),+ "spec_draft_model": spec_draft_model or None,+ "spec_drafter_kind": spec_kind,+ "spec_fallback_reason": spec_reason,}# Capture chat_template_info for the worker IPC reply and route capability classification.self._populate_chat_template_info(model_name, native_template)@@ -4966,6 +5137,7 @@ class MLXInferenceBackend:self._model_fusion.close()if model_name in self.models:del self.models[model_name]+ self._speculative_draft = Noneself._model = Noneself._tokenizer = Noneself._processor = None@@ -6075,7 +6247,14 @@ class MLXInferenceBackend:"constrain the next token. Load an autoregressive model to use ""guided decoding.")- if presence_penalty or frequency_penalty or logit_bias or constraint is not None:+ draft = self._speculative_draft+ if (+ presence_penalty+ or frequency_penalty+ or logit_bias+ or constraint is not None+ or (_rep_active and draft is not None)+ ):# These need custom processors: pass the full list (repetition + the rest) instead of the# repetition_penalty shortcut so all apply.vlm_kwargs["logits_processors"] = _mlx_sampling_processors(@@ -6142,6 +6321,24 @@ class MLXInferenceBackend:else:vlm_kwargs.update(self._kv_quant_generate_kwargs())session_scope = session if session is not None else nullcontext()+ processors = vlm_kwargs.get("logits_processors") or ()+ # The draft is told these processors, and takes only ones that read just the history (a grammar's+ # matcher advances per call); tool turns decode as before; a clip is nothing a drafter was built for.+ if (+ draft is None+ or tools+ or tool_protocol_active+ or video is not None+ or not all(getattr(processor, "history_only", False) for processor in processors)+ or _vlm_generation_is_diffusion(self._model)+ ):+ draft = None+ elif "input_ids" not in vlm_kwargs:+ vlm_kwargs.update(+ _vlm_generate_kwargs(+ _vlm_prepared_inputs(self._model, self._processor, prompt, images)+ )+ )def _stream_vlm_snapshots():nonlocal stopped@@ -6168,6 +6365,22 @@ class MLXInferenceBackend:generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))final_response = Noneclip_path = None+ if draft is not None:+ from unsloth_zoo.mlx.generate import SamplingParams++ # Under the lock: the load's one draft serves one generation at a time.+ draft.prepare(+ vlm_kwargs["input_ids"][0].tolist(),+ SamplingParams(+ temperature = temperature,+ top_p = top_p,+ top_k = max(int(top_k or 0), 0),+ min_p = float(min_p or 0.0),+ seed = seed,+ ),+ processors,+ )+ vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)try:if video is not None:clip_path = _write_video_clip(video)@@ -6264,6 +6477,9 @@ class MLXInferenceBackend:getattr(final_response, "generation_tokens", 0),max_new_tokens,),+ drafts = None+ if draft is None+ else (draft.draft_n, draft.draft_n_accepted),)if document_only:@@ -6305,6 +6521,8 @@ class MLXInferenceBackend:return "fewer than two replies were requested"if self._is_vlm:reason = self._vlm_batch_unavailable_reason(requests)+ if reason is None and self._speculative_draft is not None:+ reason = self._vlm_resident_unavailable_reason({})if reason is not None:return reasonif any(_mlx_stop_sequences(request.get("stop")) for request in requests):@@ -6398,12 +6616,18 @@ class MLXInferenceBackend:self.last_batch_generation_stats = [None] * len(requests)self.last_generation_stats = None- run = self._generate_vlm_batch if self._is_vlm else self._generate_text_batch- yield from run(requests, cancel_event = cancel_event, _adapter_state = _adapter_state)+ kwargs = {"cancel_event": cancel_event, "_adapter_state": _adapter_state}+ if not self._is_vlm:+ yield from self._generate_session_batch(requests, _TextBatchSession, **kwargs)+ elif self._speculative_draft is not None: # stream_batch has no speculative mode+ yield from self._generate_session_batch(requests, _VisionBatchSession, **kwargs)+ else:+ yield from self._generate_vlm_batch(requests, **kwargs)- def _generate_text_batch(+ def _generate_session_batch(self,requests,+ session_type,*,cancel_event = None,_adapter_state = None,@@ -6415,7 +6639,7 @@ class MLXInferenceBackend:yield row, snapshotwith self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):- session = _TextBatchSession(+ session = session_type(self,width = len(requests),adapter_state = _adapter_state,@@ -6428,7 +6652,7 @@ class MLXInferenceBackend:if prefix:yield row, prefixwhile session.rows_in_flight:- yield from report(session.step())+ yield from report(session.step(cancel_event and cancel_event.is_set))if cancel_event is not None and cancel_event.is_set():yield from report(session.withdraw(list(session.handles)))finally:diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex 35665079252..5ad29f8938f 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -5,6 +5,7 @@from __future__ import annotations+import contextvarsimport globimport functoolsimport json@@ -25,6 +26,7 @@ __all__ = ["MLX_PREFILL_CHUNK","MlxMemoryBreakdown","mlx_fit_context",+ "mlx_fit_outcome","mlx_memory_breakdown","mlx_shard_files","mlx_weight_bytes",@@ -34,6 +36,8 @@ MLX_KV_BLOCK = 256_PROBE_SHORT = 8_PROBE_LONG = 40+# Above mlx-vlm's decode block (8 tokens), inside which some targets evaluate their forward.+_DRAFTER_PROBE = (16, 48)MLX_PREFILL_CHUNK = 2048@@ -295,8 +299,16 @@ def _generation_settings(config: dict) -> tuple:)+# The route a sizing prices when the load overrides the config's own: a speculative text load goes through mlx-vlm.+_ROUTE_VISION: contextvars.ContextVar = contextvars.ContextVar("mlx_route_vision", default = None)++def _loads_as_vision(config: dict) -> bool:+ forced = _ROUTE_VISION.get()+ if forced is not None:+ return forcedfrom types import SimpleNamespace+try:from utils.models.model_config import _is_vlmexcept Exception:@@ -586,6 +598,24 @@ def _config_widths(config: dict):)+def _entries(cache, quantized = None) -> list:+ entries = []+ for index, entry in enumerate(cache):+ converts = quantized is not None and quantized[index] is not entry+ for leaf in _sub_caches(entry):+ entries.append(+ {+ "bytes": _entry_bytes(leaf),+ "quant_bytes": _entry_bytes(leaf),+ "bound_spec": _bound_spec(leaf),+ "conv_width": _conv_width(leaf),+ "block": _block(leaf),+ "converts": converts,+ }+ )+ return entries++def _probe(config: dict, dtype, n_tokens: int, kv_bits):import mlx.core as mx@@ -609,20 +639,7 @@ def _probe(config: dict, dtype, n_tokens: int, kv_bits):if cache is None:raise failurequantized = _quantize_like_runtime(cache, kv_bits)- entries = []- for index, entry in enumerate(cache):- converts = quantized is not None and quantized[index] is not entry- for leaf in _sub_caches(entry):- entries.append(- {- "bytes": _entry_bytes(leaf),- "quant_bytes": _entry_bytes(leaf),- "bound_spec": _bound_spec(leaf),- "conv_width": _conv_width(leaf),- "block": _block(leaf),- "converts": converts,- }- )+ entries = _entries(cache, quantized)if quantized is not None:converted = [leaf for entry in quantized for leaf in _sub_caches(entry)]if len(converted) == len(entries):@@ -643,30 +660,7 @@ def _cache_plan(config: dict, dtype, kv_bits):far = _probe(config, dtype, _PROBE_LONG, kv_bits)if not near["entries"] or len(near["entries"]) != len(far["entries"]):raise ValueError("cache probe returned no comparable entries")- span = _PROBE_LONG - _PROBE_SHORT-- def solve(key):- return [- (n[key] - (f[key] - n[key]) / span * _PROBE_SHORT, (f[key] - n[key]) / span)- for n, f in zip(near["entries"], far["entries"])- ]-- plan = []- for slot, (const, slope), (q_const, q_slope) in zip(- near["entries"], solve("bytes"), solve("quant_bytes")- ):- plan.append(- {- "const": const,- "slope": slope,- "quant_const": q_const,- "quant_slope": q_slope,- "bound_spec": slot["bound_spec"],- "conv_width": slot["conv_width"],- "block": slot["block"],- "converts": slot["converts"] and near["quantized"],- }- )+ plan = _solved(near["entries"], far["entries"], (_PROBE_SHORT, _PROBE_LONG), near["quantized"])return (plan,0 if near["quantized"] else None,@@ -678,6 +672,58 @@ def _cache_plan(config: dict, dtype, kv_bits):)+def _solved(near: list, far: list, lengths: tuple, quantized: bool) -> list:+ """Per-entry ``const + slope * T`` from the entries of caches filled to two lengths."""+ short, span = lengths[0], lengths[1] - lengths[0]++ def solve(key):+ return [+ (n[key] - (f[key] - n[key]) / span * short, (f[key] - n[key]) / span)+ for n, f in zip(near, far)+ ]++ return [+ {+ "const": const,+ "slope": slope,+ "quant_const": q_const,+ "quant_slope": q_slope,+ "bound_spec": slot["bound_spec"],+ "conv_width": slot["conv_width"],+ "block": slot["block"],+ "converts": slot["converts"] and quantized,+ }+ for slot, (const, slope), (q_const, q_slope) in zip(+ near, solve("bytes"), solve("quant_bytes")+ )+ ]+++def _drafter_sizing(config: dict, dtype, drafter: tuple) -> tuple:+ """``(weight bytes, cache plan, capture bytes per prompt token)`` of the drafter ``(path, builtin)``+ attaches, from a weightless target and a lazily built drafter."""+ from unsloth_zoo.mlx.speculative import companion_drafter, native_mtp_drafter, probe_drafter++ path, builtin = drafter+ if not builtin and (_snapshot_config(path) or {}).get("model_file"):+ raise ValueError("this drafter carries its own model module")+ target = _whole_model(config, dtype)+ built = (native_mtp_drafter if builtin else companion_drafter)(path, target, lazy = True)+ if built is None:+ raise ValueError("no MTP splitter knows this checkpoint")+ near, far = (_entries(probe_drafter(target, built, length)) for length in _DRAFTER_PROBE)+ if len(near) != len(far):+ raise ValueError("drafter cache probe returned no comparable entries")+ # A dflash or eagle3 prefill keeps every chunk's captured layers, then a concatenated copy.+ captured = len(getattr(built, "capture", {}).get("capture_layer_ids", ()))+ hidden = _config_width((config.get("text_config") or config).get("hidden_size"))+ return (+ built.weight_bytes,+ _solved(near, far, _DRAFTER_PROBE, False),+ 2 * captured * hidden * dtype.size,+ )++@functools.lru_cache(maxsize = 256)def _bounded_peak(cache_type, attribute: str, value: int, n_ctx: int, prefill_chunk: int) -> int:"""Largest slot count a bounded cache reaches, by driving the real class."""@@ -981,6 +1027,9 @@ class _MlxSizing:chunk: intkv_bits: Optional[int]kv_group_size: int+ drafter_weights: int = 0+ drafter_plan: tuple = ()+ capture_per_token: int = 0_SIZING_CACHE_MAX = 16@@ -1003,7 +1052,12 @@ def _checkpoint_fingerprint(model_dir: str) -> tuple:def _size_load(- model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool+ model_dir: str,+ kv_bits: Optional[int],+ prefill_chunk: Optional[int],+ load_in_4bit: bool,+ vision: Optional[bool] = None,+ drafter: Optional[tuple] = None,) -> Optional[_MlxSizing]:"""Cached ``_size_load_uncached``: every estimate prices the fit and the breakdown, and everyslider step prices again, each otherwise rebuilding the same weightless architecture."""@@ -1015,14 +1069,17 @@ def _size_load(bool(load_in_4bit),os.environ.get("UNSLOTH_MLX_PROMPT_CACHE_BYTES"),_checkpoint_fingerprint(model_dir),+ vision,+ drafter,+ drafter and _checkpoint_fingerprint(drafter[0]),)except Exception:- return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)with _sizing_lock:if key in _sizing_cache:_sizing_cache.move_to_end(key)return _sizing_cache[key]- sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)with _sizing_lock:_sizing_cache[key] = sizingwhile len(_sizing_cache) > _SIZING_CACHE_MAX:@@ -1036,7 +1093,26 @@ def _clear_sizing_cache() -> None:def _size_load_uncached(- model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool+ model_dir: str,+ kv_bits: Optional[int],+ prefill_chunk: Optional[int],+ load_in_4bit: bool,+ vision: Optional[bool] = None,+ drafter: Optional[tuple] = None,+) -> Optional[_MlxSizing]:+ token = _ROUTE_VISION.set(vision)+ try:+ return _sized(model_dir, kv_bits, prefill_chunk, load_in_4bit, drafter)+ finally:+ _ROUTE_VISION.reset(token)+++def _sized(+ model_dir: str,+ kv_bits: Optional[int],+ prefill_chunk: Optional[int],+ load_in_4bit: bool,+ drafter: Optional[tuple],) -> Optional[_MlxSizing]:"""Everything about a load that does not move with the context, or None if it cannot be sized."""config = _snapshot_config(model_dir)@@ -1062,6 +1138,7 @@ def _size_load_uncached(for tower, checkpoint in zip(facts["widths"], _config_widths(config)))chunk = prefill_chunk or loaded_chunk+ drafted = (0, (), 0) if drafter is None else _drafter_sizing(config, dtype, drafter)except Exception as exc:logger.debug("MLX estimate could not size %s (%s): %s", model_dir, config.get("model_type"), exc@@ -1077,6 +1154,9 @@ def _size_load_uncached(chunk = chunk,kv_bits = kv_bits,kv_group_size = loaded_group,+ drafter_weights = drafted[0],+ drafter_plan = tuple(drafted[1]),+ capture_per_token = drafted[2],)@@ -1096,14 +1176,16 @@ def _priced_at(sizing: _MlxSizing, n_ctx: int) -> Optional[MlxMemoryBreakdown]:sizing.kv_bits,sizing.kv_group_size,)+ drafter_kv = _line_bytes(list(sizing.drafter_plan), context, chunk, quantized = False)except Exception as exc:logger.debug("MLX estimate could not price %s tokens: %s", n_ctx, exc)return None- total = sizing.weights + kv + compute+ capture = sizing.capture_per_token * context+ total = sizing.weights + sizing.drafter_weights + kv + drafter_kv + compute + capturereturn MlxMemoryBreakdown(- weights_bytes = sizing.weights,- kv_bytes = kv,- compute_bytes = compute,+ weights_bytes = sizing.weights + sizing.drafter_weights,+ kv_bytes = kv + drafter_kv,+ compute_bytes = compute + capture,total_bytes = total,gpu_bytes = total,n_ctx = context,@@ -1126,13 +1208,16 @@ def mlx_memory_breakdown(kv_bits: Optional[int] = None,prefill_chunk: Optional[int] = None,load_in_4bit: bool = False,+ vision: Optional[bool] = None,+ drafter: Optional[tuple] = None,) -> Optional[MlxMemoryBreakdown]:- """Price an MLX load, or None when it cannot honestly be sized."""- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ """Price an MLX load, or None when it cannot honestly be sized. ``vision`` overrides the loader route+ the config implies; ``drafter`` is the ``(path, builtin)`` drafter a speculative load attaches."""+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)return None if sizing is None else _priced_at(sizing, n_ctx)-def mlx_fit_context(+def mlx_fit_outcome(model_dir: str,*,budget_bytes: int,@@ -1141,25 +1226,36 @@ def mlx_fit_context(kv_bits: Optional[int] = None,prefill_chunk: Optional[int] = None,load_in_4bit: bool = False,-) -> Optional[int]:- """Largest context whose estimated footprint stays inside ``budget_bytes``."""- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ vision: Optional[bool] = None,+ drafter: Optional[tuple] = None,+) -> tuple:+ """``("unsizable" | "fits" | "fitted" | "no_fit", context)``: the largest context inside+ ``budget_bytes`` for ``"fitted"``, else None."""+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)if sizing is None:- return None+ return "unsizable", Noneceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)at_ceiling = _priced_at(sizing, ceiling)- if at_ceiling is None or at_ceiling.total_bytes <= budget_bytes:- return None+ if at_ceiling is None:+ return "unsizable", None+ if at_ceiling.total_bytes <= budget_bytes:+ return "fits", Nonefloor = max(-(-int(min_ctx or 0) // MLX_KV_BLOCK) * MLX_KV_BLOCK, MLX_KV_BLOCK)low, high, best = floor, ceiling, Nonewhile low <= high:middle = (low + high) // 2priced = _priced_at(sizing, middle)if priced is None:- return None+ return "unsizable", Noneif priced.total_bytes <= budget_bytes:best = middlelow = middle + 1else:high = middle - 1- return None if best is None else (best // MLX_KV_BLOCK) * MLX_KV_BLOCK+ return ("no_fit", None) if best is None else ("fitted", (best // MLX_KV_BLOCK) * MLX_KV_BLOCK)+++def mlx_fit_context(model_dir: str, **kwargs) -> Optional[int]:+ """Largest context whose estimated footprint stays inside ``budget_bytes``; None when the ceiling+ already fits or nothing can be said."""+ return mlx_fit_outcome(model_dir, **kwargs)[1]diff --git a/studio/backend/core/inference/mlx_speculative.py b/studio/backend/core/inference/mlx_speculative.pynew file mode 100644index 00000000000..b8318eb4e52--- /dev/null+++ b/studio/backend/core/inference/mlx_speculative.py@@ -0,0 +1,400 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Which drafter an MLX load speculates with: ``speculative_type`` and ``spec_draft_model`` resolved+against the target's own MTP head and the companion drafters already in the local Hugging Face cache."""++from __future__ import annotations++import inspect+import json+import os+import re+from dataclasses import dataclass+from pathlib import Path+from typing import Optional++from loggers import get_logger++logger = get_logger(__name__)++MLX_DRAFTER_KINDS = ("mtp", "dflash", "dspark", "eagle3")+MLX_SPEC_MODES = frozenset({"auto", "off", "ngram", *MLX_DRAFTER_KINDS})+_LEGACY_MODES = {+ "mtp+ngram": "mtp",+ "default": "auto",+ "draft-mtp": "mtp",+ "draft-dspark": "dspark",+ "draft-dflash": "dflash",+ "ngram-mod": "ngram",+ "ngram-simple": "ngram",+ "none": "off",+ "disable": "off",+ "disabled": "off",+}+_COMPANION_ORDER = ("mtp", "dflash2", "dflash", "dspark", "eagle3")++DRAFTER_NOT_FOUND = "drafter_not_found"+DRAFTER_INCOMPATIBLE = "drafter_incompatible"+DRAFTER_NO_MEMORY = "drafter_no_memory"+RUNTIME_ERROR = "runtime_error"+KV_QUANT = "kv_quant"+AUTO_CONTEXT_COST = "auto_context_cost"+++def mlx_spec_mode(value) -> str:+ """``speculative_type`` as an MLX load reads it: absent or unknown is ``auto``, as on llama.cpp."""+ if not isinstance(value, str) or not value.strip():+ return "auto"+ mode = value.strip().lower()+ mode = _LEGACY_MODES.get(mode, mode)+ return mode if mode in MLX_SPEC_MODES else "auto"+++@dataclass(frozen = True)+class DrafterSource:+ kind: str+ path: str+ builtin: bool+++@dataclass(frozen = True)+class SpecResolution:+ mode: str+ sources: tuple = ()+ copies: bool = False # n-gram copies alongside a drafter, or alone when an explicit kind's drafter does not attach+ reason: Optional[str] = None++ @property+ def speculative(self) -> bool:+ return bool(self.sources) or self.copies+++def _read_config(path) -> dict:+ try:+ with open(os.path.join(path, "config.json"), encoding = "utf-8") as handle:+ config = json.load(handle)+ return config if isinstance(config, dict) else {}+ except (OSError, ValueError):+ return {}+++def companion_kind(config: dict) -> Optional[str]:+ """``dflash2``, ``dflash``, ``dspark``, ``eagle3`` or ``mtp`` (a separate MTP head or a Gemma assistant) for a companion drafter config."""+ # mlx-vlm executes a checkpoint's own model_file on load; a drafter never gets the target's remote-code grant.+ if config.get("model_file"):+ return None+ architectures = " ".join(str(name) for name in config.get("architectures") or ())+ # mlx-vlm loads EAGLE-3 only in the speculators format, not e.g. llama-typed Eagle3 exports.+ if "eagle3" in (config.get("model_type"), config.get("speculators_model_type")):+ return "eagle3"+ model_type = str(config.get("model_type", ""))+ if model_type.endswith("_mtp") or (+ model_type.endswith("_assistant") and "Assistant" in architectures+ ):+ return "mtp"+ if "dflash_config" in config:+ return (+ "dspark"+ if "DSpark" in architectures+ else "dflash2"+ if "DFlash2" in architectures+ else "dflash"+ )+ return None+++def _reported(kind: str) -> str:+ return "dflash" if kind == "dflash2" else kind+++_DRAFTER_WORDS = frozenset(+ {"dflash", "dflash2", "dspark", "eagle3", "speculator", "assistant", "mtp", "drafter"}+)+_WEIGHT_WORDS = re.compile(r"^(mlx|bf16|fp16|fp8|mxfp4|nvfp4|qat|unquantized|\d+bit|dwq)$")+_GGUF_QUANT = re.compile(r"(?<![a-z0-9])q\d(_[a-z0-9]+)+(?![a-z0-9])")+++def _alnum(name: str) -> str:+ return re.sub(r"[^a-z0-9]", "", os.path.basename(str(name).rstrip("/")).lower())+++def _stem(name: str) -> str:+ """A drafter repo's name without its organization, drafter and weight-format words, as alphanumerics."""+ base = _GGUF_QUANT.sub("", os.path.basename(str(name).rstrip("/")).lower())+ words = re.split(r"[-_]", base.replace("speculator.eagle3", ""))+ return _alnum(+ "".join(w for w in words if w not in _DRAFTER_WORDS and not _WEIGHT_WORDS.match(w))+ )+++def _text(config: dict) -> dict:+ scope = config.get("text_config")+ return scope if isinstance(scope, dict) else config+++def _vocab_size(config: dict) -> Optional[int]:+ for scope in (+ config,+ config.get("text_config") or {},+ config.get("transformer_layer_config") or {},+ ):+ if isinstance(scope, dict) and isinstance(scope.get("vocab_size"), int):+ return scope["vocab_size"]+ return None+++def _fits(kind: str, config: dict, target: dict) -> bool:+ """Whether a drafter's config is built for a target of this shape. A dimension either side does not state cannot refuse."""+ text, own = _text(target), _text(config)+ layers, flash = text.get("num_hidden_layers"), config.get("dflash_config") or {}+ if kind == "eagle3":+ wants = config.get("target_hidden_size") or (+ config.get("transformer_layer_config") or {}+ ).get("hidden_size")+ taps = config.get("eagle_aux_hidden_state_layer_ids") or flash.get("target_layer_ids")+ elif kind == "mtp":+ wants = config.get("backbone_hidden_size") or own.get("hidden_size")+ taps = None+ if "backbone_hidden_size" not in config:+ pairs = [+ (own.get("num_hidden_layers"), layers),+ (own.get("model_type"), text.get("model_type")),+ ]+ if any(a is not None and b is not None and a != b for a, b in pairs):+ return False+ else:+ wants = config.get("target_hidden_size") or config.get("hidden_size")+ taps = flash.get("target_layer_ids") or config.get("target_layer_ids")+ stated = config.get("num_target_layers") or flash.get("num_target_layers")+ if stated is not None and layers is not None and stated != layers:+ return False+ vocab = _vocab_size(target)+ return not (+ (wants is not None and text.get("hidden_size") not in (None, wants))+ # Families number tapped layers from 0 or from 1; only an id past both is out of range.+ or (taps and layers is not None and max(taps) > layers)+ or (vocab is not None and _vocab_size(config) not in (None, vocab))+ )+++def _has_weights(snapshot) -> bool:+ try:+ return any(name.endswith(".safetensors") for name in os.listdir(snapshot))+ except OSError:+ return False+++def _cached_repos():+ from utils.utils import _hf_cache_roots+ seen = set()+ for root in _hf_cache_roots():+ try:+ entries = sorted(os.listdir(root))+ except OSError:+ continue+ for entry in entries:+ if entry.startswith("models--") and entry not in seen:+ seen.add(entry)+ yield entry[len("models--") :].replace("--", "/")+++def cached_drafters(target_name: str, target_config: dict) -> list:+ """``(repo, source, named)`` for each cached drafter with weights whose config fits the target's shape.++ ``named`` drafters carry the target's model name and come first, in auto's order; the rest fit+ by shape alone (another generation or fine-tune of the same architecture) and follow.+ """+ from utils.utils import hf_cache_snapshot_dir_for_repo++ target, found = _alnum(target_name), []+ for repo in _cached_repos():+ if repo.lower() == str(target_name).lower():+ continue+ snapshot = hf_cache_snapshot_dir_for_repo(repo)+ config = _read_config(snapshot) if snapshot is not None else {}+ kind = companion_kind(config)+ if kind is None or not _fits(kind, config, target_config) or not _has_weights(snapshot):+ continue+ stem = _stem(repo)+ named = bool(stem) and target.startswith(stem)+ source = DrafterSource(_reported(kind), str(snapshot), False)+ found.append((not named, _COMPANION_ORDER.index(kind), repo, source))+ return [(repo, source, not unnamed) for unnamed, _, repo, source in sorted(found)]+++def discover_companions(+ target_name: str,+ target_config: dict,+ allowed = None,+) -> list:+ """The cached drafters auto attaches: those named for the target, within ``allowed`` repos when given."""+ found = cached_drafters(target_name, target_config)+ return [+ source for repo, source, named in found if named and (allowed is None or repo in allowed)+ ]+++def _named_companion(spec_draft_model: str) -> Optional[DrafterSource]:+ from utils.utils import hf_cache_snapshot_dir_for_repo++ path = Path(os.path.expanduser(spec_draft_model))+ snapshot = path if path.is_dir() else hf_cache_snapshot_dir_for_repo(spec_draft_model)+ kind = companion_kind(_read_config(snapshot)) if snapshot is not None else None+ return None if kind is None else DrafterSource(_reported(kind), str(snapshot), False)+++def has_builtin_head(model_dir: Optional[str]) -> bool:+ if not model_dir:+ return False+ try:+ from mlx_vlm.speculative.drafters.mtp_split import detect_mtp_splitter+ return detect_mtp_splitter(Path(model_dir)) is not None+ except Exception as exc:+ logger.debug("MLX speculative: no MTP splitter check for %s: %s", model_dir, exc)+ return False+++def resolve_speculation(+ speculative_type,+ spec_draft_model: Optional[str],+ *,+ model_dir: Optional[str],+ target_name: str,+ allowed = None,+) -> SpecResolution:+ """The drafters an MLX load tries: a named ``spec_draft_model``, then cached companions (any kind under auto) and the built-in head, in ``_COMPANION_ORDER``."""+ mode = mlx_spec_mode(speculative_type)+ if mode == "off":+ return SpecResolution(mode)+ if mode == "ngram":+ return SpecResolution(mode, copies = True)+ auto = mode == "auto"+ kind = "mtp" if auto else mode+ sources, reason = [], None+ if spec_draft_model:+ named = _named_companion(spec_draft_model)+ if named is None:+ reason = DRAFTER_NOT_FOUND+ elif named.kind != kind and not auto:+ reason = DRAFTER_INCOMPATIBLE+ else:+ sources.append(named)+ companions = discover_companions(+ target_name, _read_config(model_dir) if model_dir else {}, allowed+ )+ companions = [+ source for source in companions if (auto or source.kind == kind) and source not in sources+ ]+ if kind == "mtp" and has_builtin_head(model_dir):+ companions.insert(0, DrafterSource("mtp", str(model_dir), True))+ sources += companions+ if not sources and auto:+ return SpecResolution(mode, reason = reason)+ if not sources and reason is None:+ reason = DRAFTER_NOT_FOUND+ return SpecResolution(mode, tuple(sources), copies = True, reason = reason)+++def speculates_on_route(+ mode: str,+ vision: bool,+ named = None,+) -> bool:+ """Whether a load in ``mode`` looks for a drafter: auto only where mlx-vlm serves it anyway, or for a drafter the request names."""+ return mode != "off" and (mode != "auto" or vision or bool(named))+++_MAX_DEPTH = (+ 8 # a drafter without its own block size; the controller picks each round's depth below it+)+_MAX_COPY = 16+_EXACT_HEAD_DEPTH = 3 # MTP heads and assistants chain a forward per drafted token+++def speculation_refusal(*, kv_quant: bool, distributed: bool, lora: bool) -> Optional[str]:+ """The reason an MLX load cannot speculate at all, or None."""+ if kv_quant:+ return KV_QUANT+ if distributed or lora:+ return RUNTIME_ERROR+ try:+ from unsloth_zoo.mlx.speculative import speculative_unavailable_reason+ except ImportError as exc:+ logger.info("MLX speculative decoding unavailable: %s", exc)+ return RUNTIME_ERROR+ unavailable = speculative_unavailable_reason()+ if unavailable is not None:+ logger.info("MLX speculative decoding unavailable: %s", unavailable)+ return RUNTIME_ERROR+ return None+++def _draft(drafter, copies: bool, draft_n_max: Optional[int]):+ from unsloth_zoo.mlx.speculative import (+ DraftController,+ install_speculative_seam,+ SpeculativeDraft,+ )++ depth = 0 if drafter is None else int(getattr(drafter, "max_depth", _MAX_DEPTH))+ copy, exact = _MAX_COPY, False+ if draft_n_max:+ exact = draft_n_max <= getattr(drafter, "max_depth", _EXACT_HEAD_DEPTH)+ depth, copy = min(depth, draft_n_max), min(copy, draft_n_max)+ install_speculative_seam()+ return SpeculativeDraft(+ DraftController(max_depth = depth, max_copy = copy, can_copy = copies, fixed_depth = exact), drafter+ )+++def _carries_encoder_state(target) -> bool:+ # generate_step threads these from forward to forward (Mllama, Florence2, Nemotron Parse); the+ # speculative rounds refuse them only after prefill, too late to decode the reply unspeculated.+ for module in (target, getattr(target, "language_model", None)):+ try:+ parameters = inspect.signature(module).parameters+ except (TypeError, ValueError):+ continue+ if "cross_attention_states" in parameters or "encoder_outputs" in parameters:+ return True+ return False+++def build_draft(+ target,+ resolution: SpecResolution,+ *,+ fits,+ draft_n_max: Optional[int] = None,+) -> tuple:+ """``(draft, kind, reason, context)`` for the first source ``fits`` accepts that builds, else copies+ alone when the mode allows. ``reason`` names a passed-over choice, so a substitute never attaches silently."""+ from unsloth_zoo.mlx.speculative import companion_drafter, native_mtp_drafter++ if _carries_encoder_state(target):+ logger.info("MLX speculative decoding: %s carries encoder state", type(target).__name__)+ return None, None, None if resolution.mode == "auto" else RUNTIME_ERROR, None+ reason = resolution.reason+ for source in resolution.sources:+ ok, context = fits(source)+ if not ok:+ if resolution.mode != "auto":+ reason = DRAFTER_NO_MEMORY+ elif ok is not None: # priced, and the drafter would shrink it+ reason = AUTO_CONTEXT_COST+ continue+ try:+ drafter = (native_mtp_drafter if source.builtin else companion_drafter)(+ source.path, target+ )+ if drafter is None:+ raise ValueError("no MTP splitter knows this checkpoint")+ except Exception as exc:+ logger.warning("MLX drafter %s (%s) not attached: %s", source.path, source.kind, exc)+ reason = DRAFTER_INCOMPATIBLE+ continue+ return _draft(drafter, resolution.copies, draft_n_max), source.kind, reason, context+ if resolution.copies and resolution.mode != "auto": # auto copies only alongside a drafter+ return _draft(None, True, draft_n_max), "ngram", reason, None+ return None, None, reason, Nonediff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 72b6fdab7fb..894ec5f3777 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -257,6 +257,11 @@ _MLX_RUNTIME_MIRROR_FIELDS = ("mlx_int8_prefill_reason","chat_template_override_requested","chat_template_override_reason",+ "speculative_type",+ "spec_draft_n_max",+ "spec_draft_model",+ "spec_drafter_kind",+ "spec_fallback_reason",)@@ -2006,6 +2011,10 @@ class InferenceOrchestrator:mlx_distributed: bool = False,mlx_kv_quant: Optional[str] = None,mlx_int8_prefill: bool = False,+ speculative_type: Optional[str] = None,+ spec_draft_n_max: Optional[int] = None,+ spec_draft_model: Optional[str] = None,+ spec_drafters_allowed: Optional[list] = None,chat_template_override: Optional[str] = None,load_cancel_event: Optional[threading.Event] = None,post_handoff_expected_free_gb: Optional[dict[int, float]] = None,@@ -2079,6 +2088,10 @@ class InferenceOrchestrator:else None,"mlx_kv_quant": mlx_kv_quant,"mlx_int8_prefill": bool(mlx_int8_prefill),+ "speculative_type": speculative_type,+ "spec_draft_n_max": spec_draft_n_max,+ "spec_draft_model": spec_draft_model,+ "spec_drafters_allowed": spec_drafters_allowed,"chat_template_override": chat_template_override,# Read in the worker, which hides the accelerators before detection."audio_device": audio_device,diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 39fb949b4dc..8e6e0f54dcf 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -741,6 +741,10 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")load_kwargs["kv_quant"] = config.get("mlx_kv_quant")load_kwargs["int8_prefill"] = bool(config.get("mlx_int8_prefill"))+ load_kwargs["speculative_type"] = config.get("speculative_type")+ load_kwargs["spec_draft_n_max"] = config.get("spec_draft_n_max")+ load_kwargs["spec_draft_model"] = config.get("spec_draft_model")+ load_kwargs["spec_drafters_allowed"] = config.get("spec_drafters_allowed")load_kwargs["chat_template_override"] = config.get("chat_template_override")success = backend.load_model(**load_kwargs)finally:@@ -829,6 +833,11 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:"mlx_int8_prefill_reason","chat_template_override_requested","chat_template_override_reason",+ "speculative_type",+ "spec_draft_n_max",+ "spec_draft_model",+ "spec_drafter_kind",+ "spec_fallback_reason",)if k in _entry}@@ -953,6 +962,9 @@ class _Stops:def __contains__(self, request_id) -> bool:return request_id in self._stopped+ def unread(self) -> bool:+ return self._ledger is not None and self._ledger.snapshot(self._written)[1] is not None+class _StopWhileItRuns:def __init__(self, cancel_event, stops, request_id: str):@@ -1474,11 +1486,11 @@ class _ResidentBatch:if request_id in stopped:self.cancel(request_id)- def step(self) -> None:+ def step(self, waiting = None) -> None:if self.session is None or not self.session.rows_in_flight:returntry:- for handle, snapshot in self.session.step():+ for handle, snapshot in self.session.step(waiting):self._report(handle, snapshot)except Exception as exc:logger.error("Batched generation error: %s", exc, exc_info = True)@@ -2065,6 +2077,7 @@ def run_inference_process(warmth = _MLXIdleWarmth()deferred: list[dict] = []stops = _Stops(stop_ledger, resp_queue, batch, deferred)+ waiting = lambda: stops.unread() or not cmd_queue.empty()if stop_ledger is not None:stop_ledger.worker_reads_this()while True:@@ -2073,7 +2086,7 @@ def run_inference_process(if not tearing_down:if batch.rows_in_flight:warmth.active()- batch.step()+ batch.step(waiting)from_deferred = Falseif _held_head_leaves_the_hold(batch, deferred):cmd = deferred.pop(0)diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex b7c8c32b076..70f2ef69016 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -215,7 +215,11 @@ class LoadRequest(BaseModel):"Legacy values 'default' (-> auto), 'draft-mtp' (-> mtp), ""'draft-dspark' (-> dspark), 'draft-dflash' (-> dflash), ""'ngram-mod' (-> ngram), and 'ngram-simple' (kept as-is) are "- "still accepted. Ignored for non-GGUF models."+ "still accepted. MLX models read 'auto', 'mtp', 'dflash', 'dspark', 'eagle3', "+ "'ngram' and 'off'; every drafter kind also copies repeated text (n-gram), "+ "so 'mtp+ngram' reads as 'mtp'. On MLX, 'auto' attaches the first "+ "cached drafter (an MTP head or assistant, then DFlash2, DFlash, DSpark, EAGLE-3) that "+ "costs no context, on loads served through mlx-vlm. Ignored for other non-GGUF models."),)spec_draft_n_max: Optional[int] = Field(@@ -228,7 +232,21 @@ class LoadRequest(BaseModel):"CPU/Mac when unset (upstream-bench sweet spot for dense Qwen3.6 ""MTP quants, and the measured sweet spot for DFlash too). Only ""applied when speculative_type resolves to 'mtp', 'mtp+ngram', "- "'dspark' or 'dflash'."+ "'dspark' or 'dflash'. On MLX, up to the drafter's trained depth (3 for "+ "MTP heads and assistants) every step drafts exactly this many (or takes an "+ "n-gram copy expected to yield more), even when "+ "plain decoding would be faster; above it, or for n-gram copies alone, it "+ "is a ceiling and the controller decides when to draft."+ ),+ )+ spec_draft_model: Optional[str] = Field(+ None,+ max_length = 1024,+ description = (+ "MLX only: a companion drafter (DFlash, DFlash2, DSpark, EAGLE-3 or Gemma "+ "assistant) to speculate with, as a local directory or an already-cached "+ "repo id; never downloaded. Tried before the model's own MTP head and the "+ "cached companions discovery would pick."),)n_parallel: Optional[int] = Field(@@ -994,6 +1012,11 @@ class EstimateMemoryRequest(BaseModel):speculative_type: Optional[str] = Field(None, description = "Speculative mode; decides which drafter's weights are charged.")+ spec_draft_model: Optional[str] = Field(+ None,+ max_length = 1024,+ description = "MLX companion drafter intended for the follow-up load, as on /load.",+ )spec_draft_n_max: Optional[int] = Field(None,ge = 1,@@ -1084,6 +1107,28 @@ class Int8PrefillAvailabilityResponse(BaseModel):)+class MlxDraftersRequest(BaseModel):+ model_path: str = Field(..., description = "Model identifier or local path of the target")++ _resolve_the_handle = field_validator("model_path")(resolve_inventory_handle)+++class MlxDrafter(BaseModel):+ repo_id: str = Field(..., description = "Cached Hugging Face repo, usable as spec_draft_model")+ kind: str = Field(..., description = "'mtp', 'dflash', 'dspark' or 'eagle3'")+ named: bool = Field(+ True,+ description = "Whether the repo is named for this model. False: it fits the model's "+ "architecture only (another generation or a fine-tune), so Auto does not use it.",+ )+++class MlxDraftersResponse(BaseModel):+ """Cached drafters an MLX load of the target could name: those named for it first, in Auto's order."""++ drafters: list[MlxDrafter] = Field(default_factory = list)++class EstimateMemoryResponse(BaseModel):"""Itemized memory an inference load would occupy, or why it could not be sized."""@@ -1665,6 +1710,55 @@ class _InferenceRuntimeFields(BaseModel):"None when the platform default is in effect."),)+ spec_draft_model: Optional[str] = Field(+ None,+ description = "The companion drafter an MLX load was asked for; None elsewhere.",+ )+ spec_drafter_kind: Optional[str] = Field(+ None,+ description = (+ "Which drafter the resolution was about: 'mtp', 'dspark' or "+ "'dflash' ('eagle3' or 'ngram' too on MLX, where 'mtp' also names a Gemma "+ "assistant). Needed "+ "because Auto resolves the kind itself, so speculative_type still "+ "reads 'auto', and a fallback leaves the engaged type at 'default': "+ "neither still says which file the UI should tell the user to fix."+ ),+ )+ spec_fallback_reason: Optional[str] = Field(+ None,+ description = (+ "Why a speculative drafter was disabled despite being requested. "+ "'binary_no_mtp' / 'binary_outdated' -> a newer prebuilt would "+ "re-enable it (show the update affordance); 'runtime_error' -> the "+ "current build could not run it; 'drafter_not_found' -> the model's "+ "separate MTP or DSpark drafter could not be resolved; "+ "'drafter_no_vram' -> an Auto-mode fit downgrade: the model pins on "+ "GPU but the drafter's reserve does not, and Auto keeps the context "+ "rather than shrink it; select the drafter in Settings to force it. "+ "'mla_mtp_disabled' -> "+ "an Auto-mode policy downgrade: the model is MLA (GLM-5.2 et al.) "+ "whose llama.cpp MTP path runs slower than no speculation, so Auto "+ "used ngram-mod or spec-off instead -- updating won't help; choose "+ "MTP in Settings (or set UNSLOTH_MLA_MTP_ENABLED=1) to force it. "+ "'mtp_partial_offload' -> an Auto-mode policy downgrade: the model "+ "has an embedded Hybrid Mamba MTP head and the placement offloads "+ "only part of it, where the recurrent rollback copies cost more "+ "layers than the drafting wins back -- updating won't help; choose "+ "MTP in Settings to force it. "+ "MLX loads report 'drafter_not_found' (no drafter of the requested kind "+ "is cached), 'drafter_incompatible' (the named or found drafter does not "+ "fit this model), 'drafter_no_memory' (its weights and cache leave no "+ "context that fits), 'kv_quant' (KV cache quantization is on) or "+ "'runtime_error' (the installed MLX packages cannot speculate, or cannot "+ "load this model the way speculation needs), also when a later drafter "+ "or n-gram copies stood in for the one asked for. Under auto an MLX load "+ "reports no refusal; it reports 'auto_context_cost' (a found drafter would "+ "shrink the fitted context) or the drafter codes above for a named or "+ "unbuildable drafter. "+ "None when the requested strategy engaged or was not requested."+ ),+ )tensor_parallel: bool = Field(False,description = "Whether tensor-parallel split (--split-mode tensor) is active.",@@ -2045,40 +2139,6 @@ class InferenceStatusResponse(_InferenceRuntimeFields):"False -> recommend `unsloth studio update`."),)- spec_drafter_kind: Optional[str] = Field(- None,- description = (- "Which drafter the resolution was about: 'mtp', 'dspark' or "- "'dflash'. Needed "- "because Auto resolves the kind itself, so speculative_type still "- "reads 'auto', and a fallback leaves the engaged type at 'default': "- "neither still says which file the UI should tell the user to fix."- ),- )- spec_fallback_reason: Optional[str] = Field(- None,- description = (- "Why a speculative drafter was disabled despite being requested. "- "'binary_no_mtp' / 'binary_outdated' -> a newer prebuilt would "- "re-enable it (show the update affordance); 'runtime_error' -> the "- "current build could not run it; 'drafter_not_found' -> the model's "- "separate MTP or DSpark drafter could not be resolved; "- "'drafter_no_vram' -> an Auto-mode fit downgrade: the model pins on "- "GPU but the drafter's reserve does not, and Auto keeps the context "- "rather than shrink it; select the drafter in Settings to force it. "- "'mla_mtp_disabled' -> "- "an Auto-mode policy downgrade: the model is MLA (GLM-5.2 et al.) "- "whose llama.cpp MTP path runs slower than no speculation, so Auto "- "used ngram-mod or spec-off instead -- updating won't help; choose "- "MTP in Settings (or set UNSLOTH_MLA_MTP_ENABLED=1) to force it. "- "'mtp_partial_offload' -> an Auto-mode policy downgrade: the model "- "has an embedded Hybrid Mamba MTP head and the placement offloads "- "only part of it, where the recurrent rollback copies cost more "- "layers than the drafting wins back -- updating won't help; choose "- "MTP in Settings to force it. "- "None when the requested strategy engaged or was not requested."- ),- )spec_fallback_binary_changed: Optional[bool] = Field(None,description = (diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 2f2f9d198ca..6d3ee5d50b4 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3740,6 +3740,8 @@ from models.inference import (EstimateMemoryResponse,Int8PrefillAvailabilityRequest,Int8PrefillAvailabilityResponse,+ MlxDraftersRequest,+ MlxDraftersResponse,TransformersUpgradeInfo,TransformersUpgradeCheckRequest,TransformersUpgradeCheckResponse,@@ -8374,6 +8376,11 @@ def _unsloth_serving_fields(model_info: dict) -> dict:"context_length_enforced": model_info.get("context_length_enforced"),"context_length_fitted": _positive_int_or_none(model_info.get("context_length_fitted")),"context_unbounded_when_batched": bool(model_info.get("context_unbounded_when_batched")),+ "speculative_type": model_info.get("speculative_type"),+ "spec_draft_n_max": model_info.get("spec_draft_n_max"),+ "spec_draft_model": model_info.get("spec_draft_model"),+ "spec_drafter_kind": model_info.get("spec_drafter_kind"),+ "spec_fallback_reason": model_info.get("spec_fallback_reason"),}@@ -8402,6 +8409,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:mlx_int8_prefill_requested = None,mlx_int8_prefill_reason = None,mlx_context_budget = None,+ spec_draft_model = None,chat_template_override_reason = None,context_length_enforced = True,context_length_fitted = None,@@ -11928,6 +11936,7 @@ async def load_model_for_preview() -> None:if account_access.managed_account():await asyncio.to_thread(account_access.require_model_access, request.model_path)+ await _require_drafter_access(request)account_access.require_idle_other_accounts()from core.inference.llama_keepwarm import (inference_lifecycle_gate,@@ -14988,7 +14997,7 @@ def _mlx_estimate_ceiling(model_dir: str) -> Optional[int]:return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)-def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits):+def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits, **route):"""The window a load naming no Context Length would be fitted to."""from core.inference.mlx_inference import (mlx_fit_to_memory,@@ -15006,9 +15015,48 @@ def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_not getattr(config, "is_vision", False) or mlx_vlm_snapshot_store_available()),kv_bits = kv_bits,+ **route,)+def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_in_4bit: bool):+ """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it+ would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""+ from core.inference import mlx_speculative+ from core.inference.mlx_inference import mlx_drafter_fit, parse_mlx_kv_quant+ from core.inference.mlx_memory import _loads_as_vision, _snapshot_config++ mode = mlx_speculative.mlx_spec_mode(request.speculative_type)+ if not mlx_speculative.speculates_on_route(+ mode, _loads_as_vision(_snapshot_config(model_dir) or {}), request.spec_draft_model+ ):+ return None, None, False+ resolution = mlx_speculative.resolve_speculation(+ request.speculative_type,+ request.spec_draft_model,+ model_dir = model_dir,+ target_name = model_identifier,+ allowed = _discoverable_drafters(model_identifier),+ )+ bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)+ legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None+ kv_quant = bits is not None or turboquant or legacy_bits+ if mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False):+ return None, None, False+ for source in resolution.sources:+ attaches, fitted = mlx_drafter_fit(+ model_dir,+ ceiling,+ request.max_seq_length or None,+ source,+ load_in_4bit = load_in_4bit,+ costless = mode == "auto",+ )+ if attaches:+ return (source.path, source.builtin), fitted, True+ return None, None, True++def _mlx_estimate_available() -> bool:"""Whether this host would actually run the load through MLX."""try:@@ -17177,6 +17225,8 @@ def _mlx_runtime_settings_match(backend, request) -> bool:return Truefrom core.inference.mlx_inference import encode_mlx_kv_quant, parse_mlx_kv_quant+ from core.inference.mlx_speculative import mlx_spec_mode+requested = encode_mlx_kv_quant(*parse_mlx_kv_quant(getattr(request, "mlx_kv_quant", None)))return (entry["mlx_kv_quant_requested"] == requested@@ -17184,6 +17234,9 @@ def _mlx_runtime_settings_match(backend, request) -> bool:== (request.chat_template_override or None)and bool(entry.get("mlx_int8_prefill_requested"))== bool(getattr(request, "mlx_int8_prefill", False))+ and entry.get("speculative_type", "auto") == mlx_spec_mode(request.speculative_type)+ and entry.get("spec_draft_model") == (getattr(request, "spec_draft_model", None) or None)+ and entry.get("spec_draft_n_max") == _positive_int_or_none(request.spec_draft_n_max))@@ -17917,6 +17970,20 @@ async def _run_gguf_load_attempt(return False+async def _require_drafter_access(request) -> None:+ """A named MLX drafter is a second model reference, so the caller needs access to it too."""+ drafter = getattr(request, "spec_draft_model", None)+ if account_access.managed_account() and isinstance(drafter, str) and drafter.strip():+ await asyncio.to_thread(account_access.require_model_access, drafter.strip())+++def _discoverable_drafters(model_path: str) -> Optional[list]:+ """The cached drafters a load may attach on its own: any for the owner, else those the account may see."""+ if not account_access.managed_account():+ return None+ return [row["repo_id"] for row in _mlx_cached_drafters(model_path)]++def _require_resolved_base_access(config) -> None:"""Grants apply to the base in an adapter's config, so it cannot pull a foreign cached base."""base = getattr(config, "base_model", None)@@ -18133,6 +18200,7 @@ async def _load_model_impl(native_access_deferred = _defers_access_to_native_grant(request)if account_access.managed_account() and not native_access_deferred:await asyncio.to_thread(account_access.require_model_access, request.model_path)+ await _require_drafter_access(request)request = await asyncio.to_thread(_as_ollama_manifest_request, request)_requested_model_id = request.model_pathrequest = await asyncio.to_thread(@@ -19217,6 +19285,12 @@ async def _load_model_impl(subject = current_subject,mlx_kv_quant = request.mlx_kv_quant,mlx_int8_prefill = request.mlx_int8_prefill,+ speculative_type = request.speculative_type,+ spec_draft_n_max = request.spec_draft_n_max,+ spec_draft_model = request.spec_draft_model,+ spec_drafters_allowed = await asyncio.to_thread(+ _discoverable_drafters, request.model_path+ ),chat_template_override = request.chat_template_override,load_cancel_event = load_cancel_event,on_prior_worker_released = _release_chat_after_teardown,@@ -20786,6 +20860,7 @@ async def estimate_memory(native_access_deferred = _defers_access_to_native_grant(request)if account_access.managed_account() and not native_access_deferred:await asyncio.to_thread(account_access.require_model_access, request.model_path)+ await _require_drafter_access(request)from core.inference.llama_cpp import _args_place_tensors_on_cpufrom core.inference.llama_server_args import (_effective_tensor_parallel,@@ -20864,10 +20939,17 @@ async def estimate_memory(if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):mlx_kv_bits = Nonemlx_named_ctx = request.max_seq_length or 0- mlx_fitted_ctx = None- if not mlx_named_ctx:+ mlx_drafter, mlx_fitted_ctx, mlx_speculates = _mlx_estimate_drafter(+ request,+ model_identifier,+ model_dir,+ _mlx_estimate_ceiling(model_dir),+ mlx_load_in_4bit,+ )+ mlx_route = {"vision": True} if mlx_speculates else {}+ if not mlx_named_ctx and mlx_drafter is None:mlx_fitted_ctx = _mlx_estimate_fitted_context(- config, model_dir, mlx_load_in_4bit, mlx_kv_bits+ config, model_dir, mlx_load_in_4bit, mlx_kv_bits, **mlx_route)mlx_priced_ctx = mlx_named_ctx or mlx_fitted_ctx or _mlx_estimate_ceiling(model_dir)if not mlx_priced_ctx:@@ -20877,6 +20959,8 @@ async def estimate_memory(n_ctx = mlx_priced_ctx,kv_bits = mlx_kv_bits,load_in_4bit = mlx_load_in_4bit,+ **mlx_route,+ **({} if mlx_drafter is None else {"drafter": mlx_drafter}),)if mlx_breakdown is None:return EstimateMemoryResponse(available = False, reason = "unsizable")@@ -21019,6 +21103,35 @@ async def estimate_memory(return await asyncio.to_thread(_estimate)+def _mlx_cached_drafters(model_path: str) -> list:+ from core.inference import mlx_speculative+ from utils.utils import hf_cache_snapshot_dir++ model_dir = model_path if os.path.isdir(model_path) else hf_cache_snapshot_dir(model_path)+ if model_dir is None:+ return []+ config = mlx_speculative._read_config(model_dir)+ # The cache is shared between accounts: list only what this one may see.+ return account_access.filter_model_rows(+ [+ {"repo_id": repo, "kind": source.kind, "named": named}+ for repo, source, named in mlx_speculative.cached_drafters(model_path, config)+ ]+ )++[email protected]("/mlx-drafters", response_model = MlxDraftersResponse)+async def mlx_drafters(+ request: MlxDraftersRequest, current_subject: str = Depends(get_current_subject)+):+ """Cached drafters an MLX load of this model could name. Reads only local config files."""+ if account_access.managed_account():+ await asyncio.to_thread(account_access.require_model_access, request.model_path)+ return MlxDraftersResponse(+ drafters = await asyncio.to_thread(_mlx_cached_drafters, request.model_path)+ )++@router.post("/unload", response_model = UnloadResponse)@account_access.gpu_busy_routeasync def unload_model(request: UnloadRequest, current_subject: str = Depends(get_current_subject)):@@ -22028,7 +22141,6 @@ async def _slot_status(current_subject: str):**_runtime_fields,requested_context_length = llama_backend.requested_n_ctx,llama_cpp_supports_mtp = _supports_mtp,- spec_fallback_reason = llama_backend.spec_fallback_reason,spec_fallback_binary_changed = _spec_fallback_binary_changed(llama_backend),spec_probe_retry_pending = _spec_probe_retry_pending(llama_backend),spec_dflash_retry_pending = _spec_dflash_retry_pending(llama_backend),@@ -22039,7 +22151,6 @@ async def _slot_status(current_subject: str):tensor_parallel_dropped_by_arch_gate = _arch_gate_dropped_tensor_parallel(llama_backend),- spec_drafter_kind = llama_backend.spec_drafter_kind,llama_cpp_prebuilt_stale = _stale,llama_cpp_installed_tag = _installed_tag,llama_cpp_latest_tag = _latest_tag,diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex 428d9726ac2..a186fa7d9b9 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -998,6 +998,7 @@ class ModelOverridePayload(BaseModel):speculative_type: Optional[str] = Field(default = None, max_length = 32)spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16)+ spec_draft_model: Optional[str] = Field(default = None, max_length = 1024)# Parallel decode slots (llama-server --parallel), GGUF-only; None follows the server default.n_parallel: Optional[int] = Field(default = None, ge = PARALLEL_SLOTS_MIN, le = PARALLEL_SLOTS_MAX)reasoning_budget: Optional[int] = Field(default = None, ge = -1, le = 2_147_483_647)@@ -1020,6 +1021,7 @@ class ModelOverridePayload(BaseModel):# The reasoning pair came later than the four, so a build that mirrors them can still# predate it: its own flag, same contract.mirrors_reasoning_budget: bool = False+ mirrors_spec_draft_model: bool = Falsetensor_parallel: bool = Falsedisable_vision: bool = Falsemlx_int8_prefill: bool = False@@ -2496,6 +2498,7 @@ def update_openai_auto_switch_override("fill_absent_fields","mirrors_server_tuning","mirrors_reasoning_budget",+ "mirrors_spec_draft_model",},exclude_none = True,)@@ -2571,10 +2574,16 @@ def update_openai_auto_switch_override(# legacy contract is a payload carrying only model_id, which leaves remove None._tuning_fields = ("load_mode", "spec_draft_cache_type", "ctx_checkpoints", "cache_ram")_reasoning_fields = ("reasoning_budget", "reasoning_budget_message")- _kept_tuning = {name: getattr(payload, name) for name in _tuning_fields + _reasoning_fields}+ _drafter_fields = ("spec_draft_model",)+ _kept_tuning = {+ name: getattr(payload, name)+ for name in _tuning_fields + _reasoning_fields + _drafter_fields+ }# Each group is carried only for a client that does not mirror it.- _carried_fields = (() if payload.mirrors_server_tuning else _tuning_fields) + (- () if payload.mirrors_reasoning_budget else _reasoning_fields+ _carried_fields = (+ (() if payload.mirrors_server_tuning else _tuning_fields)+ + (() if payload.mirrors_reasoning_budget else _reasoning_fields)+ + (() if payload.mirrors_spec_draft_model else _drafter_fields))if _carried_fields and not is_removal:# The same spellings the extra-args carry-over walks: a cached repo is not an ordinary folded match,@@ -2705,6 +2714,7 @@ def update_openai_auto_switch_override(mlx_kv_quant = payload.mlx_kv_quant,speculative_type = payload.speculative_type,spec_draft_n_max = payload.spec_draft_n_max,+ spec_draft_model = _kept_tuning["spec_draft_model"],n_parallel = payload.n_parallel,reasoning_budget = (Nonediff --git a/studio/backend/tests/test_account_inference_isolation.py b/studio/backend/tests/test_account_inference_isolation.pyindex 6bcd6563c5c..34b59582618 100644--- a/studio/backend/tests/test_account_inference_isolation.py+++ b/studio/backend/tests/test_account_inference_isolation.py@@ -284,6 +284,16 @@ def test_a_resolved_adapter_base_needs_its_own_grant(monkeypatch, base, expectedinference._require_resolved_base_access(config)+def test_a_named_mlx_drafter_needs_its_own_grant(monkeypatch):+ monkeypatch.setattr(access, "repo_is_public", lambda repo, *a, **k: repo == "org/public")+ monkeypatch.setattr(inference, "_mlx_cached_drafters", lambda path: [{"repo_id": "org/d"}])+ assert run_as(BOB, inference._discoverable_drafters, "org/public") == ["org/d"]+ assert inference._discoverable_drafters("org/public") is None+ body = {"model_path": "org/public", "spec_draft_model": "org/private"}+ with client_for(BOB) as client:+ assert client.post("/api/inference/estimate-memory", json = body).status_code == 404++def test_preview_load_refuses_private_foreign_target_before_gpu_work():with pytest.raises(HTTPException) as exc:asyncio.run(diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.pyindex 102a848b904..4b4ba5feffd 100644--- a/studio/backend/tests/test_active_generations.py+++ b/studio/backend/tests/test_active_generations.py@@ -18,7 +18,7 @@ import pytest_backend = os.path.join(os.path.dirname(__file__), "..")sys.path.insert(0, _backend)-from core.inference.worker import _SLOTS, StopLedger+from core.inference.worker import _SLOTS, StopLedger, _Stopsfrom state import active_generations@@ -2600,6 +2600,8 @@ def test_a_request_reads_as_stopped_once_it_is():assert _stopped(ledger) == set()assert ledger.stop(mine)+ stops = _Stops(ledger, None, None, [])+ assert stops.unread() and stops._refresh() is None and mine in stops and not stops.unread()assert _stopped(ledger) == {mine}, "a stop names one request and no other"assert theirs not in _stopped(ledger)diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex 820b9b375fb..51da3829967 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1195,6 +1195,7 @@ class TestLoadHubDownloadExclusion:"mlx_int8_prefill_requested","mlx_int8_prefill_reason","mlx_context_budget",+ "spec_draft_model","chat_template_override_reason",# Constant True: llama.cpp allocates the window it reports."context_length_enforced",diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex b22cb7bd66c..b3be5d2094f 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -1016,6 +1016,89 @@ class TestEstimateMemoryRoute:monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", _breakdown)return seen+ def test_an_explicit_drafter_is_priced_as_the_load_would_attach_it(self, monkeypatch):+ import core.inference.mlx_inference as mlx_inference+ import core.inference.mlx_memory as mlx_memory+ from core.inference import mlx_speculative as spec++ self._mlx_target(monkeypatch, "/models/thing")+ sources = (spec.DrafterSource("mtp", "/big", True), spec.DrafterSource("mtp", "/ok", False))+ monkeypatch.setattr(+ spec,+ "resolve_speculation",+ lambda mode, *a, **k: spec.SpecResolution(mode, sources if mode == "mtp" else ()),+ )+ monkeypatch.setattr(+ spec, "speculation_refusal", lambda kv_quant, **_: spec.KV_QUANT if kv_quant else None+ )+ monkeypatch.setattr(+ mlx_inference,+ "mlx_drafter_fit",+ lambda d, ceiling, pinned, source, **_: (+ source.path == "/ok" and (pinned or 0) <= 8192,+ None if pinned else 12288,+ ),+ )+ monkeypatch.setattr(ri, "_mlx_estimate_ceiling", lambda model_dir: 32768)+ monkeypatch.setattr(+ ri, "_mlx_estimate_fitted_context", lambda *a, **kw: 24576 if kw else 20480+ )+ priced = []+ monkeypatch.setattr(+ mlx_memory,+ "mlx_memory_breakdown",+ lambda d, *, n_ctx, **kw: priced.append((n_ctx, kw.get("drafter"), kw.get("vision")))+ or mlx_memory.MlxMemoryBreakdown(1, 0, 0, 1, 1, n_ctx = n_ctx),+ )+ assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288+ for pinned in (4096, 65536):+ _estimate(model_path = "org/model", speculative_type = "mtp", max_seq_length = pinned)+ _estimate(model_path = "org/model", speculative_type = "mtp", mlx_kv_quant = "8")+ _estimate(model_path = "org/model")+ _estimate(model_path = "o/m", speculative_type = "ngram", mlx_kv_quant = "auto", mlx_kv_bits = 8)+ _estimate(model_path = "org/model", spec_draft_model = "org/d") # named under Auto+ assert [route for *_, route in priced] == [True, True, True, None, None, True, True]+ assert [row[:2] for row in priced[:6]] == [+ (12288, ("/ok", False)),+ (4096, ("/ok", False)),+ (65536, None),+ (20480, None),+ (20480, None),+ (24576, None),+ ]++ def test_a_drafter_attaches_where_its_priced_fit_holds_and_a_pin_fits_whole(self, monkeypatch):+ import core.inference.mlx_inference as mlx_inference+ import core.inference.mlx_memory as mlx_memory+ from core.inference.mlx_speculative import DrafterSource++ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 << 30)+ priced = []++ def fit(pinned, outcome, *bare):+ monkeypatch.setattr(+ mlx_memory,+ "mlx_fit_outcome",+ lambda d, *, max_ctx, vision, drafter = None, **_: priced.append(+ (max_ctx, drafter, vision)+ )+ or ((outcome, 4096) if drafter else ("fits", bare[0])),+ )+ source = DrafterSource("dflash", "/drafter", False)+ return mlx_inference.mlx_drafter_fit(+ "/m", 32768, pinned, source, load_in_4bit = False, costless = bool(bare)+ )++ attaches = [fit(None, o)[0] for o in ("fits", "fitted", "unsizable", "no_fit")]+ assert attaches == [True, True, None, False]+ assert (fit(8192, "fits"), fit(8192, "fitted")) == ((True, 4096), (False, 4096))+ assert (+ priced+ == [(32768, ("/drafter", False), True)] * 4 + [(8192, ("/drafter", False), True)] * 2+ )+ costless = [fit(None, *c) for c in (("fits", 4096), ("fits", 8192), ("unsizable", 4096))]+ assert costless == [(True, 4096), (False, 4096), (None, 4096)]+def test_mlx_model_not_on_disk_is_not_downloaded(self, monkeypatch):self._mlx_target(monkeypatch, None)resp = _estimate(model_path = "org/model")@@ -4030,6 +4113,11 @@ def test_a_load_is_sized_once_until_its_files_change(monkeypatch, tmp_path):assert mm._size_load(str(tmp_path), None, None, True) != first and len(built) == 2shard.write_bytes(b"xy")assert mm._size_load(str(tmp_path), 4, None, True) != first and len(built) == 3+ (drafter := tmp_path / "drafter").mkdir()+ sized = lambda: mm._size_load(str(tmp_path), 4, None, True, True, (str(drafter), False))+ first = sized()+ (drafter / "config.json").write_text("{}")+ assert sized() != first # a changed local drafter is re-sized tootry:@@ -5025,6 +5113,17 @@ class TestTheContextSearch:monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None+ def test_the_outcome_says_why_no_context_came_back(self, monkeypatch):+ self._linear(monkeypatch)+ assert mm.mlx_fit_outcome("x", budget_bytes = 9000, max_ctx = 8192) == ("fits", None)+ assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("fitted", 6912)+ assert mm.mlx_fit_outcome("x", budget_bytes = 4200, max_ctx = 8192, min_ctx = 4100) == (+ "no_fit",+ None,+ )+ monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)+ assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("unsizable", None)+@_NEEDS_MLX@pytest.mark.parametrize("budget_gib, fitted", [(6, 18_432), (12, 61_952), (24, 149_504)])@@ -5042,6 +5141,23 @@ def test_a_real_checkpoint_is_fitted_to_the_byte(budget_gib, fitted):assert mm.mlx_memory_breakdown(snapshot, n_ctx = float("nan"), load_in_4bit = True) is None+@_NEEDS_MLX+def test_a_companion_drafter_adds_its_stored_weights_and_a_cache_to_the_load():+ _on_bfloat16_chip()+ target = _local_snapshot("mlx-community/Qwen3.5-4B-4bit")+ drafter = _local_snapshot("z-lab/Qwen3.5-4B-DFlash")+ stored = 0+ for shard in glob.glob(os.path.join(drafter, "*.safetensors")):+ with open(shard, "rb") as handle:+ header = json.loads(handle.read(int.from_bytes(handle.read(8), "little")))+ header.pop("__metadata__", None)+ stored += sum(end - start for start, end in (t["data_offsets"] for t in header.values()))+ alone = mm.mlx_memory_breakdown(target, n_ctx = 8192, vision = True)+ paired = mm.mlx_memory_breakdown(target, n_ctx = 8192, vision = True, drafter = (drafter, False))+ assert paired.weights_bytes - alone.weights_bytes == stored+ assert paired.kv_bytes > alone.kv_bytes and paired.compute_bytes > alone.compute_bytes++@_NEEDS_MLX@pytest.mark.parametrize("repo, kv_bits, whole_prompt",@@ -5061,3 +5177,28 @@ def test_the_total_never_falls_as_the_context_grows(repo, kv_bits, whole_prompt)steps = range(mm.MLX_KV_BLOCK, 262_145, mm.MLX_KV_BLOCK)totals = [mm._priced_at(sizing, n).total_bytes for n in steps]assert totals == sorted(totals)+++def test_the_drafter_picker_lists_cached_drafters_of_a_cached_target(tmp_path, monkeypatch):+ from core.inference import mlx_speculative++ (tmp_path / "config.json").write_text('{"vocab_size": 7}')+ found = [+ ("o/T-DFlash", SimpleNamespace(kind = "dflash"), True),+ ("o/T-MTP", SimpleNamespace(kind = "mtp"), False),+ ]+ monkeypatch.setattr(+ mlx_speculative,+ "cached_drafters",+ lambda name, config: found if (name, config) == ("o/T", {"vocab_size": 7}) else [],+ )+ monkeypatch.setattr(+ "utils.utils.hf_cache_snapshot_dir", lambda name: tmp_path if name == "o/T" else None+ )+ assert ri._mlx_cached_drafters("o/T") == [+ {"repo_id": "o/T-DFlash", "kind": "dflash", "named": True},+ {"repo_id": "o/T-MTP", "kind": "mtp", "named": False},+ ]+ assert ri._mlx_cached_drafters("o/absent") == []+ monkeypatch.setattr(ri.account_access, "filter_model_rows", lambda rows: rows[1:])+ assert [row["repo_id"] for row in ri._mlx_cached_drafters("o/T")] == ["o/T-MTP"]diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex a4ccb2c4660..b5712df569e 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -3280,6 +3280,91 @@ def _tiny_lm(return _LM()+def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fits(monkeypatch):+ pytest.importorskip("mlx.core")+ drafters = pytest.importorskip("unsloth_zoo.mlx.speculative")++ from core.inference import mlx_inference, mlx_speculative++ attempts, fits = [], []+ failing = []++ class _Loader:+ @staticmethod+ def from_pretrained(name, **kwargs):+ attempts.append(kwargs["text_only"])+ if not kwargs["text_only"] and failing:+ raise ValueError("mlx-vlm has no such architecture")+ return SimpleNamespace(config = {}), SimpleNamespace()++ loader = types.ModuleType("unsloth_zoo.mlx.loader")+ loader.FastMLXModel = _Loader+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)+ monkeypatch.setattr(+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (8192, 8192, 8192)+ )+ source = mlx_speculative.DrafterSource("dflash", "/drafter", False)+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)+ monkeypatch.setattr(+ mlx_speculative,+ "resolve_speculation",+ lambda mode, *_a, **_k: mlx_speculative.SpecResolution(mode, (source,)),+ )+ monkeypatch.setattr(+ mlx_inference, "mlx_drafter_fit", lambda *a, **k: fits.append(a[1:4]) or (True, 4096)+ )+ monkeypatch.setattr(drafters, "companion_drafter", lambda path, target: SimpleNamespace())+ config = SimpleNamespace(identifier = "org/text", is_vision = False, is_lora = False)++ backend = mlx_inference.MLXInferenceBackend()+ backend.load_model(config, max_seq_length = 0, speculative_type = "dflash", spec_draft_n_max = 5)+ entry = backend.models["org/text"]+ assert attempts == [False] and fits == [(8192, None, source)]+ assert (entry["spec_drafter_kind"], entry["spec_fallback_reason"]) == ("dflash", None)+ assert entry["context_length"] == 4096 and entry["spec_draft_n_max"] == 5+ assert backend._speculative_draft.controller.max_depth == 5++ attempts.clear(), failing.append(True)+ backend.load_model(config, speculative_type = "dflash")+ entry = backend.models["org/text"]+ assert attempts == [False, True] and backend._speculative_draft is None+ assert entry["spec_fallback_reason"] == mlx_speculative.RUNTIME_ERROR+ attempts.clear()+ backend.load_model(config)+ assert attempts == [True] and backend.models["org/text"]["speculative_type"] == "auto"+ failing.clear()+ fit = lambda *a, **kw: fits.append(kw.get("vision")) or (None, None)+ monkeypatch.setattr(mlx_inference, "_fitted_context", fit)+ monkeypatch.setattr(mlx_inference, "mlx_drafter_fit", lambda *a, **k: (False, None))+ backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")+ assert fits[-1] is True # copies alone still load through mlx-vlm, so the fit prices that route+ backend.load_model(config, max_seq_length = 0, spec_draft_model = "org/d")+ assert fits[-2:] == [True, True] # a named drafter enters the speculative route under Auto+ # ... and is a request, so a load that cannot honour it says why.+ failing.append(True)+ backend.load_model(config, spec_draft_model = "org/d")+ monkeypatch.setattr(+ mlx_speculative, "speculation_refusal", lambda **_k: mlx_speculative.KV_QUANT+ )+ reasons = [+ entry["spec_fallback_reason"]+ for entry in (+ dict(backend.models["org/text"]),+ backend.load_model(config, spec_draft_model = "org/d") and backend.models["org/text"],+ )+ ]+ assert reasons == [mlx_speculative.RUNTIME_ERROR, mlx_speculative.KV_QUANT]+ # A diffusion model gets no draft at load, so the resident batch cannot speculate on it either.+ failing.clear()+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)+ monkeypatch.setattr(mlx_inference, "_vlm_generation_is_diffusion", lambda model: True)+ backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")+ entry = backend.models["org/text"]+ assert backend._speculative_draft is None and entry["spec_drafter_kind"] is None+ assert entry["spec_fallback_reason"] == mlx_speculative.RUNTIME_ERROR++def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypatch):pytest.importorskip("mlx.core")from core.inference import mlx_inference@@ -3435,6 +3520,16 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():"chat_template_override_reason": "it could not render a conversation",}assert _mlx_runtime_settings_match(be, req(chat_template_override = "{{ refused }}"))+ # Speculation compares canonically: an absent mode is the resident auto.+ be.models["m"] = {"mlx_kv_quant_requested": "auto", "speculative_type": "auto"}+ assert _mlx_runtime_settings_match(be, req(speculative_type = "default"))+ assert not _mlx_runtime_settings_match(be, req(speculative_type = "mtp"))+ be.models["m"].update(speculative_type = "mtp", spec_draft_n_max = 4)+ assert _mlx_runtime_settings_match(be, req(speculative_type = "draft-mtp", spec_draft_n_max = 4))+ assert not _mlx_runtime_settings_match(be, req(speculative_type = "mtp"))+ assert not _mlx_runtime_settings_match(+ be, req(speculative_type = "mtp", spec_draft_n_max = 4, spec_draft_model = "org/drafter")+ )resp = LoadResponse(status = "loaded",@@ -5261,6 +5356,7 @@ def _drive_vlm_generation(max_new_tokens = 1,session = None,video = None,+ **request,):import sysimport types@@ -5296,11 +5392,73 @@ def _drive_vlm_generation(image = image,video = video,max_new_tokens = max_new_tokens,+ **request,))return seen+def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):+ pytest.importorskip("mlx_lm")+ import sys+ import types++ from core.inference import mlx_inference++ class _Ids(list):+ def tolist(self):+ return list(self)++ utils = types.ModuleType("mlx_vlm.utils")+ utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7, 8, 9])], "extra": 1}+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)+ prepared = []+ draft = SimpleNamespace(draft_kind = "mtp", draft_n = 6, draft_n_accepted = 4)+ draft.prepare = lambda ids, sampling, given: prepared.append(+ (ids, sampling.temperature, sampling.top_k)+ ) or setattr(draft, "given", given)+ backend = mlx_inference.MLXInferenceBackend()+ backend._speculative_draft = draft++ # top_k -1 is the API's "disabled", which the drafter's sampling settings spell 0.+ seen = _drive_vlm_generation(+ backend, monkeypatch, temperature = 0.5, top_k = -1, repetition_penalty = 1.1+ )+ assert seen["draft_model"] is draft and seen["extra"] == 1 and prepared == [([7, 8, 9], 0.5, 0)]+ # The draft is told every processor the reply runs: none reaches mlx-vlm as a shortcut.+ assert "repetition_penalty" not in seen and draft.given == seen["logits_processors"] != []+ monkeypatch.setattr(+ mlx_inference, "_mlx_sampling_processors", lambda **_: [lambda tokens, logits: logits]+ )+ timings = backend.last_generation_stats["timings"]+ assert (timings["draft_n"], timings["draft_n_accepted"]) == (6, 4)+ for request in (+ {"repetition_penalty": 1.1},+ {"tools": [{"type": "function"}]},+ {"tool_protocol_active": True},+ ):+ seen = _drive_vlm_generation(backend, monkeypatch, **request)+ assert "max_tokens" in seen and "draft_model" not in seen+ assert "draft_n" not in backend.last_generation_stats["timings"]+ assert len(prepared) == 1+ opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r: None+ backend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)+ assert backend.batch_unavailable_reason([{}, {}]) == "no stream"+ monkeypatch.setattr(+ _batch_engine(monkeypatch), "BatchStream", lambda *a, **k: opened.append(k["speculative"])+ )+ backend._generate_session_batch = lambda r, kind, **_: opened.append(kind) or ()+ mlx_inference._VisionBatchSession(backend, width = 2)+ list(backend.generate_chat_batch([{}, {}]))+ assert opened == [draft, mlx_inference._VisionBatchSession]+ del backend._generate_session_batch+ steps, cancel = [], threading.Event()+ step = lambda waiting: steps.append(waiting) or setattr(session, "rows_in_flight", 0) or ()+ session = SimpleNamespace(admit = lambda *a: None, rows_in_flight = 1, step = step, close = list)+ list(backend._generate_session_batch([{}], lambda *a, **k: session, cancel_event = cancel))+ assert steps == [cancel.is_set] # a cancelled reply ends its speculative step early++def _stub_prepare_inputs(monkeypatch, per_medium = 520):import sysimport types@@ -5910,6 +6068,7 @@ def test_what_the_fit_is_asked_and_when_it_is_asked_at_all(monkeypatch, tmp_path"dir": str(tmp_path),}]+ assert fit(vision = True) and priced[0]["vision"] is True# The width the cache will take is the width the fit is priced at.for answer, bits in ((("full", "", True), 4), (("partial", "w", True), 4)):verdict["answer"] = answer@@ -6214,14 +6373,19 @@ class _TwoRowStream:):self._events, self._retired, self.withdrawn = events, set(retired), []- def step(self):- return iter(self._events)+ def iter_step(self, waiting):+ yield from self._events+ self.exhausted = Truedef withdraw(self, row):+ assert self.exhausted, "the batch cannot change mid-step"self.withdrawn.append(row)if row in self._retired:return None- return SimpleNamespace(prompt_token_count = 3, token_ids = [1, 2])+ return SimpleNamespace(prompt_token_count = 3, token_ids = [1, 2], **_DRAFTS)+++_DRAFTS = {"draft_tokens": 8, "accepted_draft_tokens": 3}def _open_vision_session(rows):@@ -6231,6 +6395,7 @@ def _open_vision_session(rows):session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)session._rows = {row.handle: row for row in rows}session._by_row, session._settled = {row.row: row for row in rows}, {}+ session._speculative = Nonereturn session@@ -6252,8 +6417,9 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, stofor number, handle in enumerate(handles)])- other = 1 - stopping+ other, session._speculative = 1 - stopping, "draft"result = SimpleNamespace(finish_reason = "length", prompt_token_count = 3, token_ids = [1, 2])+ vars(result).update(_DRAFTS)session.stream = _TwoRowStream([SimpleNamespace(@@ -6268,6 +6434,8 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, stoassert (handles[stopping], None) in reported, "the stopped row is reported finished"assert session._settled[handles[stopping]]["finish_reason"] == "stop"+ timings = session._settled[handles[stopping]]["timings"]+ assert (timings["draft_n"], timings["draft_n_accepted"]) == (8, 3)assert handles[stopping] not in session._rows, "and it has left the batch"assert handles[other] in session._rows, "its neighbour is still in the batch"assert (handles[other], "still going") in reporteddiff --git a/studio/backend/tests/test_mlx_resident_batch.py b/studio/backend/tests/test_mlx_resident_batch.pyindex 2dfa88027a1..0a7055d8bc6 100644--- a/studio/backend/tests/test_mlx_resident_batch.py+++ b/studio/backend/tests/test_mlx_resident_batch.py@@ -79,7 +79,7 @@ class _Session:self.rows.remove(handle)self.settled[handle] = self.stats_at_retire.get(handle, {})- def step(self):+ def step(self, waiting):for handle in list(self.rows):events = self.script.get(handle) or []if not events:diff --git a/studio/backend/tests/test_mlx_speculative.py b/studio/backend/tests/test_mlx_speculative.pynew file mode 100644index 00000000000..3087ac9f8ea--- /dev/null+++ b/studio/backend/tests/test_mlx_speculative.py@@ -0,0 +1,186 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import json+import os+from types import SimpleNamespace++import pytest++from core.inference import mlx_speculative as spec++_DFLASH = {"architectures": ["DFlashDraftModel"], "dflash_config": {}, "vocab_size": 10}+_EAGLE3 = {"architectures": ["Eagle3Speculator"], "speculators_model_type": "eagle3"}+_ASSISTANT = {"architectures": ["Gemma4AssistantForCausalLM"], "model_type": "gemma4_assistant"}+++_TARGET = "org/Qwen3.5-4B-4bit"+++def cache(tmp_path, monkeypatch):+ """Fills a Hugging Face cache with ``{repo: config}`` beside the target, and returns a resolver+ for the target."""+ snapshots = {}++ def fill(repos, builtin = False):+ for repo, config in {_TARGET: {"vocab_size": 10}, **repos}.items():+ snapshots[repo] = tmp_path / repo.replace("/", "--")+ snapshots[repo].mkdir(parents = True, exist_ok = True)+ (snapshots[repo] / "config.json").write_text(json.dumps(config))+ (snapshots[repo] / "model.safetensors").touch()+ monkeypatch.setattr(spec, "has_builtin_head", lambda model_dir: builtin)+ return lambda mode, named = None: spec.resolve_speculation(+ mode, named, model_dir = str(snapshots[_TARGET]), target_name = _TARGET+ )++ monkeypatch.setattr(spec, "_cached_repos", lambda: iter(list(snapshots)))+ monkeypatch.setattr(+ "utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: snapshots.get(repo)+ )+ return fill+++def test_an_explicit_kind_tries_the_named_drafter_then_the_head_then_cached_companions(cache):+ _resolve = cache(+ {+ "a/Qwen3.5-4B-DFlash": _DFLASH,+ "b/Qwen3.5-4B-Eagle3": _EAGLE3,+ "e/Qwen3.5-4B-Eagle3": {+ "architectures": ["LlamaForCausalLMEagle3"],+ "model_type": "llama",+ },+ "c/Qwen3.5-9B-DFlash": _DFLASH, # another model+ "d/Qwen3.5-4B-DFlash-bf16": {**_DFLASH, "vocab_size": 11}, # another vocabulary+ "g/Qwen3.5-4B-assistant": _ASSISTANT,+ },+ builtin = True,+ )+ assert [os.path.basename(s.path) for s in _resolve("dflash").sources] == [+ "a--Qwen3.5-4B-DFlash"+ ]+ assert _resolve("dflash").copies and spec.mlx_spec_mode("mtp+ngram") == "mtp"+ assert spec.mlx_spec_mode("eagle3+ngram") == "auto"+ assert [os.path.basename(s.path) for s in _resolve("eagle3").sources] == [+ "b--Qwen3.5-4B-Eagle3"+ ]+ named = _resolve("eagle3", "b/Qwen3.5-4B-Eagle3")+ assert [s.kind for s in named.sources] == ["eagle3"] and named.copies and named.speculative+ mtp = [(os.path.basename(s.path), s.builtin) for s in _resolve("draft-mtp").sources]+ assert mtp == [("org--Qwen3.5-4B-4bit", True), ("g--Qwen3.5-4B-assistant", False)]+ assert _resolve("mtp", "g/Qwen3.5-4B-assistant").sources[0].builtin is False+ # A named drafter of another kind is refused with a reason, never substituted silently.+ assert _resolve("dflash", "b/Qwen3.5-4B-Eagle3").reason == spec.DRAFTER_INCOMPATIBLE+ assert _resolve("dspark").reason == spec.DRAFTER_NOT_FOUND+ assert _resolve("dspark").speculative+++def test_auto_takes_every_cached_kind_in_preference_order(cache):+ _resolve = cache({"b/Qwen3.5-4B-Eagle3": _EAGLE3, "a/Qwen3.5-4B-DFlash": _DFLASH}, True)+ assert [s.kind for s in _resolve("auto").sources] == ["mtp", "dflash", "eagle3"] and not (+ _resolve("auto").reason or _resolve("off").speculative+ )+ assert [spec.speculates_on_route("auto", vision) for vision in (False, True)] == [False, True]+ listed = [(repo, s.kind) for repo, s, _ in spec.cached_drafters(_TARGET, {"vocab_size": 10})]+ assert listed == [("a/Qwen3.5-4B-DFlash", "dflash"), ("b/Qwen3.5-4B-Eagle3", "eagle3")]+ mtp_head = {"h/Qwen3.5-4B-MTP-bf16": {"model_type": "qwen3_5_mtp"}}+ _resolve = cache({"g/Qwen3.5-4B-assistant": _ASSISTANT, **mtp_head}, True)+ for mode in (None, "bogus", "auto"):+ assert [(s.builtin, s.kind) for s in _resolve(mode).sources] == [(True, "mtp")] + [+ (False, "mtp")+ ] * 2 + [+ (False, "dflash"),+ (False, "eagle3"),+ ]+ assert _resolve("ngram-mod").speculative+++def test_cached_drafters_are_matched_by_shape_and_weights_then_marked_by_name(cache, tmp_path):+ shaped = {**_DFLASH, "hidden_size": 8, "num_target_layers": 4}+ narrow = {"target_hidden_size": 8, "transformer_layer_config": {"hidden_size": 2}}+ cache(+ {+ "a/Qwen3.5-4B-DFlash": shaped,+ "n/Other-DFlash": shaped, # fits, but is not named for the target+ "h/Qwen3.5-4B-DFlash": {**shaped, "hidden_size": 9},+ "t/Qwen3.5-4B-DFlash": {**shaped, "num_target_layers": 5},+ "l/Qwen3.5-4B-DFlash": {**shaped, "dflash_config": {"target_layer_ids": [1, 5]}},+ "w/Qwen3.5-4B-DFlash": shaped,+ "b/Qwen3.5-4B-assistant": {**_ASSISTANT, "backbone_hidden_size": 9},+ "m/Qwen3.5-4B-MTP": {"model_type": "x_mtp", "text_config": {"num_hidden_layers": 5}},+ "e/Qwen3.5-4B-Eagle3": {+ **_EAGLE3,+ **narrow,+ "eagle_aux_hidden_state_layer_ids": [1, 4],+ },+ }+ )+ (tmp_path / "w--Qwen3.5-4B-DFlash" / "model.safetensors").unlink()+ target = {"vocab_size": 10, "text_config": {"hidden_size": 8, "num_hidden_layers": 4}}+ found = spec.cached_drafters("unsloth/Qwen3.5-4B-UD-MLX-4bit", target)+ assert [(repo, named) for repo, _, named in found] == [+ ("a/Qwen3.5-4B-DFlash", True),+ ("e/Qwen3.5-4B-Eagle3", True),+ ("n/Other-DFlash", False),+ ]+ assert len(spec.discover_companions("unsloth/Qwen3.5-4B-UD-MLX-4bit", target)) == 2+ assert len(spec.discover_companions("unsloth/Qwen3.5-4B-UD-MLX-4bit", target, found[1])) == 1+ assert spec._stem("g/gemma-4-31B-it-qat-q4_0-unquantized-assistant") == "gemma431bit"+++def test_a_drafter_passed_over_keeps_its_reason_on_the_one_that_attaches(monkeypatch):+ drafters = pytest.importorskip("unsloth_zoo.mlx.speculative")++ built = []++ def factory(path, target):+ if path == "broken":+ raise ValueError("mismatched vocabulary")+ built.append(path)+ return SimpleNamespace(kind = "dflash", max_depth = 15)++ monkeypatch.setattr(drafters, "companion_drafter", factory)+ monkeypatch.setattr(drafters, "native_mtp_drafter", factory)+ sources = tuple(spec.DrafterSource("dflash", path, False) for path in ("big", "broken", "ok"))+ fits = lambda source: (source.path != "big", 4096)++ draft, kind, reason, context = spec.build_draft(+ None, spec.SpecResolution("dflash", sources), fits = fits, draft_n_max = 6+ )+ assert (kind, reason, context, built) == ("dflash", spec.DRAFTER_INCOMPATIBLE, 4096, ["ok"])+ assert (draft.controller.max_depth, draft.controller.max_copy) == (6, 0)+ for d, n, e in ((object(), 3, True), (object(), 4, False), (draft.drafter, 15, True)):+ assert spec._draft(d, False, n).controller.fixed_depth is e+ draft, kind, reason, _ = spec.build_draft(+ None, spec.SpecResolution("dflash", sources[:1], copies = True), fits = fits+ )+ assert (draft.drafter, kind, reason) == (None, "ngram", spec.DRAFTER_NO_MEMORY)+ assert (draft.controller.max_depth, draft.controller.max_copy) == (0, 16)+ auto = spec.SpecResolution("auto", sources[:1], copies = True)+ unfit = [spec.build_draft(None, auto, fits = lambda _: (ok, None)) for ok in (False, None)]+ assert unfit == [(None, None, spec.AUTO_CONTEXT_COST, None), (None,) * 4]+ capped = spec.build_draft(+ None, spec.SpecResolution("ngram", copies = True), fits = fits, draft_n_max = 5+ )+ assert capped[0].controller.max_copy == 5++ def encoder_decoder(inputs, encoder_outputs = None):+ pass++ def cross_attending(inputs, cross_attention_states = None):+ pass++ for target in (encoder_decoder, SimpleNamespace(language_model = cross_attending)):+ refused = spec.build_draft(+ target, spec.SpecResolution("dflash", sources[2:], copies = True), fits = fits+ )+ assert refused == (None, None, spec.RUNTIME_ERROR, None)+++def test_a_drafter_carrying_its_own_model_module_is_never_found_or_attached(cache):+ # mlx-vlm imports a config's model_file on load, so such a checkpoint would run its own code.+ custom = {**_DFLASH, "model_file": "model.py"}+ _resolve = cache({"a/Qwen3.5-4B-DFlash": custom})+ assert spec.cached_drafters(_TARGET, {"vocab_size": 10}) == []+ assert not _resolve("auto").sources and not _resolve("dflash").sources+ assert _resolve("dflash", "a/Qwen3.5-4B-DFlash").reason == spec.DRAFTER_NOT_FOUNDdiff --git a/studio/backend/tests/test_mlx_worker_audio_commands.py b/studio/backend/tests/test_mlx_worker_audio_commands.pyindex 610abdcfa70..21186d1ca28 100644--- a/studio/backend/tests/test_mlx_worker_audio_commands.py+++ b/studio/backend/tests/test_mlx_worker_audio_commands.py@@ -203,7 +203,7 @@ def test_mlx_loop_ticks_only_when_loaded_recent_and_idle(monkeypatch, blocked, edef rows_in_flight(self):return int(busy[0])- def step(self):+ def step(self, waiting = None):passmonkeypatch.setattr(worker, "_ResidentBatch", Batch)diff --git a/studio/backend/tests/test_model_override_schema_compatibility.py b/studio/backend/tests/test_model_override_schema_compatibility.pyindex 10a07abe1d5..c97b864a423 100644--- a/studio/backend/tests/test_model_override_schema_compatibility.py+++ b/studio/backend/tests/test_model_override_schema_compatibility.py@@ -410,6 +410,12 @@ def test_a_client_that_does_know_the_reasoning_pair_still_clears_by_omission(oveassert field not in after, f"{field} survived an explicit clear"+def test_a_client_that_does_not_know_the_drafter_cannot_erase_it(override_store):+ _put("org/m", speculative_type = "eagle3", spec_draft_model = "d", mirrors_spec_draft_model = True)+ _put("org/m", speculative_type = "eagle3")+ assert settings.get_model_override("org/m")["spec_draft_model"] == "d"++def test_the_reasoning_flag_is_not_itself_a_saved_field(override_store):_put(MODEL, **PRE_TUNING_PAYLOAD, mirrors_reasoning_budget = True)assert settings.get_model_override(MODEL)diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 39b7ac42598..ef3db688089 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -7317,6 +7317,17 @@ def test_spec_draft_n_max_only_stored_for_mtp_modes():assert "spec_draft_n_max" not in ngram+def test_mlx_speculative_modes_and_drafter_survive_to_the_load():+ stored = {"spec_draft_n_max": 4, "spec_draft_model": " d "}+ for mode, is_gguf, kept in (+ ("eagle3", False, {"spec_draft_n_max": 4, "spec_draft_model": "d"}),+ ("eagle3", True, {}),+ ):+ entry = settings.normalize_model_override({"speculative_type": mode, **stored})+ kwargs = settings.model_override_load_kwargs(entry, is_gguf = is_gguf)+ assert kwargs == kept | ({} if is_gguf else {"speculative_type": mode})++def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers():# Manual GPU memory with Auto layers hands the context to llama.cpp --fit, so# the load sends the context pin (or 0), not the stored max seq length.diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.pyindex e2bae9045e3..be92e800514 100644--- a/studio/backend/tests/test_parallel_slots_per_load.py+++ b/studio/backend/tests/test_parallel_slots_per_load.py@@ -615,4 +615,5 @@ def test_a_load_reply_carries_the_two_window_facts_and_the_width_that_selects_thbody["parallel_slots"],) == (True, True, 4)assert reply(can_batch = False, context_unbounded_when_batched = False)["parallel_slots"] == 1+ assert reply(spec_draft_model = "org/drafter")["spec_draft_model"] == "org/drafter"assert _InferenceRuntimeFields.model_fields["context_unbounded_when_batched"].default is Falsediff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.pyindex ecfeb4c01c2..ad79892aa8d 100644--- a/studio/backend/utils/openai_auto_switch_settings.py+++ b/studio/backend/utils/openai_auto_switch_settings.py@@ -336,7 +336,9 @@ def set_openai_auto_switch(VALID_KV_CACHE_DTYPES = frozenset({"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"})-VALID_SPECULATIVE_TYPES = frozenset(+# The GGUF control never offers these, and a GGUF load drops them.+MLX_ONLY_SPEC_TYPES = frozenset({"eagle3"})+VALID_SPECULATIVE_TYPES = MLX_ONLY_SPEC_TYPES | frozenset({"auto","mtp",@@ -360,7 +362,9 @@ VALID_SPECULATIVE_TYPES = frozenset(# Only these consume spec_draft_n_max (mirrors DRAFT_N_MAX_SPEC_TYPES in the UI).DRAFT_N_MAX_SPEC_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp", "dspark", "draft-dspark", "dflash", "draft-dflash"}+ | MLX_ONLY_SPEC_TYPES)+DRAFTER_MODEL_SPEC_TYPES = DRAFT_N_MAX_SPEC_TYPES | {"auto", "default"}# Only these load a separate draft model, and so a draft context for the dtype to apply to. Mirrors SEPARATE_DRAFT_MODEL_SPEC_TYPES in the UI.SEPARATE_DRAFT_MODEL_SPEC_TYPES = frozenset({"dspark", "draft-dspark", "dflash", "draft-dflash"})# Mirrors _LOAD_MODE_VALUES in llama_server_args.py. "auto" is the llama.cpp default and is not stored: an entry holding it would pin what a build may redefine.@@ -470,6 +474,10 @@ def normalize_model_override(spec_draft_n_max = _bounded_int(payload.get("spec_draft_n_max"), minimum = 1, maximum = 16)if spec_draft_n_max:entry["spec_draft_n_max"] = spec_draft_n_max+ spec_draft_model = payload.get("spec_draft_model")+ if speculative_type in DRAFTER_MODEL_SPEC_TYPES and isinstance(spec_draft_model, str):+ if 0 < len(spec_draft_model.strip()) <= 1024:+ entry["spec_draft_model"] = spec_draft_model.strip()# Same rule, narrower set: the dtype needs a separate draft model, and only the sidecar modes always load one.if speculative_type in SEPARATE_DRAFT_MODEL_SPEC_TYPES:spec_draft_cache_type = _clean_str(@@ -648,12 +656,19 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di)override = {**override, "llama_extra_args": kept}+ # MLX drafter settings; a GGUF load reads an MLX-only mode as no stored mode, keeping its launch flags.+ gguf_drops = set()+ if is_gguf:+ gguf_drops.add("spec_draft_model")+ if override.get("speculative_type") in MLX_ONLY_SPEC_TYPES:+ gguf_drops.update(("speculative_type", "spec_draft_n_max"))for source, target in (("llama_extra_args", "llama_extra_args"),("kv_cache_dtype", "cache_type_kv"),("n_parallel", "n_parallel"),("speculative_type", "speculative_type"),("spec_draft_n_max", "spec_draft_n_max"),+ ("spec_draft_model", "spec_draft_model"),("reasoning_budget", "reasoning_budget"),("reasoning_budget_message", "reasoning_budget_message"),("tensor_parallel", "tensor_parallel"),@@ -661,7 +676,7 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di("chat_template_override", "chat_template_override"),("mlx_int8_prefill", "mlx_int8_prefill"),):- if override.get(source) is not None:+ if override.get(source) is not None and source not in gguf_drops:kwargs[target] = override[source]mlx_kv_quant = _mlx_kv_quant_of(override)diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex a22c43abf51..aacb4b30ae5 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -48,6 +48,7 @@ import { loadManagedLlamaFlags } from "@/features/model-picker/api/llama-flags";import { fetchLoadExtraArgs } from "@/features/model-picker/api/model-overrides";import { sanitizeStoredExtraArgs } from "@/features/model-picker/model-config/llama-extra-args";import { usePlatformStore } from "@/config/env";+import { resolveSpeculativeType } from "@/lib/speculative-modes";import { getSkillsSnapshot, settleSkillsForText } from "./skills-api";@@ -2693,6 +2694,8 @@ const VISIBLE_MODEL_RUNTIME_KEYS = ["specFallbackReason","specDraftNMax","loadedSpecDraftNMax",+ "specDraftModel",+ "loadedSpecDraftModel",] as const satisfies readonly (keyof ChatRuntimeState)[];type VisibleModelRuntimeState = Pick<@@ -3657,10 +3660,25 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{config.customContextLength ?? null,effectiveMaxSeqLength,);- const effectiveSpeculativeType =- config.speculativeType ?? specSettings.speculativeType;+ const candidateIsMlx = isServedByMlx(+ candidate.kind === "gguf",+ platform.deviceType,+ platform.chatOnlyReason,+ );+ // MLX falls back to the standing preference: the live store may hold another model's settings.+ const standingSpec = candidateIsMlx+ ? resolveSpeculativeSettingsForLoad({ usePersistedPreference: true })+ : specSettings;+ const effectiveSpeculativeType = resolveSpeculativeType(+ config.speculativeType,+ standingSpec.speculativeType ?? "auto",+ candidateIsMlx,+ );const effectiveSpecDraftNMax =- config.specDraftNMax ?? specSettings.specDraftNMax;+ config.specDraftNMax ?? standingSpec.specDraftNMax;+ const mlxDrafter = candidateIsMlx+ ? { spec_draft_model: config.specDraftModel ?? null }+ : {};const effectiveChatTemplateOverride = config.chatTemplateOverride?.trim()? config.chatTemplateOverride: null;@@ -3735,6 +3753,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{mlx_int8_prefill: config.mlxInt8Prefill ?? false,speculative_type: effectiveSpeculativeType,spec_draft_n_max: effectiveSpecDraftNMax,+ ...mlxDrafter,reasoning_budget:candidate.kind === "gguf" && !isDiffusion ? config.reasoningBudget : -1,reasoning_budget_message:@@ -3773,8 +3792,8 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{options?.abortSignal?.throwIfAborted();applyAutoLoadRuntimeState(options, () => {// Persist the global preference only when the value came from global settings, or autoloading- // a remembered model rewrites the default.- if (config.speculativeType == null) {+ // a remembered model rewrites the default. MLX only reads it.+ if (config.speculativeType == null && !candidateIsMlx) {saveSpeculativeType(effectiveSpeculativeType);}// Self-gates on is_gguf (skips diffusion), so persists only for a real GGUF load.diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsxindex e8420da53a1..45b50113f17 100644--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx@@ -128,6 +128,7 @@ import {} from "./types/runtime";import { effectiveMinPMode, isMinPMode } from "./lib/min-p-policy";+import { mlxSpecFallbackMessage } from "./lib/mlx-spec-fallback";export { defaultInferenceParams, type Preset } from "./presets/preset-policy";export type { InferenceParams } from "./types/runtime";@@ -642,13 +643,19 @@ export function ChatSettingsPanel({// ngram-mod runs no drafter, so only the binary stand-down reaches it. Without// this the panel shows ngram selected, no speculation running, and no reason.speculativeType === "ngram");+ const loadedIsMlx = useChatRuntimeStore((s) => s.loadedIsMlx);+ const mlxSpecFallback =+ !isExternalModel && loadedIsMlx && specFallbackReason != null+ ? mlxSpecFallbackMessage(specFallbackReason, specDrafterKind)+ : null;const showContextVramWarning =!isExternalModel &&isGguf &&maxContextLength != null &&loadedEffectiveContext != null &&loadedEffectiveContext > maxContextLength;- const showLoadedDiagnostics = showSpecFallback || showContextVramWarning;+ const showLoadedDiagnostics =+ showSpecFallback || mlxSpecFallback != null || showContextVramWarning;const hasModelContent = showLoadedDiagnostics;const setActivePresetSource = useChatRuntimeStore((s) => s.setActivePresetSource,@@ -1179,6 +1186,11 @@ export function ChatSettingsPanel({)}</div>)}+ {mlxSpecFallback && (+ <div className="rounded-lg bg-amber-500/[0.08] px-3 py-2 text-ui-12 leading-[1.4] text-nav-fg/80">+ <p>{mlxSpecFallback}</p>+ </div>+ )}{showContextVramWarning && (<p className="text-ui-11 text-amber-500">{isUnifiedMemory ? (diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex 9427ef5c1b7..85a3ae8d24e 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -21,6 +21,7 @@ import {isSilencedDesktopUpdateFailure,} from "@/lib/desktop-update-activity";import { subscribeModelLifecycle } from "@/lib/model-lifecycle-events";+import { resolveSpeculativeType } from "@/lib/speculative-modes";import {type TransferSample,appendSample,@@ -2245,6 +2246,7 @@ export function useChatModelRuntime() {: stateBeforeUnload.speculativeType;let loadSpecDraftNMax =pendingLoadConfig?.specDraftNMax ?? stateBeforeUnload.specDraftNMax;+ let loadSpecDraftModel = stateBeforeUnload.specDraftModel;let loadNParallel =pendingLoadConfig?.nParallel ?? stateBeforeUnload.nParallel;let loadReasoningBudget =@@ -2506,6 +2508,8 @@ export function useChatModelRuntime() {loadedSpeculativeType: persistedSpeculativeType,specDraftNMax: null,loadedSpecDraftNMax: null,+ specDraftModel: null,+ loadedSpecDraftModel: null,// Per-model too: a different model follows the server default unless its staged config overrides it.nParallel: null,loadedNParallel: null,@@ -2577,6 +2581,14 @@ export function useChatModelRuntime() {platform.deviceType,platform.chatOnlyReason,);+ // Only a same-model reload without a staged config reuses the resident's settings; otherwise the+ // drafter is this model's own and a standing mode goes through the resolver.+ if (targetIsMlx && (pendingLoadConfig != null || switchingModelOrVariant)) {+ loadSpecDraftModel = pendingLoadConfig?.specDraftModel ?? null;+ if (loadSpeculativeType && pendingLoadConfig?.speculativeType == null) {+ loadSpeculativeType = resolveSpeculativeType(null, loadSpeculativeType, true);+ }+ }const explicitCtxPin = loadRequestContextPin(loadCustomContextLength,targetIsMlx,@@ -2658,6 +2670,7 @@ export function useChatModelRuntime() {mlx_int8_prefill: loadMlxInt8Prefill,speculative_type: loadSpeculativeType,spec_draft_n_max: loadSpecDraftNMax,+ ...(targetIsMlx ? { spec_draft_model: loadSpecDraftModel } : {}),n_parallel: loadNParallel,reasoning_budget:isGguf && !targetIsDiffusion ? loadReasoningBudget : -1,@@ -2707,8 +2720,9 @@ export function useChatModelRuntime() {// The load applied this spec mode, so persist the user's standing preference now: the requested// intent, not the resolved echo, since saveSpeculativeType keeps only the universal auto/ngram/off.- // Skipped for a per-model config (keepSpeculative), whose choice must not overwrite the global.- if (!keepSpeculative) {+ // Skipped for a per-model config (keepSpeculative), whose choice must not overwrite the global,+ // and for MLX, which only reads it.+ if (!(keepSpeculative || targetIsMlx)) {saveSpeculativeType(loadSpeculativeType);}// Persist the GPU Memory mode only on a successful load, so an abandoned selection does not stick.@@ -2884,6 +2898,8 @@ export function useChatModelRuntime() {loadedSpeculativeType: loadedSpec,specDraftNMax: loadResponse.spec_draft_n_max ?? null,loadedSpecDraftNMax: loadResponse.spec_draft_n_max ?? null,+ specDraftModel: loadResponse.spec_draft_model ?? null,+ loadedSpecDraftModel: loadResponse.spec_draft_model ?? null,// Keep the click-time value: the echo is the resolved count, and adopting it would pin a blank// "server default" control.nParallel: committedSlots,@@ -3053,6 +3069,7 @@ export function useChatModelRuntime() {rollbackState.loadedSpeculativeType,spec_draft_n_max:rollbackState.loadedSpecDraftNMax,+ spec_draft_model: rollbackState.loadedSpecDraftModel,n_parallel: rollbackState.loadedNParallel,reasoning_budget:rollbackState.loadedReasoningBudgetRequested ?? -1,@@ -3112,6 +3129,7 @@ export function useChatModelRuntime() {// from its reload echo.speculativeType: rollbackState.loadedSpeculativeType ?? null,specDraftNMax: rollbackState.loadedSpecDraftNMax ?? null,+ specDraftModel: rollbackState.loadedSpecDraftModel ?? null,// Control keeps its intent; only the baseline takes the echo.nParallel: previousNParallel,loadedNParallel: rollbackState.loadedNParallel ?? null,@@ -3149,6 +3167,7 @@ export function useChatModelRuntime() {loadedSpeculativeType: rollbackSpeculativeType,loadedSpecDraftNMax:rollbackResponse.spec_draft_n_max ?? null,+ loadedSpecDraftModel: rollbackResponse.spec_draft_model ?? null,loadedKvCacheDtype: rollbackResponse.cache_type_kv ?? null,...mlxRuntimeStateFrom(rollbackResponse),// After the spread, which seeds the control from the echo; the control keeps its intent, likediff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex 03dd97b25b3..48bdb82937a 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -496,6 +496,14 @@ export function applyActiveModelStatusToStore(specDraftNMax: status.spec_draft_n_max ?? null,}),}),+ ...(seedLoadParams &&+ status.spec_draft_model !== undefined && {+ loadedSpecDraftModel: status.spec_draft_model ?? null,+ ...((hydratingExistingModel ||+ prevState.specDraftModel === prevState.loadedSpecDraftModel) && {+ specDraftModel: status.spec_draft_model ?? null,+ }),+ }),...(seedLoadParams &&status.cache_type_kv !== undefined && {loadedKvCacheDtype: status.cache_type_kv,diff --git a/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.tsnew file mode 100644index 00000000000..c8ecfb4dee7--- /dev/null+++ b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts@@ -0,0 +1,40 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++const MLX_DRAFTER_LABELS: Record<string, string> = {+ mtp: "MTP",+ dflash: "DFlash",+ dspark: "DSpark",+ eagle3: "EAGLE-3",+};++/** Copy for an MLX load's spec_fallback_reason, or null for a code MLX does not report. Several+ * backend paths share each code, so the copy states only what every one of them implies, and the+ * outcome follows what runs: the reason can name a drafter passed over while another attached. */+export function mlxSpecFallbackMessage(+ reason: string,+ drafterKind: string | null,+): string | null {+ const outcome =+ drafterKind == null+ ? "so it is running without speculative decoding"+ : drafterKind === "ngram"+ ? "so it is drafting with n-gram copies only"+ : `so it is drafting with ${MLX_DRAFTER_LABELS[drafterKind] ?? drafterKind} instead`;+ switch (reason) {+ case "drafter_not_found":+ return `A drafter this load asked for was not found locally, or is not a drafter, ${outcome}. Download it, or check the drafter named in this model's settings, then reload.`;+ case "drafter_incompatible":+ return `A drafter was found but could not be used with this model or mode, ${outcome}. Try a drafter made for this model and mode, then reload.`;+ case "drafter_no_memory":+ return `The drafter could not be confirmed to fit in memory beside this model, ${outcome}. When memory is the limit, a shorter context length or a smaller drafter can help.`;+ case "runtime_error":+ return `Speculative decoding could not start for this load, ${outcome}.`;+ case "kv_quant":+ return "Speculative decoding cannot run with KV cache quantization, which this load asked for. Setting KV Cache Dtype to Auto in this model's settings removes that obstacle.";+ case "auto_context_cost":+ return `Auto left out a drafter that does not fit beside this model at its full context length, ${outcome}. To try it anyway, choose its mode in this model's settings, with a shorter or automatic context length.`;+ default:+ return null;+ }+}diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex 2834e824b80..d5bb60b4214 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -11,6 +11,10 @@ import {} from "./llama-extra-args-normalize";import { reconcileTensorSplit } from "@/hooks/gpu-tensor-split";import type { GpuIndexKind } from "@/hooks/gpu-selection";+import {+ mlxSpeculativeMode,+ resolveSpeculativeType,+} from "@/lib/speculative-modes";import type { InferenceStatusResponse } from "../types/api";@@ -27,6 +31,7 @@ type ResidentRuntime = Pick<| "mlx_int8_prefill_requested"| "speculative_type"| "spec_draft_n_max"+ | "spec_draft_model"| "requested_parallel_slots"| "requested_n_batch"| "requested_n_ubatch"@@ -48,6 +53,7 @@ type ResidentRuntime = Pick<| "requested_gpu_ids"| "gpu_ids"| "is_gguf"+ | "is_mlx"| "is_diffusion"| "diffusion_requested_ngl"| "diffusion_split_supported"@@ -182,12 +188,17 @@ export function residentSpeculativeNeedsRepair(| "spec_dflash_retry_pending"| "spec_dspark_sidecar_absent"| "spec_drafter_kind"+ | "is_mlx">,resolvedSpeculativeType: string | null,/** Whether the load carries a `gguf_path`, which the route sets from the identifier alone:* `source.gguf_path if model_identifier.lower().endswith(".gguf") else None`. */sendsGgufPath = false,): boolean {+ // Every arm below is llama.cpp's; an identical MLX load dedupes.+ if (status.is_mlx === true) {+ return false;+ }const mode = resolvedSpeculativeType ?? "auto";// Two arms that record no fallback reason, so the reason check below cannot see them. The probe// arm is not gated on a mode; the DFlash one is, as the backend gates it.@@ -279,14 +290,28 @@ const SETTING_CHECKS: SettingCheck[] = [{// Always pinned: an unset mode resolves to the standing preference and the load sends it. Reading// it as silence let a pick asking for "off" adopt a resident MTP runtime.+ mlxComparable: true,pinned: () => true,- agrees: (c, s, standing) =>- (standing.normalizeSpeculative(c.speculativeType) ??- standing.speculativeType) ===- (standing.normalizeSpeculative(s.speculative_type) ??- standing.speculativeType),+ agrees: (c, s, standing) => {+ const unset = resolveSpeculativeType(+ null,+ standing.speculativeType ?? "auto",+ s.is_mlx === true,+ );+ const mode = (value: string | null | undefined) => {+ const resolved = standing.normalizeSpeculative(value) ?? unset;+ return s.is_mlx === true ? mlxSpeculativeMode(resolved) : resolved;+ };+ return mode(c.speculativeType) === mode(s.speculative_type);+ },+ },+ {+ mlxComparable: true,+ pinned: () => true,+ agrees: (c, s) => (c.specDraftModel ?? null) === (s.spec_draft_model ?? null),},{+ mlxComparable: true,// Pinned like the rest: _runtime_matches_intent reloads for the null-against-explicit// flip, so an unset limit asks for the default. No `draft_depth_matters` gate here,// as the status carries a count only when a depth-consuming load recorded an override.diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex 0f4a7dd7eef..0b6fddb9432 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -44,7 +44,10 @@ import {import { NonModalDropdownMenu } from "@/components/ui/non-modal-dropdown-menu";import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params";import { FIND_SKIP_ATTRIBUTE } from "@/features/find-in-page";-import { DRAFT_N_MAX_SPEC_TYPES } from "@/lib/speculative-modes";+import {+ DRAFT_N_MAX_SPEC_TYPES,+ resolveSpeculativeType,+} from "@/lib/speculative-modes";import {StudioDictationAdapter,isStudioDictationAvailable,@@ -1622,8 +1625,16 @@ export function SharedComposer({const effectiveChatTemplateOverride = cleanCompareChatTemplate(ownConfig.chatTemplateOverride,);- const effectiveSpeculativeType =- ownConfig.speculativeType ?? specSettings.speculativeType;+ const targetIsMlx = isServedByMlx(+ targetIsGguf,+ platform.deviceType,+ platform.chatOnlyReason,+ );+ const effectiveSpeculativeType = resolveSpeculativeType(+ ownConfig.speculativeType,+ specSettings.speculativeType ?? "auto",+ targetIsMlx,+ );const effectiveSpecDraftNMax = ownRemembered? resolveCompareSpecDraftNMax(effectiveSpeculativeType,@@ -1823,6 +1834,7 @@ export function SharedComposer({mlx_int8_prefill: ownConfig.mlxInt8Prefill ?? false,speculative_type: effectiveSpeculativeType,spec_draft_n_max: effectiveSpecDraftNMax,+ ...(targetIsMlx ? { spec_draft_model: ownConfig.specDraftModel ?? null } : {}),reasoning_budget:targetIsGguf && !resolvedIsDiffusion? ownConfig.reasoningBudget@@ -1868,8 +1880,8 @@ export function SharedComposer({compareRunsRef.current.setLoadingModel(run, null);throwIfCompareCancelled(compareSignal);// Keep a compare pane's per-model speculative choice load-local: persist the global preference- // only when it came from global settings.- if (ownConfig.speculativeType == null) {+ // only when it came from global settings, and never from MLX, which only reads it.+ if (ownConfig.speculativeType == null && !targetIsMlx) {saveSpeculativeType(effectiveSpeculativeType);}// Persist the GPU Memory mode on a non-diffusion GGUF compare-load too, so an applied manualdiff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex 1ba870104b0..06a3d99e575 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -1957,6 +1957,7 @@ export function normalizeSpeculativeType(return "ngram";}if (s === "mtp+ngram") return "mtp+ngram";+ if (s === "eagle3") return s;// Comma-chained legacy values (e.g. from older backend echoes).const parts = s.split(",")@@ -1976,21 +1977,27 @@ export function normalizeSpeculativeType(export function resolveLoadedSpeculativeSettings(response: {speculative_type?: string | null;spec_draft_n_max?: number | null;+ spec_draft_model?: string | null;}): {speculativeType: string | null;loadedSpeculativeType: string | null;specDraftNMax: number | null;loadedSpecDraftNMax: number | null;+ specDraftModel: string | null;+ loadedSpecDraftModel: string | null;} {const loadedSpeculativeType = normalizeSpeculativeType(response.speculative_type,);const loadedSpecDraftNMax = response.spec_draft_n_max ?? null;+ const loadedSpecDraftModel = response.spec_draft_model ?? null;return {speculativeType: loadedSpeculativeType,loadedSpeculativeType,specDraftNMax: loadedSpecDraftNMax,loadedSpecDraftNMax,+ specDraftModel: loadedSpecDraftModel,+ loadedSpecDraftModel,};}@@ -2462,6 +2469,9 @@ type ChatRuntimeStore = {/** User --spec-draft-n-max override (null = platform default). */specDraftNMax: number | null;loadedSpecDraftNMax: number | null;+ /** MLX companion drafter, a repo id or local path (null = discovery). */+ specDraftModel: string | null;+ loadedSpecDraftModel: string | null;/** User reply-width override, for either backend (null = server default). GGUF spends it* on llama-server's --parallel slots; MLX on how many replies decode together.* Never re-seeded from an echo: the resolved count would pin a blank control. */@@ -4244,6 +4254,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({specDrafterKind: null,specDraftNMax: null,loadedSpecDraftNMax: null,+ specDraftModel: null,+ loadedSpecDraftModel: null,nParallel: null,loadedNParallel: null,reasoningBudget: -1,@@ -5193,6 +5205,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({specDrafterKind: null,specDraftNMax: null,loadedSpecDraftNMax: null,+ specDraftModel: null,+ loadedSpecDraftModel: null,nParallel: null,loadedNParallel: null,reasoningBudget: -1,diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 6d001e17b9a..da4f0fe96c1 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -81,6 +81,8 @@ export interface LoadModelRequest {/** Override --spec-draft-n-max for drafter speculative decoding. Applied only when speculative_type* resolves to "mtp", "mtp+ngram", "dspark" or "dflash". */spec_draft_n_max?: number | null;+ /** MLX only: a companion drafter, a local directory or an already-cached repo id. */+ spec_draft_model?: string | null;/** Parallel decode slots for llama-server (--parallel), 1..64. Omit/null = the launch default. The* VRAM fitter may launch fewer to stay on GPU. */n_parallel?: number | null;@@ -289,6 +291,7 @@ export interface LoadModelResponse {/** Canonical UI-facing mode the load request resolved to. See LoadModelRequest. */speculative_type?: string | null;spec_draft_n_max?: number | null;+ spec_draft_model?: string | null;/** Whether tensor-parallel split (--split-mode tensor) is active. */tensor_parallel?: boolean;/** The load ran with the vision projector deliberately left unloaded. Echoes the request, so it@@ -434,6 +437,7 @@ export interface InferenceStatusResponse {/** Canonical UI-facing mode currently active. See LoadModelRequest. */speculative_type?: string | null;spec_draft_n_max?: number | null;+ spec_draft_model?: string | null;/** Whether tensor-parallel split (--split-mode tensor) is active. */tensor_parallel?: boolean;/** The load ran with the vision projector deliberately left unloaded. Echoes the request, so itdiff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.tsindex b1392953847..f3b823a30af 100644--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts@@ -81,6 +81,7 @@ export interface MemoryEstimateRequest {ctxCheckpoints?: number | null;speculativeType?: string | null;specDraftNMax?: number | null;+ specDraftModel?: string | null;specDraftCacheType?: string | null;tensorParallel?: boolean;disableVision?: boolean;@@ -166,6 +167,7 @@ function estimateRequestBody(ctx_checkpoints: payload.ctxCheckpoints ?? null,speculative_type: payload.speculativeType ?? null,spec_draft_n_max: payload.specDraftNMax ?? null,+ spec_draft_model: payload.specDraftModel ?? null,spec_draft_cache_type: payload.specDraftCacheType ?? null,tensor_parallel: payload.tensorParallel ?? false,disable_vision: payload.disableVision ?? false,diff --git a/studio/frontend/src/features/model-picker/api/mlx-drafters.ts b/studio/frontend/src/features/model-picker/api/mlx-drafters.tsnew file mode 100644index 00000000000..a587b45c49a--- /dev/null+++ b/studio/frontend/src/features/model-picker/api/mlx-drafters.ts@@ -0,0 +1,25 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { authFetch } from "@/features/auth";+import type { MlxDrafter } from "@/lib/speculative-modes";++/** Cached drafters an MLX load of `modelPath` could name; empty when the backend cannot say. */+export async function fetchMlxDrafters(+ modelPath: string,+ signal?: AbortSignal,+): Promise<MlxDrafter[]> {+ const response = await authFetch("/api/inference/mlx-drafters", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ signal,+ body: JSON.stringify({ model_path: modelPath }),+ });+ if (!response.ok) return [];+ const body = (await response.json()) as { drafters?: unknown };+ return Array.isArray(body.drafters)+ ? body.drafters+ .filter((d) => typeof d?.repo_id === "string" && typeof d?.kind === "string")+ .map((d) => ({ repo: d.repo_id, kind: d.kind, named: d.named !== false }))+ : [];+}diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.tsindex e189f93c732..0c6c69bd01c 100644--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts@@ -47,6 +47,8 @@ export interface ApiModelOverride {// biome-ignore lint/style/useNamingConvention: API schemaspec_draft_n_max?: number;// biome-ignore lint/style/useNamingConvention: API schema+ spec_draft_model?: string;+ // biome-ignore lint/style/useNamingConvention: API scheman_parallel?: number;// biome-ignore lint/style/useNamingConvention: API schemareasoning_budget?: number;@@ -347,6 +349,7 @@ export function fromApiOverride(: (local.mlxKvQuant ?? null),speculativeType: override.speculative_type ?? local.speculativeType,specDraftNMax: override.spec_draft_n_max ?? local.specDraftNMax,+ specDraftModel: override.spec_draft_model ?? local.specDraftModel,specDraftCacheDtype:override.spec_draft_cache_type ?? local.specDraftCacheDtype,nParallel: override.n_parallel ?? local.nParallel,@@ -417,6 +420,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {if (config.specDraftNMax && config.specDraftNMax > 0) {payload.spec_draft_n_max = config.specDraftNMax;}+ if (config.specDraftModel) {+ payload.spec_draft_model = config.specDraftModel;+ }// Blank follows the server-wide --parallel default, which is the app default here.if (config.nParallel && config.nParallel > 0) {payload.n_parallel = config.nParallel;@@ -568,6 +574,8 @@ async function sendModelOverride(// can still predate.// biome-ignore lint/style/useNamingConvention: API schemamirrors_reasoning_budget: true,+ // biome-ignore lint/style/useNamingConvention: API schema+ mirrors_spec_draft_model: true,// Only sent when set, so an older backend is not handed an unknown key every save....(options?.fillAbsentFields? // biome-ignore lint/style/useNamingConvention: API schemadiff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 77fc9801c5e..bc4e39c9060 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -67,6 +67,14 @@ import {resolveMemoryCapacityGb,} from "@/hooks/gpu-vram";import { ChevronDownStandardIcon } from "@/lib/chevron-icons";+import {+ DRAFTER_MODEL_SPEC_TYPES,+ MLX_SPECULATIVE_TYPES,+ type MlxDrafter,+ mlxDrafterChoices,+ mlxSpeculativeMode,+ resolveSpeculativeType,+} from "@/lib/speculative-modes";import { toast } from "@/lib/toast";import {type ReactNode,@@ -94,6 +102,7 @@ import {subscribeLlamaFlagCatalog,} from "../api/llama-flags";import { type MemoryEstimate } from "../api/memory-estimate";+import { fetchMlxDrafters } from "../api/mlx-drafters";import {resolveEstimateContext,resolveMlxEstimateContext,@@ -186,10 +195,12 @@ import {normalizeMaxSeqLength,normalizePerModelConfig,perModelConfigStorageChanged,+ pinSpeculativeMode,readAdvancedSettingsOpen,resolveInitialConfig,saveAdvancedSettingsOpen,savePerModelConfig,+ storedSpeculativeAuto,subscribeAdvancedSettingsOpen,VRAM_BUDGET_PERCENT_STEP,vramFractionToPercent,@@ -255,6 +266,18 @@ const SPECULATIVE_TYPE_LABELS: Record<"mtp+ngram": "MTP+Ngram",off: "Off",};+const MLX_SPECULATIVE_TYPE_LABELS: Record<+ (typeof MLX_SPECULATIVE_TYPES)[number],+ string+> = {+ auto: "Auto",+ mtp: "MTP",+ dflash: "DFlash",+ dspark: "DSpark",+ eagle3: "EAGLE-3",+ ngram: "Ngram",+ off: "Off",+};// Lower-case where the value is the flag's own spelling, so the pick reads the same as the launch command.const LOAD_MODE_LABELS: Record<(typeof LOAD_MODES)[number], string> = {@@ -314,6 +337,7 @@ function hasNonDefaultAdvanced(config: PerModelConfig): boolean {(config.speculativeType ?? "auto") !== "auto" ||config.specDraftNMax != null ||config.specDraftCacheDtype != null ||+ config.specDraftModel != null ||config.nParallel != null ||config.reasoningBudget !== -1 ||config.reasoningBudgetMessage !== "" ||@@ -1181,6 +1205,157 @@ function ParallelSlotsRow({);}+/** Unset shows the mode the load would send. */+function MlxSpeculativeRows({+ config,+ update,+ speculativeFallback,+ modelPath,+}: {+ config: PerModelConfig;+ update: (patch: Partial<PerModelConfig>) => void;+ speculativeFallback: string;+ modelPath: string;+}) {+ const [drafters, setDrafters] = useState<MlxDrafter[]>([]);+ useEffect(() => {+ const controller = new AbortController();+ setDrafters([]);+ fetchMlxDrafters(modelPath, controller.signal)+ .then((found) => !controller.signal.aborted && setDrafters(found))+ .catch(() => {});+ return () => controller.abort();+ }, [modelPath]);+ const mode = mlxSpeculativeMode(+ config.speculativeType ??+ resolveSpeculativeType(null, speculativeFallback, true),+ );+ return (+ <>+ <div className={ROW_CLASS}>+ <div className="flex min-w-0 items-center gap-1.5">+ <span className={LABEL_CLASS_WRAP}>Speculative Decoding</span>+ <InfoHint>+ Faster generation. Auto uses the first cached drafter (an MTP head+ or assistant, then DFlash2, DFlash, DSpark, EAGLE-3), tunes the+ draft length to this machine and drafts only while that is faster.+ Choose a kind to force it. Every drafter also copies repeated text;+ Ngram copies without one. Drafters are read from the local Hugging+ Face cache, never downloaded.+ </InfoHint>+ </div>+ <Select+ value={mode}+ onValueChange={(v) =>+ update({+ speculativeType: v,+ specDraftNMax: DRAFT_N_MAX_SPEC_TYPES.has(v)+ ? config.specDraftNMax+ : null,+ specDraftModel: DRAFTER_MODEL_SPEC_TYPES.has(v)+ ? config.specDraftModel+ : null,+ })+ }+ >+ <SelectTrigger+ animateRadius={false}+ icon={ChevronDownStandardIcon}+ iconClassName="size-3.5"+ className={SELECT_TRIGGER_CLASS}+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">+ {MLX_SPECULATIVE_TYPES.map((type) => (+ <SelectItem key={type} value={type}>+ {MLX_SPECULATIVE_TYPE_LABELS[type]}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>+ {DRAFT_N_MAX_SPEC_TYPES.has(mode) && (+ <div className={ROW_CLASS}>+ <div className="flex min-w-0 items-center gap-1.5">+ <span className={LABEL_CLASS}>Draft Tokens</span>+ <InfoHint>+ Tokens per draft: up to the drafter's depth (3 for MTP heads),+ every step drafts exactly this many, even when plain decoding would+ be faster (a copy of repeated text replaces a draft when it should+ yield more). Above that depth, or for Ngram, it is a+ ceiling and Unsloth decodes plainly when drafting would be slower.+ Leave blank to let Unsloth tune it for this machine.+ </InfoHint>+ </div>+ <input+ type="number"+ min={1}+ max={16}+ step={1}+ value={config.specDraftNMax ?? ""}+ placeholder="auto"+ onChange={(event) => {+ const parsed = Number.parseInt(event.target.value, 10);+ update(+ pinSpeculativeMode(config, mode, {+ specDraftNMax: Number.isFinite(parsed)+ ? Math.max(1, Math.min(16, parsed))+ : null,+ }),+ );+ }}+ aria-label="Speculative decoding draft tokens"+ className={NUMBER_INPUT_CLASS}+ />+ </div>+ )}+ {DRAFTER_MODEL_SPEC_TYPES.has(mode) && (+ <div className={ROW_CLASS}>+ <div className="flex min-w-0 items-center gap-1.5">+ <span className={LABEL_CLASS}>Drafter</span>+ <InfoHint>+ A drafter from the local Hugging Face cache that fits this+ model. Auto uses the model's own head or the first cached drafter+ named for it.+ </InfoHint>+ </div>+ <Select+ value={config.specDraftModel ?? "auto"}+ onValueChange={(v) =>+ update(+ pinSpeculativeMode(config, mode, {+ specDraftModel: v === "auto" ? null : v,+ }),+ )+ }+ >+ <SelectTrigger+ animateRadius={false}+ icon={ChevronDownStandardIcon}+ iconClassName="size-3.5"+ aria-label="Speculative decoding drafter"+ className={SELECT_TRIGGER_CLASS}+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">+ <SelectItem value="auto">Auto</SelectItem>+ {mlxDrafterChoices(drafters, mode, config.specDraftModel ?? null).map(+ ([repo, label]) => (+ <SelectItem key={repo} value={repo}>+ {label}+ </SelectItem>+ ),+ )}+ </SelectContent>+ </Select>+ </div>+ )}+ </>+ );+}+function MlxAdvancedSettings({config,update,@@ -1188,6 +1363,8 @@ function MlxAdvancedSettings({servedByMlx,int8PrefillAvailable,onInt8PrefillChange,+ speculativeFallback,+ modelPath,onEditTemplate,templateOutcome,}: {@@ -1199,6 +1376,8 @@ function MlxAdvancedSettings({servedByMlx: boolean;int8PrefillAvailable: boolean;onInt8PrefillChange: (checked: boolean) => void;+ speculativeFallback: string;+ modelPath: string;onEditTemplate: () => void;/** Why the loaded model could not take the override it was given. */templateOutcome: string | null;@@ -1268,6 +1447,14 @@ function MlxAdvancedSettings({/></div>)}+ {servedByMlx && (+ <MlxSpeculativeRows+ config={config}+ update={update}+ speculativeFallback={speculativeFallback}+ modelPath={modelPath}+ />+ )}{servedByMlx && (<ParallelSlotsRow config={config} update={update} hint={MLX_PARALLEL_HINT} />)}@@ -2646,7 +2833,7 @@ export function ModelConfigPage({const hydrationSaved = savePerModelConfig(configId,target.ggufVariant,- rememberedConfig,+ storedSpeculativeAuto(rememberedConfig, !target.isGguf),hydrationEvicted,);setSavedRemember(hydrationSaved);@@ -2927,8 +3114,13 @@ export function ModelConfigPage({// override sends null, which the backend reads as Auto, while the selector has been showing// the global fallback. With the global Off and an 11 GB DSpark sidecar in the repo, the row// charged the sidecar for a load that disables it.- speculativeType: runtimeConfig.speculativeType ?? speculativeFallback ?? null,+ speculativeType: resolveSpeculativeType(+ runtimeConfig.speculativeType,+ speculativeFallback,+ targetIsMlx,+ ),specDraftNMax: runtimeConfig.specDraftNMax,+ specDraftModel: targetIsMlx ? (runtimeConfig.specDraftModel ?? null) : null,specDraftCacheType: runtimeConfig.specDraftCacheDtype ?? null,tensorParallel: runtimeConfig.tensorParallel,disableVision: runtimeConfig.disableVision,@@ -3229,7 +3421,7 @@ export function ModelConfigPage({const persistConfig = (next: PerModelConfig) => {// Judge what storage keeps: savePerModelConfig normalizes first, so the raw object over-reports.- const normalized = normalizePerModelConfig(next);+ const normalized = normalizePerModelConfig(storedSpeculativeAuto(next, targetIsMlx));const evicted: { modelId: string; ggufVariant: string | null }[] = [];const saved = remember? savePerModelConfig(configId, target.ggufVariant, normalized, evicted)@@ -3610,6 +3802,8 @@ export function ModelConfigPage({? setInt8PrefillConfirmOpen(true): update({ mlxInt8Prefill: false })}+ speculativeFallback={speculativeFallback}+ modelPath={target.id}onEditTemplate={() => setTemplateOpen(true)}templateOutcome={chatTemplateOutcome}/>diff --git a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.tsindex 532dfb94f95..96915398db7 100644--- a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts@@ -44,6 +44,7 @@ function estimateKey(request: MemoryEstimateRequest | null): string | null {request.ctxCheckpoints ?? null,request.speculativeType ?? null,request.specDraftNMax ?? null,+ request.specDraftModel ?? null,request.specDraftCacheType ?? null,request.tensorParallel ?? false,request.disableVision ?? false,diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.tsindex f13223e8e61..f9d9cba8e59 100644--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts@@ -65,6 +65,7 @@ export function applyPerModelConfigToRuntime(readPersistedSpeculativeType(),specDraftNMax: config.specDraftNMax ?? null,specDraftCacheDtype: config.specDraftCacheDtype ?? null,+ specDraftModel: config.specDraftModel ?? null,nParallel: config.nParallel ?? null,reasoningBudget: options.isDiffusion ? -1 : config.reasoningBudget,reasoningBudgetMessage: options.isDiffusion@@ -133,6 +134,7 @@ export function currentRuntimePerModelConfig(speculativeType: normalizeSpeculativeType(s.speculativeType),specDraftNMax: s.specDraftNMax ?? null,specDraftCacheDtype: s.specDraftCacheDtype ?? null,+ specDraftModel: s.specDraftModel ?? null,nParallel: s.nParallel ?? null,reasoningBudget:s.reasoningBudget === s.loadedReasoningBudget@@ -184,6 +186,7 @@ export function perModelConfigsEqual(speculative(a.speculativeType) === speculative(b.speculativeType) &&(a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) &&(a.specDraftCacheDtype ?? null) === (b.specDraftCacheDtype ?? null) &&+ (a.specDraftModel ?? null) === (b.specDraftModel ?? null) &&(a.nParallel ?? null) === (b.nParallel ?? null) &&a.reasoningBudget === b.reasoningBudget &&a.reasoningBudgetMessage === b.reasoningBudgetMessage &&diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.tsindex c9f86156b76..2b7645fe82a 100644--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts@@ -59,6 +59,7 @@ export function loadedConfigSignature(config.speculativeType ?? "",config.specDraftNMax ?? "",config.specDraftCacheDtype ?? "",+ config.specDraftModel ?? "",config.nParallel ?? "",config.reasoningBudget ?? "",`${(config.reasoningBudgetMessage ?? "").length}:${hashString(config.reasoningBudgetMessage ?? "")}`,diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex 63ff51aea6a..dcea694b9b6 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -16,6 +16,8 @@ import {import { isExternalModelId } from "@/features/chat/external-providers";import {DRAFT_N_MAX_SPEC_TYPES,+ DRAFTER_MODEL_SPEC_TYPES,+ MLX_ONLY_SPEC_TYPES,SEPARATE_DRAFT_MODEL_SPEC_TYPES,} from "@/lib/speculative-modes";@@ -34,6 +36,8 @@ export interface PerModelConfig {/** KV cache dtype for the DRAFT context, sized and quantized independently of kvCacheDtype.* Optional so older blobs parse. */specDraftCacheDtype?: string | null;+ /** MLX companion drafter, a repo id or local path. Optional so older blobs parse. */+ specDraftModel?: string | null;nParallel: number | null;reasoningBudget: number;reasoningBudgetMessage: string;@@ -78,6 +82,7 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {speculativeType: null,specDraftNMax: null,specDraftCacheDtype: null,+ specDraftModel: null,nParallel: null,reasoningBudget: -1,reasoningBudgetMessage: "",@@ -390,9 +395,10 @@ const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";// would normalize the unknown field straight back out of the record.// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,-// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split. v8 is skipped: nightly builds stamped it for the reverted custom+// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split, v11 the MLX drafter. v8 is skipped: nightly builds stamped it for the reverted custom// llama.cpp config (#12725), so a v8 client must not claim to understand an int8 prefill record.-const STORAGE_SCHEMA_VERSION = 10;+const STORAGE_SCHEMA_VERSION = 11;+const PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;@@ -440,6 +446,7 @@ const STORED_CONFIG_FIELDS = new Set(["speculativeType","specDraftNMax","specDraftCacheDtype",+ "specDraftModel","nParallel","reasoningBudget","reasoningBudgetMessage",@@ -530,9 +537,9 @@ function canonicalizeSpeculativeType(value: string): string | null {if (!s) {return null;}- // "auto"/"default" is the follow-global sentinel; store as null so it is never an override.+ // Kept: an MLX model's explicit Auto beats a standing "off". GGUF saves fold it via storedSpeculativeAuto.if (s === "auto" || s === "default") {- return null;+ return "auto";}// _LEGACY_SPEC_MODE_MAP's four. A stored override arrives raw, and the null below// is "follow the global preference", which would enable a drafter under Auto.@@ -551,12 +558,35 @@ function canonicalizeSpeculativeType(value: string): string | null {if (s === "ngram" || s === "ngram-mod" || s === "ngram-simple") {return "ngram";}- if (s === "mtp+ngram") {- return "mtp+ngram";+ if (s === "mtp+ngram" || (MLX_ONLY_SPEC_TYPES as readonly string[]).includes(s)) {+ return s;}return null;}+/** The config as storage keeps it: GGUF reads Auto as "follow the standing preference", spelled null,+ * while an MLX model stores its explicit Auto. */+export function storedSpeculativeAuto<T extends Pick<PerModelConfig, "speculativeType">>(+ config: T,+ isMlx: boolean,+): T {+ return !isMlx && config.speculativeType === "auto"+ ? { ...config, speculativeType: null }+ : config;+}++/** An edit to draft tokens or the drafter. Made while the mode is unset, it pins the shown mode:+ * storage keeps either value only beside a mode that reads it. */+export function pinSpeculativeMode(+ config: Pick<PerModelConfig, "speculativeType">,+ shownMode: string,+ patch: Partial<PerModelConfig>,+): Partial<PerModelConfig> {+ return config.speculativeType == null+ ? { ...patch, speculativeType: shownMode }+ : patch;+}+/** Canonicalize a stored --load-mode, or null to follow the llama.cpp default. "auto" folds* to null: it IS the default, so storing it would pin a value the build may redefine. */export function canonicalizeLoadMode(value: unknown): string | null {@@ -835,7 +865,8 @@ function parseLegacyModelKey(}function legacyEntryToConfig(raw: Record<string, unknown>): PerModelConfig {- return normalizeV1({+ // Written before any MLX control, so its Auto is GGUF's.+ return storedSpeculativeAuto(normalizeV1({customContextLength:typeof raw.contextLength === "number" ? raw.contextLength : null,maxSeqLength: null,@@ -870,7 +901,7 @@ function legacyEntryToConfig(raw: Record<string, unknown>): PerModelConfig {: Array.isArray(raw.selectedGpuIds)? (raw.selectedGpuIds as number[]): undefined,- });+ }), false);}function mergeLegacyEntries(@@ -1028,6 +1059,14 @@ function normalizeV1(partial: RawConfig): PerModelConfig {VALID_KV_CACHE_DTYPES.has(partial.specDraftCacheDtype)? partial.specDraftCacheDtype: null;+ const specDraftModel =+ speculativeType != null &&+ DRAFTER_MODEL_SPEC_TYPES.has(speculativeType) &&+ typeof partial.specDraftModel === "string" &&+ partial.specDraftModel.trim().length > 0 &&+ partial.specDraftModel.trim().length <= 1024+ ? partial.specDraftModel.trim()+ : null;return {engineParallelism: partial.engineParallelism === "pipeline" || partial.engineParallelism === "data"? partial.engineParallelism : "tensor",@@ -1057,6 +1096,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {speculativeType,specDraftNMax,specDraftCacheDtype,+ specDraftModel,loadMode: canonicalizeLoadMode(partial.loadMode),ctxCheckpoints: normalizeCtxCheckpoints(partial.ctxCheckpoints),cacheRam: normalizeCacheRam(partial.cacheRam),@@ -1108,8 +1148,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {};}-/** A config in the exact shape storage keeps: the UI carries sentinels storage does not- * (Speculative Decoding "auto" canonicalizes to null), which would read as non-default. */+/** A config in the exact shape storage keeps, so a UI value storage drops cannot read as non-default. */export function normalizePerModelConfig(raw: unknown): PerModelConfig {return normalize(raw);}@@ -1132,9 +1171,15 @@ function normalize(raw: unknown): PerModelConfig {* client reconstructs anyway, and stamping every record v4 would put the whole store out of reach.* The tuning group and the reasoning pair follow the same rule. */function storedSchemaVersion(normalized: PerModelConfig): number {- if (normalized.tensorSplit != null) {+ const mode = normalized.speculativeType;+ // A kept Auto and the MLX-only modes are values an older client folds back to unset.+ const mlxMode = mode === "auto" || (MLX_ONLY_SPEC_TYPES as readonly string[]).includes(mode ?? "");+ if (normalized.specDraftModel != null || mlxMode) {return STORAGE_SCHEMA_VERSION;}+ if (normalized.tensorSplit != null) {+ return PRE_MLX_DRAFTER_SCHEMA_VERSION;+ }if (normalized.mlxInt8Prefill) {return PRE_TENSOR_SPLIT_SCHEMA_VERSION;}@@ -1329,6 +1374,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean {// default, so a config changing only these was dropped on the way to storage. Compared against null, not truth:// 0 checkpoints and a 0 or -1 cache are values.(config.specDraftCacheDtype ?? null) === null &&+ (config.specDraftModel ?? null) === null &&(config.loadMode ?? null) === null &&config.ctxCheckpoints == null &&config.cacheRam == null &&diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.tsindex b8c14f74d31..bf0a9556809 100644--- a/studio/frontend/src/features/model-picker/sharing/fields.ts+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts@@ -50,6 +50,8 @@ export type SharedConfigKey = Exclude<| "engine"| "enginePrecision"| "engineParallelism"+ // A named drafter can be a local directory.+ | "specDraftModel">;type Field = { label: string; valid: Validator; error?: string };diff --git a/studio/frontend/src/lib/speculative-modes.ts b/studio/frontend/src/lib/speculative-modes.tsindex 83b21cd7927..e0af1f7871f 100644--- a/studio/frontend/src/lib/speculative-modes.ts+++ b/studio/frontend/src/lib/speculative-modes.ts@@ -21,6 +21,49 @@ export const SPECULATIVE_TYPES = ["off",] as const;+/** Values only an MLX load reads (studio/backend/core/inference/mlx_speculative.py). */+export const MLX_ONLY_SPEC_TYPES = ["eagle3"] as const;++export const MLX_SPECULATIVE_TYPES = [+ "auto",+ "mtp",+ "dflash",+ "dspark",+ "eagle3",+ "ngram",+ "off",+] as const;++export interface MlxDrafter {+ repo: string;+ kind: string;+ /** False when the drafter only fits the model's architecture and is not named for it. */+ named: boolean;+}++/** Drafters the Drafter picker offers in `mode`, as `[repo, label]`: every cached kind under Auto,+ * else that kind, plus a saved choice that is not cached (or is a local folder) so it stays visible. */+export function mlxDrafterChoices(+ drafters: readonly MlxDrafter[],+ mode: string,+ selected: string | null,+): [string, string][] {+ const choices = drafters+ .filter((drafter) => mode === "auto" || drafter.kind === mode)+ .map((drafter): [string, string] => [+ drafter.repo,+ drafter.named ? drafter.repo : `${drafter.repo} (same architecture)`,+ ]);+ return selected != null && !choices.some(([repo]) => repo === selected)+ ? [[selected, selected], ...choices]+ : choices;+}++/** The mode an MLX load runs: every drafter kind also copies repeated text, so llama.cpp's `mtp+ngram` is `mtp`. */+export function mlxSpeculativeMode(mode: string): string {+ return mode === "mtp+ngram" ? "mtp" : mode;+}+/*** The modes that consume spec_draft_n_max, i.e. the ones that launch a drafter* with a configurable depth. Named for the setting rather than for MTP: DSpark@@ -32,8 +75,24 @@ export const DRAFT_N_MAX_SPEC_TYPES: ReadonlySet<string> = new Set(["mtp+ngram","dspark","dflash",+ ...MLX_ONLY_SPEC_TYPES,]);+export const DRAFTER_MODEL_SPEC_TYPES: ReadonlySet<string> = new Set([+ ...DRAFT_N_MAX_SPEC_TYPES,+ "auto",+]);++/** The mode a load sends: the model's own choice, else the standing preference, which GGUF loads write.+ * On MLX its ngram reads as auto: n-gram copying alone would move a text model onto the vision runtime. */+export function resolveSpeculativeType(+ chosen: string | null,+ standing: string,+ isMlx: boolean,+): string {+ return chosen ?? (isMlx && standing === "ngram" ? "auto" : standing);+}+/*** The modes that always launch a SEPARATE draft model, and so a second context* with its own KV cache for the draft cache dtype to apply to.diff --git a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts b/studio/frontend/tests/hydration-clears-stale-local-record.test.tsindex 707efd9c722..5126b8666cb 100644--- a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts+++ b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts@@ -156,7 +156,7 @@ test("hydration does not mark itself saved when the write failed", () => {assert.match(src,- /const hydrationSaved = savePerModelConfig\( configId, target\.ggufVariant, rememberedConfig, hydrationEvicted, \); setSavedRemember\(hydrationSaved\);/,+ /const hydrationSaved = savePerModelConfig\( configId, target\.ggufVariant, storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\), hydrationEvicted, \); setSavedRemember\(hydrationSaved\);/,);});@@ -171,7 +171,7 @@ test("hydration propagates what its own write evicted", () => {// The write hands savePerModelConfig somewhere to report evictions.assert.match(src,- /savePerModelConfig\( configId, target\.ggufVariant, rememberedConfig, hydrationEvicted, \)/,+ /savePerModelConfig\( configId, target\.ggufVariant, storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\), hydrationEvicted, \)/,);// And they are cleared the way the save path clears them: mirrored fields only.assert.match(diff --git a/studio/frontend/tests/llama-extra-args-panel-hydration.test.ts b/studio/frontend/tests/llama-extra-args-panel-hydration.test.tsindex c7e145b35c3..cd5b639682c 100644--- a/studio/frontend/tests/llama-extra-args-panel-hydration.test.ts+++ b/studio/frontend/tests/llama-extra-args-panel-hydration.test.ts@@ -186,7 +186,7 @@ test("the panel adopts a shared server config without overwriting a live edit",// Whitespace-tolerant: the call carries an eviction list now, so it spans lines.assert.match(PANEL,- /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*rememberedConfig,/,+ /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\),/,);assert.match(PANEL, /configRef\.current === configAtStart/);assert.match(PANEL, /rememberRef\.current === rememberAtStart/);@@ -279,7 +279,7 @@ test("the hydration write-back rejects a stale server response", () => {const writeBack = PANEL.slice(adoptionStart, writeBackEnd);assert.match(writeBack,- /const storedConfig = resolveInitialConfig\(\s*configId,\s*target\.ggufVariant,\s*\);\s*if \(perModelConfigStorageChanged\(storedAtStart, storedConfig\)\) \{\s*return;\s*\}[\s\S]*const rememberedConfig = fromApiOverride\(\s*resolvedRow,\s*storedConfig\.config,\s*\);[\s\S]*savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*rememberedConfig,/,+ /const storedConfig = resolveInitialConfig\(\s*configId,\s*target\.ggufVariant,\s*\);\s*if \(perModelConfigStorageChanged\(storedAtStart, storedConfig\)\) \{\s*return;\s*\}[\s\S]*const rememberedConfig = fromApiOverride\(\s*resolvedRow,\s*storedConfig\.config,\s*\);[\s\S]*savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\),/,);});diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.tsindex 35f71c60b34..f14ae33b2fd 100644--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts@@ -208,8 +208,8 @@ const ROWS: Row[] = ["and reads normally.",},{- name: "version 11 (genuinely future)",- raw: { version: 11, customContextLength: 32768 },+ name: "version 12 (genuinely future)",+ raw: { version: 12, customContextLength: 32768 },normalizedPin: null,rawPin: 32768,isDefault: true,@@ -357,7 +357,7 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by vetrue,);assert.equal(- stage({ version: 11, customContextLength: 32768 }).remembered,+ stage({ version: 12, customContextLength: 32768 }).remembered,false,);});@@ -365,6 +365,8 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by vetest("a stored mlxKvQuant stamps a version older builds refuse", () => {assert.equal(stampedVersion({ mlxKvQuant: "tq-4" }), 7);+ // An older client folds a kept Auto and an MLX-only mode back to unset, so both lock it out.+ assert.deepEqual(["auto", "eagle3", "mtp"].map((speculativeType) => stampedVersion({ speculativeType })), [11, 11, 1]);assert.equal(stampedVersion({ mlxKvQuant: "4" }), 7);assert.equal(stampedVersion({ loadMode: "mmap" }), 5);assert.equal(stampedVersion({ mlxKvQuant: "tq-4", loadMode: "mmap" }), 7);diff --git a/studio/frontend/tests/mlx-spec-fallback-copy.test.ts b/studio/frontend/tests/mlx-spec-fallback-copy.test.tsnew file mode 100644index 00000000000..1a941c0a1c4--- /dev/null+++ b/studio/frontend/tests/mlx-spec-fallback-copy.test.ts@@ -0,0 +1,23 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { mlxSpecFallbackMessage } from "../src/features/chat/lib/mlx-spec-fallback.ts";++test("an MLX fallback notice says what runs in the passed-over drafter's place", () => {+ assert.match(mlxSpecFallbackMessage("drafter_not_found", "mtp") ?? "", /with MTP instead/);+ assert.match(mlxSpecFallbackMessage("drafter_no_memory", "ngram") ?? "", /n-gram copies only/);+ assert.match(mlxSpecFallbackMessage("auto_context_cost", null) ?? "", /without speculative/);+ // llama.cpp's codes are not MLX's, so an unknown one shows nothing rather than a wrong remedy.+ assert.equal(mlxSpecFallbackMessage("binary_no_mtp", null), null);+});++test("an MLX fallback notice claims no more than every path behind its code", () => {+ // No code guarantees its remedy works, and drafter_no_memory also covers an unpriced fit.+ for (const code of ["drafter_not_found", "drafter_incompatible", "drafter_no_memory", "runtime_error", "kv_quant", "auto_context_cost"]) {+ assert.doesNotMatch(mlxSpecFallbackMessage(code, null) ?? "missing", /to use it|missing/);+ }+ assert.doesNotMatch(mlxSpecFallbackMessage("drafter_no_memory", null) ?? "", /does not fit/);+});diff --git a/studio/frontend/tests/per-model-spec-disable-aliases.test.ts b/studio/frontend/tests/per-model-spec-disable-aliases.test.tsindex 704d177d565..638952fbc7a 100644--- a/studio/frontend/tests/per-model-spec-disable-aliases.test.ts+++ b/studio/frontend/tests/per-model-spec-disable-aliases.test.ts@@ -8,9 +8,22 @@ import { registerBundlerResolver } from "./helpers/kit.ts";registerBundlerResolver();-const { normalizePerModelConfig } = await import(+const { normalizePerModelConfig, pinSpeculativeMode, storedSpeculativeAuto } = await import("../src/features/model-picker/model-config/per-model-config.ts");+const { loadedConfigSignature } = await import(+ "../src/features/model-picker/model-config/config-signature.ts"+);++const { mlxDrafterChoices } = await import("../src/lib/speculative-modes.ts");++test("the drafter picker offers the mode's cached drafters and keeps a saved one visible", () => {+ const cached = [{ repo: "o/m-MTP", kind: "mtp", named: true }, { repo: "o/n-DFlash", kind: "dflash", named: false }];+ const pick = (mode: string, saved: string | null) => mlxDrafterChoices(cached, mode, saved);+ assert.deepEqual(pick("auto", null).map(([repo]: [string, string]) => repo), ["o/m-MTP", "o/n-DFlash"]);+ assert.deepEqual(pick("dflash", "o/n-DFlash"), [["o/n-DFlash", "o/n-DFlash (same architecture)"]]);+ assert.deepEqual(pick("mtp", "/local/drafter")[0], ["/local/drafter", "/local/drafter"]);+});const specOf = (value: string) =>normalizePerModelConfig({ speculativeType: value }).speculativeType;@@ -38,13 +51,32 @@ test("a stored disable alias is an override, not a fall-through to the global de});test("the rest of the mapping still resolves as before", () => {- // "auto" and an unknown value are the follow-global sentinel, which is what the- // aliases above must not be confused with.- assert.equal(specOf("auto"), null);- assert.equal(specOf("default"), null);+ // An unknown value is the follow-global sentinel the aliases must not become; Auto is kept.+ assert.equal(specOf("auto"), "auto");+ assert.equal(specOf("default"), "auto");assert.equal(specOf("bogus"), null);assert.equal(specOf("mtp"), "mtp");assert.equal(specOf("draft-mtp"), "mtp");assert.equal(specOf("ngram-mod"), "ngram");assert.equal(specOf("mtp+ngram"), "mtp+ngram");});++test("MLX modes and an explicit Auto survive storage; the drafter only with a drafter mode", () => {+ assert.deepEqual([specOf("eagle3"), specOf("dflash+ngram")], ["eagle3", null]);+ const drafterOf = (speculativeType: string, specDraftModel: string) =>+ normalizePerModelConfig({ speculativeType, specDraftModel }).specDraftModel;+ assert.deepEqual(+ [drafterOf("auto", " o/d "), drafterOf("ngram", "o/d"), drafterOf("mtp", " ")],+ ["o/d", null, null],+ );+ // GGUF's Auto is "follow the standing preference", which storage spells null.+ assert.equal(storedSpeculativeAuto({ speculativeType: "auto" }, false).speculativeType, null);+ assert.equal(storedSpeculativeAuto({ speculativeType: "auto" }, true).speculativeType, "auto");+ // A drafter typed while the mode is unset pins the shown one, or storage would drop it.+ const pinned = pinSpeculativeMode({ speculativeType: null }, "auto", { specDraftModel: "o/d" });+ assert.equal(normalizePerModelConfig(pinned).specDraftModel, "o/d");+ assert.equal(pinSpeculativeMode({ speculativeType: "mtp" }, "auto", {}).speculativeType, undefined);+ // A new live drafter re-seeds an open editor.+ const auto = normalizePerModelConfig({ speculativeType: "auto" });+ assert.notEqual(loadedConfigSignature(auto), loadedConfigSignature({ ...auto, specDraftModel: "o/d" }));+});diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.tsindex 5d20fb28c19..d603d2708d4 100644--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts@@ -173,6 +173,7 @@ const FIELDS: FieldCase[] = [same: 16,different: 8,},+ { key: "specDraftModel", statusKey: "spec_draft_model", same: "org/d", different: "org/e" },{key: "nParallel",statusKey: "requested_parallel_slots",diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.tsindex f1788aa6fe7..e584400749f 100644--- a/studio/frontend/tests/resident-config-match.test.ts+++ b/studio/frontend/tests/resident-config-match.test.ts@@ -720,6 +720,27 @@ test("a binary stand-down that cannot repair does not decline the shortcut", ());});+test("an MLX resident is judged on its speculative settings, a standing ngram reading as auto", () => {+ const mlx = { ...DEFAULTS, is_gguf: false, is_mlx: true, speculative_type: "auto" };+ const ngram = { ...STANDING, speculativeType: "ngram" };+ assert.equal(matches(mlx, BLANK, ngram), true);+ assert.equal(matches({ ...mlx, speculative_type: "ngram" }, BLANK, ngram), false);+ assert.equal(matches(mlx, { ...BLANK, speculativeType: "eagle3" }), false);+ const drafted = { ...BLANK, speculativeType: "auto", specDraftModel: "o/d" };+ assert.equal(matches({ ...mlx, spec_draft_model: "o/d" }, drafted), true);+ assert.equal(matches(mlx, drafted), false);+ const deep = { ...BLANK, speculativeType: "mtp", specDraftNMax: 6 };+ assert.equal(matches({ ...mlx, speculative_type: "mtp", spec_draft_n_max: 6 }, deep), true);+ assert.equal(matches({ ...mlx, speculative_type: "mtp", spec_draft_n_max: 4 }, deep), false);+ // An MLX load reads "+ngram" as its kind, which already copies; GGUF keeps the two apart.+ const copying = { ...BLANK, speculativeType: "mtp+ngram" };+ assert.equal(matches({ ...mlx, speculative_type: "mtp" }, copying), true);+ assert.equal(matches({ ...DEFAULTS, speculative_type: "mtp" }, copying), false);+ // llama.cpp's retry arms: an identical MLX load dedupes.+ const notFound = { ...mlx, spec_fallback_reason: "drafter_not_found" };+ assert.equal(residentSpeculativeNeedsRepair(notFound, "auto"), false);+});+test("a non-GGUF resident is not judged on a GGUF invocation field", () => {// requested_context_length is set only by the llama.cpp path. A safetensors or MLX// status never carries it, and the resolver answers the generation length for adiff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.tsindex 204ffad93df..f27dae0a843 100644--- a/studio/frontend/tests/server-tuning-settings.test.ts+++ b/studio/frontend/tests/server-tuning-settings.test.ts@@ -113,7 +113,9 @@ test("a record only claims the new schema version when it carries one", () => {// toStoredConfig stamps the OLDEST version that understands every field// present, so an older client can still rewrite a record it fully knows.const source = readSrc("features/model-picker/model-config/per-model-config.ts");- assert.match(source, /const STORAGE_SCHEMA_VERSION = 10;/);+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 11;\nconst PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;/);+ assert.match(source, /specDraftModel != null \|\| mlxMode\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);+ assert.match(source, /tensorSplit != null\)\s*\{\s*return PRE_MLX_DRAFTER_SCHEMA_VERSION;/);assert.match(source, /const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;/);assert.match(source, /const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;/);assert.match(source, /const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;/);diff --git a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.tsindex 3c451ee82ef..37caaba342f 100644--- a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts+++ b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts@@ -724,7 +724,7 @@ test("opening the panel ticks Remember for any model with a resolvable row", ()// the entry, so a merge that comes out default is a clear that has to travel.assert.match(PANEL,- /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*rememberedConfig,/,+ /savePerModelConfig\(\s*configId,\s*target\.ggufVariant,\s*storedSpeculativeAuto\(rememberedConfig, !target\.isGguf\),/,);// Whatever that write evicted is cleared before the block returns, or a dropped// model keeps applying its server row with nothing able to forget it.@@ -769,3 +769,18 @@ test("a localStorage that throws degrades on every path instead of propagating",asWindow.window.localStorage = storage;}});+++test("the MLX drafter mirrors to the server and back", () => {+ const config = normalizePerModelConfig({ speculativeType: "eagle3", specDraftModel: "o/d" });+ assert.equal(toApiOverride(config).spec_draft_model, "o/d");+ // biome-ignore lint/style/useNamingConvention: API schema+ const row = { speculative_type: "auto", spec_draft_model: "o/d" };+ assert.equal(fromApiOverride(row).specDraftModel, "o/d");+ // Without the flag the backend keeps a drafter the save cleared.+ const overrides = path.join(+ path.dirname(fileURLToPath(import.meta.url)),+ "../src/features/model-picker/api/model-overrides.ts",+ );+ assert.match(readFileSync(overrides, "utf8"), /mirrors_spec_draft_model: true/);+});diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.pyindex 4c380b24f80..0231277e779 100644--- a/tests/studio/test_chat_autoload_failure_gate.py+++ b/tests/studio/test_chat_autoload_failure_gate.py@@ -258,6 +258,10 @@ function isExternalModelId(value: unknown) {function resolvePreserveThinkingOnLoad(resp: any) {return Boolean(resp?.supports_preserve_thinking && resp?.preserve_thinking_default);}+// Verbatim from lib/speculative-modes.ts, so a load asks for the mode production would.+function resolveSpeculativeType(chosen: string | null, standing: string, isMlx: boolean) {+ return chosen ?? (isMlx && standing === "ngram" ? "auto" : standing);+}function resolveInferenceCheckpointId(status: any) {return status.active_modeldiff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex d45bcfdb6ff..b00f764762b 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -1125,14 +1125,19 @@ def test_blur_cache_cleared_on_every_settled_render():def test_auto_defaults_not_persisted_as_overrides():- """Auto GPU memory mode and Auto/default speculative type are follow-global- defaults; normalization must not persist them as per-model overrides, else a- model stops following later changes to the global preference."""+ """Auto GPU memory mode and a GGUF model's Auto/default speculative type are+ follow-global defaults; they must not persist as per-model overrides, else a+ model stops following later changes to the global preference. An MLX model+ keeps its explicit Auto, which beats a standing "off"."""src = _read("features/model-picker/model-config/per-model-config.ts")assert 'if (partial.gpuMemoryMode === "manual") {' in srcassert 'partial.gpuMemoryMode === "auto" || partial.gpuMemoryMode === "manual"' not in srcspec = re.search(r'if \(s === "auto" \|\| s === "default"\) \{\s*return ([^;]+);', src)- assert spec and spec.group(1).strip() == "null"+ assert spec and spec.group(1).strip() == '"auto"'+ fold = " ".join(src.split())+ assert (+ '!isMlx && config.speculativeType === "auto" ? { ...config, speculativeType: null }' in fold+ )def test_compare_pane_context_from_own_config_only():@@ -2446,7 +2451,7 @@ def test_parallel_slots_reach_an_api_load_through_the_server_mirror():store = _read_backend("utils/openai_auto_switch_settings.py")assert 'entry["n_parallel"] = n_parallel' in store# Ungated: MLX sizes its batch by the same width llama-server sizes its slots by.- shared, gguf_block = store.split(" if is_gguf:", 1)+ shared, gguf_block = store.rsplit(" if is_gguf:", 1)assert '("n_parallel", "n_parallel"),' in sharedassert "n_parallel" not in gguf_block.split("\n\n", 1)[0]@@ -3249,10 +3254,15 @@ def test_backfill_splits_a_quant_suffix_the_way_the_backend_does():def test_the_settings_page_judges_the_config_storage_actually_keeps():"""savePerModelConfig normalizes before deciding, and the runtime hands this page- Speculative Decoding "auto", which canonicalizes to null."""+ Speculative Decoding "auto", which a GGUF model stores as null."""src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split())# Load and Save (#10216) both go through persistConfig.- assert "const normalized = normalizePerModelConfig(next);" in src+ assert (+ "const normalized = normalizePerModelConfig(storedSpeculativeAuto(next, targetIsMlx));"+ in src+ )+ # Server hydration writes the same record, so a GGUF row's stored "auto" folds there too.+ assert "storedSpeculativeAuto(rememberedConfig, !target.isGguf)" in srcassert "defaultConfig: isDefaultConfig(normalized)" in src# The same object goes to storage and to the server, or they disagree again.assert "savePerModelConfig(configId, target.ggufVariant, normalized, evicted)" in src