Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
feat(studio): draft MLX tool, video, grammar, KV and LoRA replies (#16441)
Long Yixingb0a6f4b8ec9b
diff --git a/studio/backend/core/inference/grammar_constraint.py b/studio/backend/core/inference/grammar_constraint.pyindex 6637fee4741..8b5a806f415 100644--- a/studio/backend/core/inference/grammar_constraint.py+++ b/studio/backend/core/inference/grammar_constraint.py@@ -273,6 +273,9 @@ class GrammarConstraint:self._strips_special_tokens = strips_special_tokensself._dropped_ids = Noneself._runtime_stops = None+ self._settled = None+ self._drafted = []+ self._deferred = {}self.allows_reasoning = allows_reasoningdef _bind(self, n_vocab: int) -> None:@@ -343,6 +346,46 @@ class GrammarConstraint:raise ResponseFormatError(self._desync_message(int(token_id)))raise GrammarDesyncError(f"guided decoding desynced on token {int(token_id)}: {error}")+ def follow(self, drafted) -> bool:+ kept = _common_prefix(self._drafted, drafted)+ rest = drafted[kept:]+ if kept < len(self._drafted):+ self._matcher, self._settled, self._drafted, rest = self._settled, None, [], drafted+ self._deferred = {+ depth: error for depth, error in self._deferred.items() if depth <= kept+ }+ if not rest:+ return True+ if self._settled is None:+ # llguidance's rollback misrestores a max_tokens rule (the prelude) and an end token.+ self._settled, self._matcher = self._matcher, self._matcher.deep_copy()+ admitted = self._matcher.try_consume_tokens(rest)+ self._drafted.extend(rest[:admitted])+ return admitted == len(rest)++ def mask_ahead(self, logits):+ try:+ return self.mask_logits(logits)+ except ResponseFormatError as error:+ self._deferred.setdefault(len(self._drafted), error)+ return logits++ def commit(self, tokens) -> None:+ reached = [+ depth for depth in self._deferred if depth <= _common_prefix(self._drafted, tokens)+ ]+ if reached:+ raise self._deferred[min(reached)]+ if self._settled is not None:+ self._matcher, self._settled = self._settled, None+ self._drafted, self._deferred = [], {}+ for token_id in tokens:+ if self._matcher is not None and not self._matcher.is_stopped():+ if self._matcher.try_consume_tokens([int(token_id)]):+ continue+ _llg_mlx.fill_next_token_bitmask(self._matcher, self._bitmask)+ self.advance(token_id)+def _mask_allowed(self, token_id: int) -> bool:try:import numpy as np@@ -361,19 +404,39 @@ class GrammarConstraint:)+def _common_prefix(held, tokens) -> int:+ kept = 0+ for a, b in zip(held, tokens):+ if a != b:+ break+ kept += 1+ return kept++def make_grammar_logits_processor(constraint: GrammarConstraint):- state = {"seen": None}+ """Each call commits its history until ``settle`` is first called; then only ``settle`` does."""+ state = {"seen": None, "committed": None}def _processor(tokens, logits):- seen = state["seen"]+ seen, committed = state["seen"], state["committed"]if seen is None:state["seen"] = int(tokens.shape[0])- else:+ elif committed is None:for token_id in tokens[seen:].tolist():constraint.advance(int(token_id))state["seen"] = int(tokens.shape[0])+ elif not constraint.follow(tokens[committed:].tolist()):+ return logits+ elif int(tokens.shape[0]) > committed:+ return constraint.mask_ahead(logits)return constraint.mask_logits(logits)+ def settle(tokens):+ start = state["seen"] if state["committed"] is None else state["committed"]+ constraint.commit(tokens[start:].tolist())+ state["committed"] = int(tokens.shape[0])++ _processor.settle = settlereturn _processordiff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 62d7a9e8ed0..84d61793145 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -1431,6 +1431,7 @@ def mlx_resolve_drafter_offload(model_dir, knobs, drafter, *, auto: bool, **pricpriced["pinned"],SimpleNamespace(path = drafter[0], builtin = drafter[1]),load_in_4bit = priced["load_in_4bit"],+ kv_bits = priced.get("kv_bits"),**kwargs,**mlx_offload_route(plan),)[0]@@ -3322,6 +3323,7 @@ def mlx_drafter_fit(prefill_chunk = None,baseline = None,reserve = True,+ kv_bits = None,):"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit@@ -3336,6 +3338,8 @@ def mlx_drafter_fit(try:from core.inference.mlx_memory import mlx_fit_outcome+ if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):+ kv_bits = Nonefit = functools.partial(mlx_fit_outcome,model_dir,@@ -3345,6 +3349,7 @@ def mlx_drafter_fit(vision = True,offload = offload,prefill_chunk = prefill_chunk,+ kv_bits = kv_bits,)outcome, fitted = fit(drafter = (source.path, source.builtin))if outcome == "unsizable":@@ -3365,6 +3370,7 @@ def mlx_prompt_cache_room(load_in_4bit: bool,offload = None,prefill_chunk = None,+ kv_bits = None,):"""Bytes MLX's own cap leaves the prompt cache beside *source*'s drafter at *n_ctx*; None where unsized."""budget = mlx_memory_budget(retains_history = False)@@ -3382,6 +3388,7 @@ def mlx_prompt_cache_room(drafter = (source.path, source.builtin),offload = offload,prefill_chunk = prefill_chunk,+ kv_bits = kv_bits,)except Exception as exc:logger.info("MLX prompt cache room beside %s not priced: %s", source.path, exc)@@ -4256,7 +4263,7 @@ def _vlm_add_special_tokens(model_type, processor):)-def _vlm_prepared_inputs(model, processor, prompt, images):+def _vlm_prepared_inputs(model, processor, prompt, images, **media):"""The processor's inputs for *prompt* and *images*, as mlx-vlm's ``stream_generate`` prepares them."""from mlx_vlm.utils import prepare_inputs@@ -4268,6 +4275,7 @@ def _vlm_prepared_inputs(model, processor, prompt, images):prompts = prompt,image_token_index = read("image_token_index"),add_special_tokens = _vlm_add_special_tokens(read("model_type"), processor),+ **media,)@@ -4595,8 +4603,17 @@ def _failing_alone(row, processors):row.failure = excreturn logits- # A drafted batch drafts only for rows whose processors all read just the history._processor.history_only = getattr(processor, "history_only", False)+ if hasattr(processor, "settle"):++ def _settle(tokens):+ if row.failure is None:+ try:+ processor.settle(tokens)+ except Exception as exc:+ row.failure = exc++ _processor.settle = _settlereturn _processorreturn [guarded(processor) for processor in processors]@@ -4885,7 +4902,6 @@ class _VLMRowPlan:"max_tokens","processors","streamed",- "speculates",)def __init__(@@ -4900,7 +4916,6 @@ class _VLMRowPlan:max_tokens,processors = None,streamed = None,- speculates = True,videos = None,):self.prompt = prompt@@ -4913,7 +4928,6 @@ class _VLMRowPlan:self.max_tokens = max_tokensself.processors = processorsself.streamed = streamed- self.speculates = speculatesclass _VisionBatchRow:@@ -4976,6 +4990,14 @@ def _row_retire_gap():return row_retire_unavailable_reason()+def _speculative_quantized_row_gap():+ try:+ from unsloth_zoo.mlx.generate import speculative_quantized_rows_unavailable_reason+ except ImportError:+ return "the installed unsloth-zoo cannot draft a batch on a quantized KV cache"+ return speculative_quantized_rows_unavailable_reason()++def _row_quantized_cache_gap():try:from unsloth_zoo.mlx.generate import row_quantized_prompt_cache_unavailable_reason@@ -5056,19 +5078,21 @@ class _VisionBatchSession:plan = backend._plan_vlm_request(request, decodes_ids = self._decodes_ids)row = _VisionBatchRow(handle = handle, plan = plan)alone = self._alone(handle) if self._retires_rows else None+ # A clip's row resumes nothing: token ids do not vary with the clip.state = (backend._vlm_batch_row_cache(self._adapter_state, plan.images, alone = alone)- if self._resumes_rows+ if self._resumes_rows and not plan.videoselse None)if state is None and backend._kv_quant_bits() is not None:# mlx-vlm's own batch cache would be float.raise RowRefused("this row cannot carry the load's quantized KV cache")- if state is None and plan.images:+ if state is None and (plan.images or plan.videos):backend._release_vlm_snapshots()elif state is None:backend._release_held_vlm_snapshots()resume = {} if state is None else {"prompt_cache_state": state}+ clips = _batch_clips(self._held, [plan], backend._processor)[0] if plan.videos else {}admitted_at = time.perf_counter()try:row_number = self.stream.add(@@ -5078,8 +5102,8 @@ class _VisionBatchSession:max_tokens = plan.max_tokens,sampling = SamplingParams(**plan.sampling),logits_processors = plan.processors and _failing_alone(row, plan.processors),+ **clips,**resume,- **({} if self._speculative is None or plan.speculates else {"drafts": False}),))except BatchRowRefused as refusal:@@ -6000,9 +6024,9 @@ class MLXInferenceBackend:)if speculates:spec_reason = mlx_speculative.speculation_refusal(- kv_quant = kv_bits is not None or self._turboquant,distributed = is_distributed,- lora = is_lora,+ # A text architecture's adapter loads only through mlx-lm, and speculation runs on mlx-vlm.+ lora = is_lora and not (is_vision or zoo_multimodal),)speculates = spec_reason is Noneif (@@ -6175,7 +6199,7 @@ class MLXInferenceBackend:if spec_mode == "auto" and not spec_draft_modelelse mlx_speculative.RUNTIME_ERROR)- elif self._turboquant:+ if self._turboquant:self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOADuse_vlm = Falseif model is None:@@ -6257,6 +6281,16 @@ class MLXInferenceBackend:allowed = spec_drafters_allowed,)+ if not self._turboquant and kv_bits is not None and _eligibility is None:+ _eligibility = _kv_quant_eligibility(+ self._model, use_vlm, _normalize_mlx_kv_bits(kv_bits)+ )+ _drafter_bits = (+ _normalize_mlx_kv_bits(kv_bits)+ if not self._turboquant and _eligibility and _eligibility[0] in ("full", "partial")+ else None+ )+def _fits(source, reserve = True):if not _priceable:return None, None@@ -6269,6 +6303,7 @@ class MLXInferenceBackend:costless = spec_mode == "auto",baseline = offload_baseline["route"],reserve = reserve,+ kv_bits = _drafter_bits,**_offloaded,)@@ -6332,7 +6367,12 @@ class MLXInferenceBackend:and getattr(self._speculative_draft, "drafter", None) is not None):self._prompt_cache_room = mlx_prompt_cache_room(- model_dir, _served_ctx, _stretched[-1], load_in_4bit = load_in_4bit, **_offloaded+ model_dir,+ _served_ctx,+ _stretched[-1],+ load_in_4bit = load_in_4bit,+ kv_bits = _drafter_bits,+ **_offloaded,)if self._prompt_cache_room is not None:logger.info(@@ -7845,8 +7885,6 @@ class MLXInferenceBackend:max_tokens = vlm_kwargs["max_tokens"],processors = processors,streamed = streamed,- # As _generate_vlm, which drops the draft for tool turns.- speculates = not (tools or tool_protocol_active),)def _generate_vlm(@@ -8067,18 +8105,16 @@ class MLXInferenceBackend:vlm_kwargs.update(self._prefill_step_kwargs())session_scope = session if session is not None else nullcontext()processors = vlm_kwargs.get("logits_processors") or ()- # The draft is told these processors, and takes only ones that read just the history (a grammar's- # matcher advances per call); tool turns decode as before; a clip is nothing a drafter was built for.if (draft is None- or tools- or tool_protocol_active- or videos- or not all(getattr(processor, "history_only", False) for processor in processors)+ or not all(+ getattr(processor, "history_only", False) or hasattr(processor, "settle")+ for processor in processors+ )or _vlm_generation_is_diffusion(self._model)):draft = None- elif "input_ids" not in vlm_kwargs:+ elif "input_ids" not in vlm_kwargs and not videos:vlm_kwargs.update(_vlm_generate_kwargs(_vlm_prepared_inputs(self._model, self._processor, prompt, images)@@ -8107,30 +8143,39 @@ class MLXInferenceBackend:)final_response = Noneclip_paths = []- if draft is not None:- from unsloth_zoo.mlx.generate import SamplingParams-- # Under the lock: the load's one draft serves one generation at a time.- draft.prepare(- vlm_kwargs["input_ids"][0].tolist(),- SamplingParams(- temperature = temperature,- top_p = top_p,- top_k = max(int(top_k or 0), 0),- min_p = float(min_p or 0.0),- seed = seed,- ),- processors,- )- vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)try:if videos:for clip in videos:clip_paths.append(_write_video_clip(clip))- vlm_kwargs["video"] = clip_paths+ clip_media = {"video": clip_paths}frame_rate = _video_sampling_fps(clip_paths, self._processor)if frame_rate is not None:- vlm_kwargs["fps"] = frame_rate+ clip_media["fps"] = frame_rate+ if draft is None or "input_ids" in vlm_kwargs:+ vlm_kwargs.update(clip_media)+ else:+ clip_media["videos"] = clip_media.pop("video")+ vlm_kwargs.update(+ _vlm_generate_kwargs(+ _vlm_prepared_inputs(+ self._model, self._processor, prompt, images, **clip_media+ )+ )+ )+ if draft is not None:+ from unsloth_zoo.mlx.generate import SamplingParams+ draft.prepare(+ vlm_kwargs["input_ids"][0].tolist(),+ SamplingParams(+ temperature = temperature,+ top_p = top_p,+ top_k = max(int(top_k or 0), 0),+ min_p = float(min_p or 0.0),+ seed = seed,+ ),+ processors,+ )+ vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)# Emit prefilled <think> first; inside the adapter so unsupported requests raise before any output.if prefill:yield prefill@@ -8285,6 +8330,10 @@ class MLXInferenceBackend:return None# Only resident vision rows carry their own cache, where the quantization lives.if resident and self._is_vlm:+ if self._kv_quant_bits() is not None and getattr(self, "_speculative_draft", None):+ if getattr(self, "_turboquant", False):+ return "a speculative batch cannot run TurboQuant caches"+ return _speculative_quantized_row_gap()return _row_quantized_cache_gap()return "the load quantizes or budgets its KV cache, which a batch does not carry"@@ -8298,7 +8347,9 @@ class MLXInferenceBackend:reason = self._video_batch_gap(requests)if reason is not None:return reason- reason = self._kv_policy_batch_reason()+ reason = self._kv_policy_batch_reason(+ resident = getattr(self, "_speculative_draft", None) is not None+ )if reason is not None:return reasonif len(requests) < 2:@@ -8336,8 +8387,9 @@ class MLXInferenceBackend:# The one-at-a-time decode refuses these clips by name.if not self._is_vlm or not self._reads_vision_input() or not _mlx_vlm_decodes_video():return "a reply carries a video clip this load does not read"- if self._speculative_draft is not None:- return "a video clip decodes without the draft"+ if self._speculative_draft is not None and self._kv_quant_bits() is not None:+ # A clip's row resumes no cache, and only a resumed row carries the quantization.+ return "a quantized speculative batch does not decode a video clip"try:from unsloth_zoo.mlx import generate as engineexcept ImportError:diff --git a/studio/backend/core/inference/mlx_speculative.py b/studio/backend/core/inference/mlx_speculative.pyindex 29c530e5db3..cc2b188d6d1 100644--- a/studio/backend/core/inference/mlx_speculative.py+++ b/studio/backend/core/inference/mlx_speculative.py@@ -37,7 +37,6 @@ DRAFTER_NOT_FOUND = "drafter_not_found"DRAFTER_INCOMPATIBLE = "drafter_incompatible"DRAFTER_NO_MEMORY = "drafter_no_memory"RUNTIME_ERROR = "runtime_error"-KV_QUANT = "kv_quant"AUTO_CONTEXT_COST = "auto_context_cost"@@ -319,10 +318,8 @@ _MAX_COPY = 16_EXACT_HEAD_DEPTH = 3 # MTP heads and assistants chain a forward per drafted token-def speculation_refusal(*, kv_quant: bool, distributed: bool, lora: bool) -> Optional[str]:+def speculation_refusal(*, distributed: bool, lora: bool) -> Optional[str]:"""The reason an MLX load cannot speculate at all, or None."""- if kv_quant:- return KV_QUANTif distributed or lora:return RUNTIME_ERRORtry:diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 1ab5bc82e1c..3a532abb1f6 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -14943,7 +14943,6 @@ def _mlx_offload_warning(info: dict) -> Optional[str]:def _mlx_estimate_speculates(request, model_dir) -> bool:from core.inference import mlx_speculative- from core.inference.mlx_inference import parse_mlx_kv_quantfrom core.inference.mlx_memory import _loads_as_vision, _snapshot_configmode = mlx_speculative.mlx_spec_mode(request.speculative_type)@@ -14951,10 +14950,7 @@ def _mlx_estimate_speculates(request, model_dir) -> bool:mode, _loads_as_vision(_snapshot_config(model_dir) or {}), request.spec_draft_model):return False- bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)- legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None- kv_quant = bits is not None or turboquant or legacy_bits- return not mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False)+ return not mlx_speculative.speculation_refusal(distributed = False, lora = False)def _mlx_estimate_drafter(@@ -14965,6 +14961,7 @@ def _mlx_estimate_drafter(load_in_4bit: bool,offload_route: Optional[dict] = None,baseline: Optional[dict] = None,+ kv_bits: Optional[int] = None,):"""``((path, builtin), fitted context, speculates, stretched)`` of an MLX load of *request*: the drafter itwould attach, chosen as the load chooses it, whether it speculates at all (so loads through mlx-vlm), and@@ -14992,6 +14989,7 @@ def _mlx_estimate_drafter(load_in_4bit = load_in_4bit,costless = mode == "auto",baseline = baseline,+ kv_bits = kv_bits,**kwargs,**(offload_route or {}),)@@ -15055,6 +15053,7 @@ def _mlx_load_estimate(load_in_4bit,offload,offload_baseline,+ kv_bits,)route = {**({"vision": True} if speculates else {}), **offload}if not named_ctx and fitted_ctx is None:@@ -15081,6 +15080,7 @@ def _mlx_load_estimate(priced_ctx,SimpleNamespace(path = drafter[0], builtin = drafter[1]),load_in_4bit = load_in_4bit,+ kv_bits = kv_bits,**offload,)if breakdown is None:diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex b2443db63c4..6b56a8d3fb3 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -817,10 +817,8 @@ class TestEstimateMemoryRoute:"resolve_speculation",lambda mode, *a, **k: spec.SpecResolution(mode, sources if mode == "mtp" else ()),)- monkeypatch.setattr(- spec, "speculation_refusal", lambda kv_quant, **_: spec.KV_QUANT if kv_quant else None- )- fit = False+ monkeypatch.setattr(spec, "speculation_refusal", lambda **_: None)+ fit, fit_widths = False, []monkeypatch.setattr(mlx_inference,"mlx_drafter_fit",@@ -833,6 +831,12 @@ class TestEstimateMemoryRoute:None if pinned or fit in ("unpriced", "unsizable") else 12288,),)+ priced_fit = mlx_inference.mlx_drafter_fit+ monkeypatch.setattr(+ mlx_inference,+ "mlx_drafter_fit",+ lambda *a, kv_bits = None, **k: fit_widths.append(kv_bits) or priced_fit(*a, **k),+ )monkeypatch.setattr(ri, "_mlx_estimate_ceiling", lambda model_dir: 32768)monkeypatch.setattr(ri, "_mlx_estimate_fitted_context", lambda *a, **kw: 24576 if kw else 20480@@ -863,6 +867,7 @@ class TestEstimateMemoryRoute:for pinned in (4096, 16384, 65536):_estimate(model_path = "org/model", speculative_type = "mtp", max_seq_length = pinned)_estimate(model_path = "org/model", speculative_type = "mtp", mlx_kv_quant = "8")+ assert fit_widths[-1] == 8_estimate(model_path = "org/model")_estimate(model_path = "o/m", speculative_type = "ngram", mlx_kv_quant = "auto", mlx_kv_bits = 8)_estimate(model_path = "org/model", spec_draft_model = "org/d") # named under Auto@@ -872,13 +877,13 @@ class TestEstimateMemoryRoute:_estimate(model_path = "org/model", speculative_type = "mtp")fit = "unsizable" # one the breakdown cannot size leaves the rest of the load pricedassert _estimate(model_path = "org/model", speculative_type = "mtp").available- assert [route for *_, route in priced] == [True] * 4 + [None] * 2 + [True] * 6+ assert [route for *_, route in priced] == [True] * 5 + [None] + [True] * 6assert [row[:2] for row in priced[:7] + priced[8:]] == [(12288, ("/ok", False)),(4096, ("/ok", False)),(16384, ("/big", True)),(65536, None), # past MLX's own cap the load falls back, so no drafter is charged- (20480, None),+ (12288, ("/ok", False)),(20480, None),(24576, None),(12288, ("/big", True)),@@ -890,6 +895,30 @@ class TestEstimateMemoryRoute:assert rooms == [(16384, "/big"), (12288, "/big"), (24576, "/big")]assert [e.cache_room for e in estimates if e and e.cache_room] == [7, 7, 7]+ def test_a_drafter_is_fitted_at_the_load_kv_width(self, monkeypatch):+ import core.inference.mlx_inference as mlx_inference+ import core.inference.mlx_memory as mlx_memory+ from core.inference.mlx_speculative import DrafterSource++ asked = []+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 1)+ monkeypatch.setattr(mlx_inference, "mlx_kv_quant_is_refused", lambda _d, _b: False)+ monkeypatch.setattr(+ mlx_memory,+ "mlx_fit_outcome",+ lambda *_a, **k: asked.append(k["kv_bits"]) or ("fits", None),+ )+ monkeypatch.setattr(+ mlx_memory, "mlx_headroom", lambda *_a, **k: asked.append(k["kv_bits"]) or 1+ )+ source = DrafterSource("mtp", "/d", False)+ mlx_inference.mlx_drafter_fit("/m", 8, None, source, load_in_4bit = False, kv_bits = 8)+ mlx_inference.mlx_prompt_cache_room("/m", 8, source, load_in_4bit = False, kv_bits = 8)+ assert asked == [8, 8]+ monkeypatch.setattr(mlx_inference, "mlx_kv_quant_is_refused", lambda _d, _b: True)+ mlx_inference.mlx_drafter_fit("/m", 8, None, source, load_in_4bit = False, kv_bits = 8)+ assert asked[-1] is None+def test_a_drafter_attaches_where_its_priced_fit_holds_and_a_pin_fits_whole(self, monkeypatch):import core.inference.mlx_inference as mlx_inferenceimport core.inference.mlx_memory as mlx_memorydiff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex ee429bb8542..2484deb6cb3 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1374,6 +1374,7 @@ def test_an_auto_drafter_gets_offload_room_only_within_the_cap_where_it_then_fitTrue,{"load_in_4bit": False,+ "kv_bits": None,"costless": True,"baseline": {"offload": (0.5, 0.5), "prefill_chunk": 8192},"offload": (0.5, 0.53),@@ -1390,6 +1391,7 @@ def test_an_auto_drafter_gets_offload_room_only_within_the_cap_where_it_then_fitassert resolve(False) == ((None, 0.78, 8192), None)assert asked[-1][4] == {"load_in_4bit": False,+ "kv_bits": None,"offload": (None, 1.0),"prefill_chunk": None,"reserve": False,@@ -5376,9 +5378,7 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fi# ... and is a request, so a load that cannot honour it says why.failing.append(True)backend.load_model(config, spec_draft_model = "org/d")- monkeypatch.setattr(- mlx_speculative, "speculation_refusal", lambda **_k: mlx_speculative.KV_QUANT- )+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: "refused")reasons = [entry["spec_fallback_reason"]for entry in (@@ -5386,9 +5386,18 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fibackend.load_model(config, spec_draft_model = "org/d") and backend.models["org/text"],)]- assert reasons == [mlx_speculative.RUNTIME_ERROR, mlx_speculative.KV_QUANT]- # A diffusion model gets no draft at load, so the resident batch cannot speculate on it either.+ assert reasons == [mlx_speculative.RUNTIME_ERROR, "refused"]failing.clear()+ asked = []+ monkeypatch.setattr(+ mlx_speculative, "speculation_refusal", lambda **k: asked.append(k["lora"]) or "refused"+ )+ monkeypatch.setattr(mlx_inference, "load_mlx_adapter_processor", lambda p, *_a: p)+ monkeypatch.setattr(mlx_inference, "load_mlx_adapter_tokenizer", lambda t, *_a: t)+ for is_vision in (False, True):+ adapter = SimpleNamespace(**{**vars(config), "is_lora": True, "is_vision": is_vision})+ backend.load_model(adapter, spec_draft_model = "org/d")+ assert asked == [True, False]monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)monkeypatch.setattr(mlx_inference, "_vlm_generation_is_diffusion", lambda model: True)backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")@@ -7590,21 +7599,24 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):assert seen["draft_model"] is draft and seen["extra"] == 1 and prepared == [([7, 8, 9], 0.5, 0)]# The draft is told every processor the reply runs: none reaches mlx-vlm as a shortcut.assert "repetition_penalty" not in seen and draft.given == seen["logits_processors"] != []+ grammar = SimpleNamespace(allows_reasoning = False, stops_on = lambda ids: None)+ with monkeypatch.context() as patch:+ patch.setattr(mlx_inference, "_build_grammar_constraint", lambda *_a, **_k: grammar)+ seen = _drive_vlm_generation(backend, patch, response_format = {"type": "json_object"})+ assert seen["draft_model"] is draft and [hasattr(p, "settle") for p in draft.given] == [True]monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_: [lambda tokens, logits: logits])timings = backend.last_generation_stats["timings"]assert (timings["draft_n"], timings["draft_n_accepted"]) == (6, 4)- for request in (- {"repetition_penalty": 1.1},- {"tools": [{"type": "function"}]},- {"tool_protocol_active": True},- ):+ for request in ({"repetition_penalty": 1.1},):seen = _drive_vlm_generation(backend, monkeypatch, **request)assert "max_tokens" in seen and "draft_model" not in seenassert "draft_n" not in backend.last_generation_stats["timings"]- # Only the drafted reply entered the verify scope: open from preparing the draft through decoding.- assert len(prepared) == 1 and scopes == [["open"]] * 2 and verify_scope == ["open", "closed"]+ assert len(prepared) == 2 and verify_scope == ["open", "closed"] * 2+ assert scopes == [["open"], ["open"], ["open", "closed", "open"]]+ for request in ({"tools": [{"type": "function"}]}, {"tool_protocol_active": True}):+ assert _drive_vlm_generation(backend, monkeypatch, **request)["draft_model"] is draftopened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r, **_k: Nonebackend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)assert backend.batch_unavailable_reason([{}, {}]) == "no stream"@@ -8790,9 +8802,9 @@ def test_a_vision_row_times_its_prefill_wherever_the_batch_runs_it(monkeypatch,sampling = {},processors = None,think_prefix = "",+ videos = None,stream = _vlm_row(backend),streamed = None,- speculates = True,)backend._plan_vlm_request = lambda request, **_k: planbackend._reads_vision = False@@ -8910,6 +8922,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mosampling = {},processors = None,think_prefix = "",+ videos = None,stream = _vlm_row(backend),streamed = None,)@@ -9116,20 +9129,24 @@ def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):assert backend.resident_unavailable_reason({}) == "old mlx-vlm"[email protected]("gap", (None, "speculative", "old zoo", "no decoder", "no vision"))+ "gap", (None, "drafted", "drafted and quantized", "old zoo", "no decoder", "no vision")+)def test_a_video_reply_batches_only_as_a_fixed_command_of_a_load_that_reads_it(monkeypatch, gap):from core.inference import mlx_inferenceengine, backend = _batch_engine(monkeypatch), _vlm_backend(monkeypatch, markers = None)- backend._speculative_draft = gap == "speculative" or None+ backend._speculative_draft = gap in ("drafted", "drafted and quantized") or None+ backend._kv_quant = {"kv_bits": 8 if gap == "drafted and quantized" else None}backend._reads_vision, available = gap != "no vision", lambda request: Nonebackend._vlm_batch_unavailable_reason = backend._vlm_resident_unavailable_reason = available+ monkeypatch.setattr(mlx_inference, "_speculative_quantized_row_gap", lambda: None)monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: gap != "no decoder")fields = SimpleNamespace(__dataclass_fields__ = ("image", "videos")[: 1 + (gap != "old zoo")])monkeypatch.setattr(engine, "GenerationRequest", fields, raising = False)clip, sent = {"videos": [_CLIP_B64]}, {"videos_base64": [_CLIP_B64]}for row in (clip, sent):- assert (backend.batch_unavailable_reason([{}, row]) is None) == (gap is None)+ assert (backend.batch_unavailable_reason([{}, row]) is None) == (gap in (None, "drafted"))assert backend.resident_unavailable_reason(sent) == "a reply carries a video clip"assert backend.resident_unavailable_reason({}) is None@@ -9298,6 +9315,100 @@ def test_an_out_of_range_logit_bias_id_cannot_corrupt_a_batch(batch_backend):assert replies[3] != replies[0]+DRAFTING_MODEL = "Qwen/Qwen3.5-0.8B"+CITY_SCHEMA = {+ "type": "object",+ "properties": {+ "city": {"type": "string"},+ "country": {"type": "string"},+ "landmarks": {"type": "array", "items": {"type": "string"}},+ },+ "required": ["city", "country", "landmarks"],+ "additionalProperties": False,+}++[email protected](scope = "module")+def drafting_backend(allow_outbound_network):+ from core.inference.mlx_inference import MLXInferenceBackend++ loaded = MLXInferenceBackend()+ with allow_outbound_network():+ loaded.load_model(DRAFTING_MODEL, speculative_type = "mtp")+ assert loaded._speculative_draft is not None and loaded._speculative_draft.drafter is not None+ return loaded+++def _drafted_reply(backend, drafted, **request):+ held = backend._speculative_draft+ backend._speculative_draft = held if drafted else None+ # Each reply prefills its prompt: a drafted reply resumed from a snapshot can round a tie differently.+ backend._release_vlm_snapshots()+ try:+ text = ""+ for text in backend.generate_chat_response(**request):+ pass+ finally:+ backend._speculative_draft = held+ return text, backend.last_generation_stats["timings"]+++def _is_city(text):+ city = json.loads(text)+ return (+ set(city) == set(CITY_SCHEMA["properties"])+ and isinstance(city["city"], str)+ and isinstance(city["country"], str)+ and isinstance(city["landmarks"], list)+ and all(isinstance(landmark, str) for landmark in city["landmarks"])+ )+++@metal_only+def test_a_drafted_constrained_reply_is_the_plain_constrained_reply(drafting_backend):+ from unsloth_zoo.mlx import speculative++ if not hasattr(speculative._Row, "settle"):+ pytest.skip("this unsloth-zoo decodes a constrained reply without drafting")+ request = dict(+ messages = [+ {"role": "user", "content": "Describe Paris as JSON with three famous landmarks."}+ ],+ response_format = {"type": "json_schema", "schema": CITY_SCHEMA},+ enable_thinking = False,+ max_new_tokens = 96,+ )+ greedy = dict(request, temperature = 0.0, top_p = 1.0, top_k = 0)+ drafted, timings = _drafted_reply(drafting_backend, True, **greedy)+ plain, plain_timings = _drafted_reply(drafting_backend, False, **greedy)+ assert timings["draft_n_accepted"] > 0+ assert drafted == plain and timings["predicted_n"] == plain_timings["predicted_n"]+ assert _is_city(drafted)+ for seed in (1, 2):+ sampled, timings = _drafted_reply(+ drafting_backend, True, **request, temperature = 0.8, top_p = 0.95, top_k = 0, seed = seed+ )+ assert timings["draft_n_accepted"] > 0 and _is_city(sampled)+++def test_a_row_keeps_a_settling_failure_rather_than_raising_it_through_the_batch():+ from core.inference import mlx_inference++ def processor(tokens, logits):+ return logits++ def settle(tokens):+ raise ValueError("refused")++ processor.settle = settle+ row = SimpleNamespace(failure = None)+ (guarded,) = mlx_inference._failing_alone(row, [processor])+ guarded.settle([1])+ assert str(row.failure) == "refused"+ plain = mlx_inference._failing_alone(row, [lambda tokens, logits: logits])[0]+ assert not hasattr(plain, "settle")++def test_a_repeated_token_is_charged_the_penalty_it_earned():"""The frequency processor counts occurrences in float32 and scales once."""mx = pytest.importorskip("mlx.core")@@ -9676,6 +9787,83 @@ def test_advance_before_the_first_mask_raises_rather_than_dropping_the_token(tinconstraint.advance(tiny_tokenizer.encode("{", add_special_tokens = False)[0])+def _grammar_after_prompt(+ tokenizer,+ response_format = JSON_SCHEMA_FORMAT,+ prompt = (0, 0),+ **build,+):+ mx = _metal_mx()+ constraint = build_constraint(response_format, tokenizer, build.pop("text", "p"), **build)+ processor = gc.make_grammar_logits_processor(constraint)+ width = len(gc._unwrap_hf_tokenizer(tokenizer))+ processor(mx.array(prompt), mx.zeros((width,), dtype = mx.float32))+ return processor+++def _mask_at(processor, history, width):+ mx = _metal_mx()+ return np.asarray(processor(mx.array(history), mx.zeros((width,), dtype = mx.float32)))++[email protected]("in_reasoning", [False, True])+def test_a_settled_grammar_masks_each_drafted_position_as_a_plain_decode_would(+ tiny_tokenizer, in_reasoning+):+ mx, width, prompt = _metal_mx(), len(tiny_tokenizer), [0, 0]+ encode = lambda text: tiny_tokenizer.encode(text, add_special_tokens = False)+ eos = tiny_tokenizer.eos_token_id+ build = dict(text = "chat\n<think>\n", reasoning_is_extracted = True) if in_reasoning else {}+ grammar = lambda: _grammar_after_prompt(tiny_tokenizer, **build)++ def plain(generated):+ return _mask_at(grammar(), prompt + generated, width)++ drafting = grammar()+ committed = encode("ok" if in_reasoning else "{")+ drafting.settle(mx.array(prompt + committed))+ for drafted in (encode('"a":12}'), encode('"a":3'), encode('"a":45}')):+ for j in range(len(drafted) + 1):+ np.testing.assert_array_equal(+ _mask_at(drafting, prompt + committed + drafted[:j], width),+ plain(committed + drafted[:j]),+ )+ refused = [encode('"a":1}') + [eos] + encode("1")] + ([] if in_reasoning else [encode('"a"]1')])+ for drafted in refused:+ assert not np.isneginf(_mask_at(drafting, prompt + committed + drafted, width)).any()+ committed += encode('"a":7')+ drafting.settle(mx.array(prompt + committed))+ np.testing.assert_array_equal(_mask_at(drafting, prompt + committed, width), plain(committed))+ if not in_reasoning:+ # Rows take tokens past an end token the decode loop stops on, which a plain advance ignores.+ drafting.settle(mx.array(prompt + committed + encode("}") + [eos] + encode("1")))+++def test_a_drafted_state_the_schema_cannot_spell_fails_only_once_accepted():+ pytest.importorskip("mlx_lm")+ from mlx_lm.tokenizer_utils import TokenizerWrapper++ mx = _metal_mx()+ tokenizer, runtime_stop = _two_stop_tokenizer()+ wrapped, width = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop]), len(tokenizer)+ control = tokenizer.convert_ids_to_tokens(runtime_stop)+ either = {"type": "json_schema", "schema": {"anyOf": [{"const": control}, {"type": "integer"}]}}+ quote, digit = (tokenizer.encode(text, add_special_tokens = False)[0] for text in '"4')+ for accepted in ([digit], [quote], [quote, digit]):+ processor = _grammar_after_prompt(wrapped, either, prompt = [0])+ processor.settle(mx.array([0]))+ assert not np.isneginf(_mask_at(processor, [0, quote], width)).any()+ if accepted[0] == quote:+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):+ processor.settle(mx.array([0, *accepted]))+ else:+ processor.settle(mx.array([0, digit]))+ assert set(np.nonzero(~np.isneginf(_mask_at(processor, [0, digit], width)))[0]) >= {+ digit,+ runtime_stop,+ }++def _uncopyable_naive_detokenizer(detokenizers):"""The pre-0.6.0 naive detokenizer, whose copy raises; pinned by behaviour, not the installed wheel."""@@ -10842,6 +11030,102 @@ def _video_vlm_backend(monkeypatch, streams):return backend+def test_a_quantized_speculative_load_batches_unless_its_caches_are_turboquant(monkeypatch):+ from core.inference import mlx_inference++ gaps = {"row": None, "drafted": "no drafted rows"}+ monkeypatch.setattr(mlx_inference, "_row_quantized_cache_gap", lambda: gaps["row"])+ monkeypatch.setattr(mlx_inference, "_speculative_quantized_row_gap", lambda: gaps["drafted"])+ backend = mlx_inference.MLXInferenceBackend()+ backend._is_vlm, backend._kv_quant = True, {"kv_bits": 4}+ assert backend._kv_policy_batch_reason(resident = True) is None+ backend._speculative_draft = object()+ assert backend._kv_policy_batch_reason(resident = True) == "no drafted rows"+ gaps["drafted"] = None+ assert backend._kv_policy_batch_reason(resident = True) is None+ assert "does not carry" in backend._kv_policy_batch_reason()+ backend._turboquant = True+ assert "TurboQuant" in backend._kv_policy_batch_reason(resident = True)+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": None}, 1+ gaps["row"] = "no rows"+ assert backend._kv_policy_batch_reason(resident = True) == "no rows"+++def test_a_drafted_batch_row_decodes_its_own_clip_and_resumes_no_cache(monkeypatch):+ from contextlib import ExitStack++ from core.inference import mlx_inference++ engine = _batch_engine(monkeypatch)+ monkeypatch.setattr(engine, "GenerationRequest", lambda **k: k, raising = False)+ monkeypatch.setattr(engine, "SamplingParams", lambda **k: k, raising = False)+ monkeypatch.setattr(engine, "BatchRowRefused", ValueError, raising = False)+ clips = {"videos": ["clip.mp4"], "fps": 2.0}+ monkeypatch.setattr(mlx_inference, "_batch_clips", lambda stack, plans, _p: [clips])+ added, released = [], []+ plan = SimpleNamespace(+ videos = ("a",),+ images = None,+ prompt = "p",+ max_tokens = 4,+ sampling = {},+ processors = None,+ think_prefix = "",+ )+ session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)+ session.backend = SimpleNamespace(+ _plan_vlm_request = lambda *_a, **_k: plan,+ _vlm_batch_row_cache = lambda *_a, **_k: pytest.fail("a clip's row resumes no cache"),+ _kv_quant_bits = lambda: None,+ _release_vlm_snapshots = lambda: released.append(True),+ _vlm_batch_prompt = lambda prompt: {"prompt": prompt},+ _processor = None,+ _model = None,+ )+ session.stream = SimpleNamespace(add = lambda request: added.append(request) or 0)+ session._adapter_state, session._decodes_ids, session._resumes_rows = None, False, True+ session._retires_rows, session._speculative, session._held = False, object(), ExitStack()+ session._rows, session._by_row = {}, {}+ session.admit({}, 0)+ assert added[0]["videos"] == ["clip.mp4"] and added[0]["fps"] == 2.0+ assert "prompt_cache_state" not in added[0] and released == [True]+++def test_a_speculative_load_drafts_a_video_reply_on_its_expanded_prompt(monkeypatch):+ from core.inference import mlx_inference++ streams, prepared, drafted = [], [], []+ backend = _video_vlm_backend(monkeypatch, streams)+ backend._vlm_is_diffusion_model = lambda _model: False+ monkeypatch.setattr(mlx_inference, "_video_sampling_fps", lambda *_a: 2.0)++ class _Ids(list):+ def tolist(self):+ return list(self)++ utils = types.ModuleType("mlx_vlm.utils")+ utils.prepare_inputs = lambda *_a, **kwargs: prepared.append(kwargs) or {+ "input_ids": [_Ids([7, 7, 8])],+ "pixel_values_videos": "frames",+ }+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)+ backend._speculative_draft = SimpleNamespace(+ draft_kind = "mtp",+ draft_n = 0,+ draft_n_accepted = 0,+ prepare = lambda ids, *_a: drafted.append(ids),+ )+ turn = [+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}+ ]+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]+ ((_, stream_kwargs, _),) = streams+ assert drafted == [[7, 7, 8]] and stream_kwargs["draft_model"] is backend._speculative_draft+ assert stream_kwargs["pixel_values_videos"] == "frames" and prepared[0]["fps"] == 2.0+ assert len(prepared[0]["videos"]) == 1++def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch):"""A snapshot spans the video rows but is keyed by token ids the clip does not vary, so alater clip would resume the earlier one's vision rows and its mRoPE grid."""diff --git a/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.tsindex c8ecfb4dee7..881c889f9d9 100644--- a/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts+++ b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts@@ -30,8 +30,6 @@ export function mlxSpecFallbackMessage(return `The drafter could not be confirmed to fit in memory beside this model, ${outcome}. When memory is the limit, a shorter context length or a smaller drafter can help.`;case "runtime_error":return `Speculative decoding could not start for this load, ${outcome}.`;- case "kv_quant":- return "Speculative decoding cannot run with KV cache quantization, which this load asked for. Setting KV Cache Dtype to Auto in this model's settings removes that obstacle.";case "auto_context_cost":return `Auto left out a drafter that does not fit beside this model at its full context length, ${outcome}. To try it anyway, choose its mode in this model's settings, with a shorter or automatic context length.`;default:diff --git a/studio/frontend/tests/mlx-spec-fallback-copy.test.ts b/studio/frontend/tests/mlx-spec-fallback-copy.test.tsindex 1a941c0a1c4..d8cc7e48a50 100644--- a/studio/frontend/tests/mlx-spec-fallback-copy.test.ts+++ b/studio/frontend/tests/mlx-spec-fallback-copy.test.ts@@ -16,7 +16,7 @@ test("an MLX fallback notice says what runs in the passed-over drafter's place",test("an MLX fallback notice claims no more than every path behind its code", () => {// No code guarantees its remedy works, and drafter_no_memory also covers an unpriced fit.- for (const code of ["drafter_not_found", "drafter_incompatible", "drafter_no_memory", "runtime_error", "kv_quant", "auto_context_cost"]) {+ for (const code of ["drafter_not_found", "drafter_incompatible", "drafter_no_memory", "runtime_error", "auto_context_cost"]) {assert.doesNotMatch(mlxSpecFallbackMessage(code, null) ?? "missing", /to use it|missing/);}assert.doesNotMatch(mlxSpecFallbackMessage("drafter_no_memory", null) ?? "", /does not fit/);