Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

feat(studio): draft MLX tool, video, grammar, KV and LoRA replies (#16441)

Long Yixingb0a6f4b8ec9b
diff --git a/studio/backend/core/inference/grammar_constraint.py b/studio/backend/core/inference/grammar_constraint.py
index 6637fee4741..8b5a806f415 100644
--- a/studio/backend/core/inference/grammar_constraint.py
+++ b/studio/backend/core/inference/grammar_constraint.py
@@ -273,6 +273,9 @@ class GrammarConstraint:
self._strips_special_tokens = strips_special_tokens
self._dropped_ids = None
self._runtime_stops = None
+ self._settled = None
+ self._drafted = []
+ self._deferred = {}
self.allows_reasoning = allows_reasoning
def _bind(self, n_vocab: int) -> None:
@@ -343,6 +346,46 @@ class GrammarConstraint:
raise ResponseFormatError(self._desync_message(int(token_id)))
raise GrammarDesyncError(f"guided decoding desynced on token {int(token_id)}: {error}")
+ def follow(self, drafted) -> bool:
+ kept = _common_prefix(self._drafted, drafted)
+ rest = drafted[kept:]
+ if kept < len(self._drafted):
+ self._matcher, self._settled, self._drafted, rest = self._settled, None, [], drafted
+ self._deferred = {
+ depth: error for depth, error in self._deferred.items() if depth <= kept
+ }
+ if not rest:
+ return True
+ if self._settled is None:
+ # llguidance's rollback misrestores a max_tokens rule (the prelude) and an end token.
+ self._settled, self._matcher = self._matcher, self._matcher.deep_copy()
+ admitted = self._matcher.try_consume_tokens(rest)
+ self._drafted.extend(rest[:admitted])
+ return admitted == len(rest)
+
+ def mask_ahead(self, logits):
+ try:
+ return self.mask_logits(logits)
+ except ResponseFormatError as error:
+ self._deferred.setdefault(len(self._drafted), error)
+ return logits
+
+ def commit(self, tokens) -> None:
+ reached = [
+ depth for depth in self._deferred if depth <= _common_prefix(self._drafted, tokens)
+ ]
+ if reached:
+ raise self._deferred[min(reached)]
+ if self._settled is not None:
+ self._matcher, self._settled = self._settled, None
+ self._drafted, self._deferred = [], {}
+ for token_id in tokens:
+ if self._matcher is not None and not self._matcher.is_stopped():
+ if self._matcher.try_consume_tokens([int(token_id)]):
+ continue
+ _llg_mlx.fill_next_token_bitmask(self._matcher, self._bitmask)
+ self.advance(token_id)
+
def _mask_allowed(self, token_id: int) -> bool:
try:
import numpy as np
@@ -361,19 +404,39 @@ class GrammarConstraint:
)
+def _common_prefix(held, tokens) -> int:
+ kept = 0
+ for a, b in zip(held, tokens):
+ if a != b:
+ break
+ kept += 1
+ return kept
+
+
def make_grammar_logits_processor(constraint: GrammarConstraint):
- state = {"seen": None}
+ """Each call commits its history until ``settle`` is first called; then only ``settle`` does."""
+ state = {"seen": None, "committed": None}
def _processor(tokens, logits):
- seen = state["seen"]
+ seen, committed = state["seen"], state["committed"]
if seen is None:
state["seen"] = int(tokens.shape[0])
- else:
+ elif committed is None:
for token_id in tokens[seen:].tolist():
constraint.advance(int(token_id))
state["seen"] = int(tokens.shape[0])
+ elif not constraint.follow(tokens[committed:].tolist()):
+ return logits
+ elif int(tokens.shape[0]) > committed:
+ return constraint.mask_ahead(logits)
return constraint.mask_logits(logits)
+ def settle(tokens):
+ start = state["seen"] if state["committed"] is None else state["committed"]
+ constraint.commit(tokens[start:].tolist())
+ state["committed"] = int(tokens.shape[0])
+
+ _processor.settle = settle
return _processor
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 62d7a9e8ed0..84d61793145 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -1431,6 +1431,7 @@ def mlx_resolve_drafter_offload(model_dir, knobs, drafter, *, auto: bool, **pric
priced["pinned"],
SimpleNamespace(path = drafter[0], builtin = drafter[1]),
load_in_4bit = priced["load_in_4bit"],
+ kv_bits = priced.get("kv_bits"),
**kwargs,
**mlx_offload_route(plan),
)[0]
@@ -3322,6 +3323,7 @@ def mlx_drafter_fit(
prefill_chunk = None,
baseline = None,
reserve = True,
+ kv_bits = None,
):
"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:
only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit
@@ -3336,6 +3338,8 @@ def mlx_drafter_fit(
try:
from core.inference.mlx_memory import mlx_fit_outcome
+ if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):
+ kv_bits = None
fit = functools.partial(
mlx_fit_outcome,
model_dir,
@@ -3345,6 +3349,7 @@ def mlx_drafter_fit(
vision = True,
offload = offload,
prefill_chunk = prefill_chunk,
+ kv_bits = kv_bits,
)
outcome, fitted = fit(drafter = (source.path, source.builtin))
if outcome == "unsizable":
@@ -3365,6 +3370,7 @@ def mlx_prompt_cache_room(
load_in_4bit: bool,
offload = None,
prefill_chunk = None,
+ kv_bits = None,
):
"""Bytes MLX's own cap leaves the prompt cache beside *source*'s drafter at *n_ctx*; None where unsized."""
budget = mlx_memory_budget(retains_history = False)
@@ -3382,6 +3388,7 @@ def mlx_prompt_cache_room(
drafter = (source.path, source.builtin),
offload = offload,
prefill_chunk = prefill_chunk,
+ kv_bits = kv_bits,
)
except Exception as exc:
logger.info("MLX prompt cache room beside %s not priced: %s", source.path, exc)
@@ -4256,7 +4263,7 @@ def _vlm_add_special_tokens(model_type, processor):
)
-def _vlm_prepared_inputs(model, processor, prompt, images):
+def _vlm_prepared_inputs(model, processor, prompt, images, **media):
"""The processor's inputs for *prompt* and *images*, as mlx-vlm's ``stream_generate`` prepares them."""
from mlx_vlm.utils import prepare_inputs
@@ -4268,6 +4275,7 @@ def _vlm_prepared_inputs(model, processor, prompt, images):
prompts = prompt,
image_token_index = read("image_token_index"),
add_special_tokens = _vlm_add_special_tokens(read("model_type"), processor),
+ **media,
)
@@ -4595,8 +4603,17 @@ def _failing_alone(row, processors):
row.failure = exc
return logits
- # A drafted batch drafts only for rows whose processors all read just the history.
_processor.history_only = getattr(processor, "history_only", False)
+ if hasattr(processor, "settle"):
+
+ def _settle(tokens):
+ if row.failure is None:
+ try:
+ processor.settle(tokens)
+ except Exception as exc:
+ row.failure = exc
+
+ _processor.settle = _settle
return _processor
return [guarded(processor) for processor in processors]
@@ -4885,7 +4902,6 @@ class _VLMRowPlan:
"max_tokens",
"processors",
"streamed",
- "speculates",
)
def __init__(
@@ -4900,7 +4916,6 @@ class _VLMRowPlan:
max_tokens,
processors = None,
streamed = None,
- speculates = True,
videos = None,
):
self.prompt = prompt
@@ -4913,7 +4928,6 @@ class _VLMRowPlan:
self.max_tokens = max_tokens
self.processors = processors
self.streamed = streamed
- self.speculates = speculates
class _VisionBatchRow:
@@ -4976,6 +4990,14 @@ def _row_retire_gap():
return row_retire_unavailable_reason()
+def _speculative_quantized_row_gap():
+ try:
+ from unsloth_zoo.mlx.generate import speculative_quantized_rows_unavailable_reason
+ except ImportError:
+ return "the installed unsloth-zoo cannot draft a batch on a quantized KV cache"
+ return speculative_quantized_rows_unavailable_reason()
+
+
def _row_quantized_cache_gap():
try:
from unsloth_zoo.mlx.generate import row_quantized_prompt_cache_unavailable_reason
@@ -5056,19 +5078,21 @@ class _VisionBatchSession:
plan = backend._plan_vlm_request(request, decodes_ids = self._decodes_ids)
row = _VisionBatchRow(handle = handle, plan = plan)
alone = self._alone(handle) if self._retires_rows else None
+ # A clip's row resumes nothing: token ids do not vary with the clip.
state = (
backend._vlm_batch_row_cache(self._adapter_state, plan.images, alone = alone)
- if self._resumes_rows
+ if self._resumes_rows and not plan.videos
else None
)
if state is None and backend._kv_quant_bits() is not None:
# mlx-vlm's own batch cache would be float.
raise RowRefused("this row cannot carry the load's quantized KV cache")
- if state is None and plan.images:
+ if state is None and (plan.images or plan.videos):
backend._release_vlm_snapshots()
elif state is None:
backend._release_held_vlm_snapshots()
resume = {} if state is None else {"prompt_cache_state": state}
+ clips = _batch_clips(self._held, [plan], backend._processor)[0] if plan.videos else {}
admitted_at = time.perf_counter()
try:
row_number = self.stream.add(
@@ -5078,8 +5102,8 @@ class _VisionBatchSession:
max_tokens = plan.max_tokens,
sampling = SamplingParams(**plan.sampling),
logits_processors = plan.processors and _failing_alone(row, plan.processors),
+ **clips,
**resume,
- **({} if self._speculative is None or plan.speculates else {"drafts": False}),
)
)
except BatchRowRefused as refusal:
@@ -6000,9 +6024,9 @@ class MLXInferenceBackend:
)
if speculates:
spec_reason = mlx_speculative.speculation_refusal(
- kv_quant = kv_bits is not None or self._turboquant,
distributed = is_distributed,
- lora = is_lora,
+ # A text architecture's adapter loads only through mlx-lm, and speculation runs on mlx-vlm.
+ lora = is_lora and not (is_vision or zoo_multimodal),
)
speculates = spec_reason is None
if (
@@ -6175,7 +6199,7 @@ class MLXInferenceBackend:
if spec_mode == "auto" and not spec_draft_model
else mlx_speculative.RUNTIME_ERROR
)
- elif self._turboquant:
+ if self._turboquant:
self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD
use_vlm = False
if model is None:
@@ -6257,6 +6281,16 @@ class MLXInferenceBackend:
allowed = spec_drafters_allowed,
)
+ if not self._turboquant and kv_bits is not None and _eligibility is None:
+ _eligibility = _kv_quant_eligibility(
+ self._model, use_vlm, _normalize_mlx_kv_bits(kv_bits)
+ )
+ _drafter_bits = (
+ _normalize_mlx_kv_bits(kv_bits)
+ if not self._turboquant and _eligibility and _eligibility[0] in ("full", "partial")
+ else None
+ )
+
def _fits(source, reserve = True):
if not _priceable:
return None, None
@@ -6269,6 +6303,7 @@ class MLXInferenceBackend:
costless = spec_mode == "auto",
baseline = offload_baseline["route"],
reserve = reserve,
+ kv_bits = _drafter_bits,
**_offloaded,
)
@@ -6332,7 +6367,12 @@ class MLXInferenceBackend:
and getattr(self._speculative_draft, "drafter", None) is not None
):
self._prompt_cache_room = mlx_prompt_cache_room(
- model_dir, _served_ctx, _stretched[-1], load_in_4bit = load_in_4bit, **_offloaded
+ model_dir,
+ _served_ctx,
+ _stretched[-1],
+ load_in_4bit = load_in_4bit,
+ kv_bits = _drafter_bits,
+ **_offloaded,
)
if self._prompt_cache_room is not None:
logger.info(
@@ -7845,8 +7885,6 @@ class MLXInferenceBackend:
max_tokens = vlm_kwargs["max_tokens"],
processors = processors,
streamed = streamed,
- # As _generate_vlm, which drops the draft for tool turns.
- speculates = not (tools or tool_protocol_active),
)
def _generate_vlm(
@@ -8067,18 +8105,16 @@ class MLXInferenceBackend:
vlm_kwargs.update(self._prefill_step_kwargs())
session_scope = session if session is not None else nullcontext()
processors = vlm_kwargs.get("logits_processors") or ()
- # The draft is told these processors, and takes only ones that read just the history (a grammar's
- # matcher advances per call); tool turns decode as before; a clip is nothing a drafter was built for.
if (
draft is None
- or tools
- or tool_protocol_active
- or videos
- or not all(getattr(processor, "history_only", False) for processor in processors)
+ or not all(
+ getattr(processor, "history_only", False) or hasattr(processor, "settle")
+ for processor in processors
+ )
or _vlm_generation_is_diffusion(self._model)
):
draft = None
- elif "input_ids" not in vlm_kwargs:
+ elif "input_ids" not in vlm_kwargs and not videos:
vlm_kwargs.update(
_vlm_generate_kwargs(
_vlm_prepared_inputs(self._model, self._processor, prompt, images)
@@ -8107,30 +8143,39 @@ class MLXInferenceBackend:
)
final_response = None
clip_paths = []
- if draft is not None:
- from unsloth_zoo.mlx.generate import SamplingParams
-
- # Under the lock: the load's one draft serves one generation at a time.
- draft.prepare(
- vlm_kwargs["input_ids"][0].tolist(),
- SamplingParams(
- temperature = temperature,
- top_p = top_p,
- top_k = max(int(top_k or 0), 0),
- min_p = float(min_p or 0.0),
- seed = seed,
- ),
- processors,
- )
- vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)
try:
if videos:
for clip in videos:
clip_paths.append(_write_video_clip(clip))
- vlm_kwargs["video"] = clip_paths
+ clip_media = {"video": clip_paths}
frame_rate = _video_sampling_fps(clip_paths, self._processor)
if frame_rate is not None:
- vlm_kwargs["fps"] = frame_rate
+ clip_media["fps"] = frame_rate
+ if draft is None or "input_ids" in vlm_kwargs:
+ vlm_kwargs.update(clip_media)
+ else:
+ clip_media["videos"] = clip_media.pop("video")
+ vlm_kwargs.update(
+ _vlm_generate_kwargs(
+ _vlm_prepared_inputs(
+ self._model, self._processor, prompt, images, **clip_media
+ )
+ )
+ )
+ if draft is not None:
+ from unsloth_zoo.mlx.generate import SamplingParams
+ draft.prepare(
+ vlm_kwargs["input_ids"][0].tolist(),
+ SamplingParams(
+ temperature = temperature,
+ top_p = top_p,
+ top_k = max(int(top_k or 0), 0),
+ min_p = float(min_p or 0.0),
+ seed = seed,
+ ),
+ processors,
+ )
+ vlm_kwargs.update(draft_model = draft, draft_kind = draft.draft_kind)
# Emit prefilled <think> first; inside the adapter so unsupported requests raise before any output.
if prefill:
yield prefill
@@ -8285,6 +8330,10 @@ class MLXInferenceBackend:
return None
# Only resident vision rows carry their own cache, where the quantization lives.
if resident and self._is_vlm:
+ if self._kv_quant_bits() is not None and getattr(self, "_speculative_draft", None):
+ if getattr(self, "_turboquant", False):
+ return "a speculative batch cannot run TurboQuant caches"
+ return _speculative_quantized_row_gap()
return _row_quantized_cache_gap()
return "the load quantizes or budgets its KV cache, which a batch does not carry"
@@ -8298,7 +8347,9 @@ class MLXInferenceBackend:
reason = self._video_batch_gap(requests)
if reason is not None:
return reason
- reason = self._kv_policy_batch_reason()
+ reason = self._kv_policy_batch_reason(
+ resident = getattr(self, "_speculative_draft", None) is not None
+ )
if reason is not None:
return reason
if len(requests) < 2:
@@ -8336,8 +8387,9 @@ class MLXInferenceBackend:
# The one-at-a-time decode refuses these clips by name.
if not self._is_vlm or not self._reads_vision_input() or not _mlx_vlm_decodes_video():
return "a reply carries a video clip this load does not read"
- if self._speculative_draft is not None:
- return "a video clip decodes without the draft"
+ if self._speculative_draft is not None and self._kv_quant_bits() is not None:
+ # A clip's row resumes no cache, and only a resumed row carries the quantization.
+ return "a quantized speculative batch does not decode a video clip"
try:
from unsloth_zoo.mlx import generate as engine
except ImportError:
diff --git a/studio/backend/core/inference/mlx_speculative.py b/studio/backend/core/inference/mlx_speculative.py
index 29c530e5db3..cc2b188d6d1 100644
--- a/studio/backend/core/inference/mlx_speculative.py
+++ b/studio/backend/core/inference/mlx_speculative.py
@@ -37,7 +37,6 @@ DRAFTER_NOT_FOUND = "drafter_not_found"
DRAFTER_INCOMPATIBLE = "drafter_incompatible"
DRAFTER_NO_MEMORY = "drafter_no_memory"
RUNTIME_ERROR = "runtime_error"
-KV_QUANT = "kv_quant"
AUTO_CONTEXT_COST = "auto_context_cost"
@@ -319,10 +318,8 @@ _MAX_COPY = 16
_EXACT_HEAD_DEPTH = 3 # MTP heads and assistants chain a forward per drafted token
-def speculation_refusal(*, kv_quant: bool, distributed: bool, lora: bool) -> Optional[str]:
+def speculation_refusal(*, distributed: bool, lora: bool) -> Optional[str]:
"""The reason an MLX load cannot speculate at all, or None."""
- if kv_quant:
- return KV_QUANT
if distributed or lora:
return RUNTIME_ERROR
try:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1ab5bc82e1c..3a532abb1f6 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -14943,7 +14943,6 @@ def _mlx_offload_warning(info: dict) -> Optional[str]:
def _mlx_estimate_speculates(request, model_dir) -> bool:
from core.inference import mlx_speculative
- from core.inference.mlx_inference import parse_mlx_kv_quant
from core.inference.mlx_memory import _loads_as_vision, _snapshot_config
mode = mlx_speculative.mlx_spec_mode(request.speculative_type)
@@ -14951,10 +14950,7 @@ def _mlx_estimate_speculates(request, model_dir) -> bool:
mode, _loads_as_vision(_snapshot_config(model_dir) or {}), request.spec_draft_model
):
return False
- bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)
- legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None
- kv_quant = bits is not None or turboquant or legacy_bits
- return not mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False)
+ return not mlx_speculative.speculation_refusal(distributed = False, lora = False)
def _mlx_estimate_drafter(
@@ -14965,6 +14961,7 @@ def _mlx_estimate_drafter(
load_in_4bit: bool,
offload_route: Optional[dict] = None,
baseline: Optional[dict] = None,
+ kv_bits: Optional[int] = None,
):
"""``((path, builtin), fitted context, speculates, stretched)`` of an MLX load of *request*: the drafter it
would attach, chosen as the load chooses it, whether it speculates at all (so loads through mlx-vlm), and
@@ -14992,6 +14989,7 @@ def _mlx_estimate_drafter(
load_in_4bit = load_in_4bit,
costless = mode == "auto",
baseline = baseline,
+ kv_bits = kv_bits,
**kwargs,
**(offload_route or {}),
)
@@ -15055,6 +15053,7 @@ def _mlx_load_estimate(
load_in_4bit,
offload,
offload_baseline,
+ kv_bits,
)
route = {**({"vision": True} if speculates else {}), **offload}
if not named_ctx and fitted_ctx is None:
@@ -15081,6 +15080,7 @@ def _mlx_load_estimate(
priced_ctx,
SimpleNamespace(path = drafter[0], builtin = drafter[1]),
load_in_4bit = load_in_4bit,
+ kv_bits = kv_bits,
**offload,
)
if breakdown is None:
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index b2443db63c4..6b56a8d3fb3 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -817,10 +817,8 @@ class TestEstimateMemoryRoute:
"resolve_speculation",
lambda mode, *a, **k: spec.SpecResolution(mode, sources if mode == "mtp" else ()),
)
- monkeypatch.setattr(
- spec, "speculation_refusal", lambda kv_quant, **_: spec.KV_QUANT if kv_quant else None
- )
- fit = False
+ monkeypatch.setattr(spec, "speculation_refusal", lambda **_: None)
+ fit, fit_widths = False, []
monkeypatch.setattr(
mlx_inference,
"mlx_drafter_fit",
@@ -833,6 +831,12 @@ class TestEstimateMemoryRoute:
None if pinned or fit in ("unpriced", "unsizable") else 12288,
),
)
+ priced_fit = mlx_inference.mlx_drafter_fit
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_drafter_fit",
+ lambda *a, kv_bits = None, **k: fit_widths.append(kv_bits) or priced_fit(*a, **k),
+ )
monkeypatch.setattr(ri, "_mlx_estimate_ceiling", lambda model_dir: 32768)
monkeypatch.setattr(
ri, "_mlx_estimate_fitted_context", lambda *a, **kw: 24576 if kw else 20480
@@ -863,6 +867,7 @@ class TestEstimateMemoryRoute:
for pinned in (4096, 16384, 65536):
_estimate(model_path = "org/model", speculative_type = "mtp", max_seq_length = pinned)
_estimate(model_path = "org/model", speculative_type = "mtp", mlx_kv_quant = "8")
+ assert fit_widths[-1] == 8
_estimate(model_path = "org/model")
_estimate(model_path = "o/m", speculative_type = "ngram", mlx_kv_quant = "auto", mlx_kv_bits = 8)
_estimate(model_path = "org/model", spec_draft_model = "org/d") # named under Auto
@@ -872,13 +877,13 @@ class TestEstimateMemoryRoute:
_estimate(model_path = "org/model", speculative_type = "mtp")
fit = "unsizable" # one the breakdown cannot size leaves the rest of the load priced
assert _estimate(model_path = "org/model", speculative_type = "mtp").available
- assert [route for *_, route in priced] == [True] * 4 + [None] * 2 + [True] * 6
+ assert [route for *_, route in priced] == [True] * 5 + [None] + [True] * 6
assert [row[:2] for row in priced[:7] + priced[8:]] == [
(12288, ("/ok", False)),
(4096, ("/ok", False)),
(16384, ("/big", True)),
(65536, None), # past MLX's own cap the load falls back, so no drafter is charged
- (20480, None),
+ (12288, ("/ok", False)),
(20480, None),
(24576, None),
(12288, ("/big", True)),
@@ -890,6 +895,30 @@ class TestEstimateMemoryRoute:
assert rooms == [(16384, "/big"), (12288, "/big"), (24576, "/big")]
assert [e.cache_room for e in estimates if e and e.cache_room] == [7, 7, 7]
+ def test_a_drafter_is_fitted_at_the_load_kv_width(self, monkeypatch):
+ import core.inference.mlx_inference as mlx_inference
+ import core.inference.mlx_memory as mlx_memory
+ from core.inference.mlx_speculative import DrafterSource
+
+ asked = []
+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 1)
+ monkeypatch.setattr(mlx_inference, "mlx_kv_quant_is_refused", lambda _d, _b: False)
+ monkeypatch.setattr(
+ mlx_memory,
+ "mlx_fit_outcome",
+ lambda *_a, **k: asked.append(k["kv_bits"]) or ("fits", None),
+ )
+ monkeypatch.setattr(
+ mlx_memory, "mlx_headroom", lambda *_a, **k: asked.append(k["kv_bits"]) or 1
+ )
+ source = DrafterSource("mtp", "/d", False)
+ mlx_inference.mlx_drafter_fit("/m", 8, None, source, load_in_4bit = False, kv_bits = 8)
+ mlx_inference.mlx_prompt_cache_room("/m", 8, source, load_in_4bit = False, kv_bits = 8)
+ assert asked == [8, 8]
+ monkeypatch.setattr(mlx_inference, "mlx_kv_quant_is_refused", lambda _d, _b: True)
+ mlx_inference.mlx_drafter_fit("/m", 8, None, source, load_in_4bit = False, kv_bits = 8)
+ assert asked[-1] is None
+
def test_a_drafter_attaches_where_its_priced_fit_holds_and_a_pin_fits_whole(self, monkeypatch):
import core.inference.mlx_inference as mlx_inference
import core.inference.mlx_memory as mlx_memory
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index ee429bb8542..2484deb6cb3 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -1374,6 +1374,7 @@ def test_an_auto_drafter_gets_offload_room_only_within_the_cap_where_it_then_fit
True,
{
"load_in_4bit": False,
+ "kv_bits": None,
"costless": True,
"baseline": {"offload": (0.5, 0.5), "prefill_chunk": 8192},
"offload": (0.5, 0.53),
@@ -1390,6 +1391,7 @@ def test_an_auto_drafter_gets_offload_room_only_within_the_cap_where_it_then_fit
assert resolve(False) == ((None, 0.78, 8192), None)
assert asked[-1][4] == {
"load_in_4bit": False,
+ "kv_bits": None,
"offload": (None, 1.0),
"prefill_chunk": None,
"reserve": False,
@@ -5376,9 +5378,7 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fi
# ... and is a request, so a load that cannot honour it says why.
failing.append(True)
backend.load_model(config, spec_draft_model = "org/d")
- monkeypatch.setattr(
- mlx_speculative, "speculation_refusal", lambda **_k: mlx_speculative.KV_QUANT
- )
+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: "refused")
reasons = [
entry["spec_fallback_reason"]
for entry in (
@@ -5386,9 +5386,18 @@ def test_an_explicit_speculative_mode_loads_through_mlx_vlm_and_attaches_what_fi
backend.load_model(config, spec_draft_model = "org/d") and backend.models["org/text"],
)
]
- assert reasons == [mlx_speculative.RUNTIME_ERROR, mlx_speculative.KV_QUANT]
- # A diffusion model gets no draft at load, so the resident batch cannot speculate on it either.
+ assert reasons == [mlx_speculative.RUNTIME_ERROR, "refused"]
failing.clear()
+ asked = []
+ monkeypatch.setattr(
+ mlx_speculative, "speculation_refusal", lambda **k: asked.append(k["lora"]) or "refused"
+ )
+ monkeypatch.setattr(mlx_inference, "load_mlx_adapter_processor", lambda p, *_a: p)
+ monkeypatch.setattr(mlx_inference, "load_mlx_adapter_tokenizer", lambda t, *_a: t)
+ for is_vision in (False, True):
+ adapter = SimpleNamespace(**{**vars(config), "is_lora": True, "is_vision": is_vision})
+ backend.load_model(adapter, spec_draft_model = "org/d")
+ assert asked == [True, False]
monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)
monkeypatch.setattr(mlx_inference, "_vlm_generation_is_diffusion", lambda model: True)
backend.load_model(config, max_seq_length = 0, speculative_type = "ngram")
@@ -7590,21 +7599,24 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):
assert seen["draft_model"] is draft and seen["extra"] == 1 and prepared == [([7, 8, 9], 0.5, 0)]
# The draft is told every processor the reply runs: none reaches mlx-vlm as a shortcut.
assert "repetition_penalty" not in seen and draft.given == seen["logits_processors"] != []
+ grammar = SimpleNamespace(allows_reasoning = False, stops_on = lambda ids: None)
+ with monkeypatch.context() as patch:
+ patch.setattr(mlx_inference, "_build_grammar_constraint", lambda *_a, **_k: grammar)
+ seen = _drive_vlm_generation(backend, patch, response_format = {"type": "json_object"})
+ assert seen["draft_model"] is draft and [hasattr(p, "settle") for p in draft.given] == [True]
monkeypatch.setattr(
mlx_inference, "_mlx_sampling_processors", lambda **_: [lambda tokens, logits: logits]
)
timings = backend.last_generation_stats["timings"]
assert (timings["draft_n"], timings["draft_n_accepted"]) == (6, 4)
- for request in (
- {"repetition_penalty": 1.1},
- {"tools": [{"type": "function"}]},
- {"tool_protocol_active": True},
- ):
+ for request in ({"repetition_penalty": 1.1},):
seen = _drive_vlm_generation(backend, monkeypatch, **request)
assert "max_tokens" in seen and "draft_model" not in seen
assert "draft_n" not in backend.last_generation_stats["timings"]
- # Only the drafted reply entered the verify scope: open from preparing the draft through decoding.
- assert len(prepared) == 1 and scopes == [["open"]] * 2 and verify_scope == ["open", "closed"]
+ assert len(prepared) == 2 and verify_scope == ["open", "closed"] * 2
+ assert scopes == [["open"], ["open"], ["open", "closed", "open"]]
+ for request in ({"tools": [{"type": "function"}]}, {"tool_protocol_active": True}):
+ assert _drive_vlm_generation(backend, monkeypatch, **request)["draft_model"] is draft
opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r, **_k: None
backend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)
assert backend.batch_unavailable_reason([{}, {}]) == "no stream"
@@ -8790,9 +8802,9 @@ def test_a_vision_row_times_its_prefill_wherever_the_batch_runs_it(monkeypatch,
sampling = {},
processors = None,
think_prefix = "",
+ videos = None,
stream = _vlm_row(backend),
streamed = None,
- speculates = True,
)
backend._plan_vlm_request = lambda request, **_k: plan
backend._reads_vision = False
@@ -8910,6 +8922,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
sampling = {},
processors = None,
think_prefix = "",
+ videos = None,
stream = _vlm_row(backend),
streamed = None,
)
@@ -9116,20 +9129,24 @@ def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):
assert backend.resident_unavailable_reason({}) == "old mlx-vlm"
[email protected]("gap", (None, "speculative", "old zoo", "no decoder", "no vision"))
+ "gap", (None, "drafted", "drafted and quantized", "old zoo", "no decoder", "no vision")
+)
def test_a_video_reply_batches_only_as_a_fixed_command_of_a_load_that_reads_it(monkeypatch, gap):
from core.inference import mlx_inference
engine, backend = _batch_engine(monkeypatch), _vlm_backend(monkeypatch, markers = None)
- backend._speculative_draft = gap == "speculative" or None
+ backend._speculative_draft = gap in ("drafted", "drafted and quantized") or None
+ backend._kv_quant = {"kv_bits": 8 if gap == "drafted and quantized" else None}
backend._reads_vision, available = gap != "no vision", lambda request: None
backend._vlm_batch_unavailable_reason = backend._vlm_resident_unavailable_reason = available
+ monkeypatch.setattr(mlx_inference, "_speculative_quantized_row_gap", lambda: None)
monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: gap != "no decoder")
fields = SimpleNamespace(__dataclass_fields__ = ("image", "videos")[: 1 + (gap != "old zoo")])
monkeypatch.setattr(engine, "GenerationRequest", fields, raising = False)
clip, sent = {"videos": [_CLIP_B64]}, {"videos_base64": [_CLIP_B64]}
for row in (clip, sent):
- assert (backend.batch_unavailable_reason([{}, row]) is None) == (gap is None)
+ assert (backend.batch_unavailable_reason([{}, row]) is None) == (gap in (None, "drafted"))
assert backend.resident_unavailable_reason(sent) == "a reply carries a video clip"
assert backend.resident_unavailable_reason({}) is None
@@ -9298,6 +9315,100 @@ def test_an_out_of_range_logit_bias_id_cannot_corrupt_a_batch(batch_backend):
assert replies[3] != replies[0]
+DRAFTING_MODEL = "Qwen/Qwen3.5-0.8B"
+CITY_SCHEMA = {
+ "type": "object",
+ "properties": {
+ "city": {"type": "string"},
+ "country": {"type": "string"},
+ "landmarks": {"type": "array", "items": {"type": "string"}},
+ },
+ "required": ["city", "country", "landmarks"],
+ "additionalProperties": False,
+}
+
+
[email protected](scope = "module")
+def drafting_backend(allow_outbound_network):
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ loaded = MLXInferenceBackend()
+ with allow_outbound_network():
+ loaded.load_model(DRAFTING_MODEL, speculative_type = "mtp")
+ assert loaded._speculative_draft is not None and loaded._speculative_draft.drafter is not None
+ return loaded
+
+
+def _drafted_reply(backend, drafted, **request):
+ held = backend._speculative_draft
+ backend._speculative_draft = held if drafted else None
+ # Each reply prefills its prompt: a drafted reply resumed from a snapshot can round a tie differently.
+ backend._release_vlm_snapshots()
+ try:
+ text = ""
+ for text in backend.generate_chat_response(**request):
+ pass
+ finally:
+ backend._speculative_draft = held
+ return text, backend.last_generation_stats["timings"]
+
+
+def _is_city(text):
+ city = json.loads(text)
+ return (
+ set(city) == set(CITY_SCHEMA["properties"])
+ and isinstance(city["city"], str)
+ and isinstance(city["country"], str)
+ and isinstance(city["landmarks"], list)
+ and all(isinstance(landmark, str) for landmark in city["landmarks"])
+ )
+
+
+@metal_only
+def test_a_drafted_constrained_reply_is_the_plain_constrained_reply(drafting_backend):
+ from unsloth_zoo.mlx import speculative
+
+ if not hasattr(speculative._Row, "settle"):
+ pytest.skip("this unsloth-zoo decodes a constrained reply without drafting")
+ request = dict(
+ messages = [
+ {"role": "user", "content": "Describe Paris as JSON with three famous landmarks."}
+ ],
+ response_format = {"type": "json_schema", "schema": CITY_SCHEMA},
+ enable_thinking = False,
+ max_new_tokens = 96,
+ )
+ greedy = dict(request, temperature = 0.0, top_p = 1.0, top_k = 0)
+ drafted, timings = _drafted_reply(drafting_backend, True, **greedy)
+ plain, plain_timings = _drafted_reply(drafting_backend, False, **greedy)
+ assert timings["draft_n_accepted"] > 0
+ assert drafted == plain and timings["predicted_n"] == plain_timings["predicted_n"]
+ assert _is_city(drafted)
+ for seed in (1, 2):
+ sampled, timings = _drafted_reply(
+ drafting_backend, True, **request, temperature = 0.8, top_p = 0.95, top_k = 0, seed = seed
+ )
+ assert timings["draft_n_accepted"] > 0 and _is_city(sampled)
+
+
+def test_a_row_keeps_a_settling_failure_rather_than_raising_it_through_the_batch():
+ from core.inference import mlx_inference
+
+ def processor(tokens, logits):
+ return logits
+
+ def settle(tokens):
+ raise ValueError("refused")
+
+ processor.settle = settle
+ row = SimpleNamespace(failure = None)
+ (guarded,) = mlx_inference._failing_alone(row, [processor])
+ guarded.settle([1])
+ assert str(row.failure) == "refused"
+ plain = mlx_inference._failing_alone(row, [lambda tokens, logits: logits])[0]
+ assert not hasattr(plain, "settle")
+
+
def test_a_repeated_token_is_charged_the_penalty_it_earned():
"""The frequency processor counts occurrences in float32 and scales once."""
mx = pytest.importorskip("mlx.core")
@@ -9676,6 +9787,83 @@ def test_advance_before_the_first_mask_raises_rather_than_dropping_the_token(tin
constraint.advance(tiny_tokenizer.encode("{", add_special_tokens = False)[0])
+def _grammar_after_prompt(
+ tokenizer,
+ response_format = JSON_SCHEMA_FORMAT,
+ prompt = (0, 0),
+ **build,
+):
+ mx = _metal_mx()
+ constraint = build_constraint(response_format, tokenizer, build.pop("text", "p"), **build)
+ processor = gc.make_grammar_logits_processor(constraint)
+ width = len(gc._unwrap_hf_tokenizer(tokenizer))
+ processor(mx.array(prompt), mx.zeros((width,), dtype = mx.float32))
+ return processor
+
+
+def _mask_at(processor, history, width):
+ mx = _metal_mx()
+ return np.asarray(processor(mx.array(history), mx.zeros((width,), dtype = mx.float32)))
+
+
[email protected]("in_reasoning", [False, True])
+def test_a_settled_grammar_masks_each_drafted_position_as_a_plain_decode_would(
+ tiny_tokenizer, in_reasoning
+):
+ mx, width, prompt = _metal_mx(), len(tiny_tokenizer), [0, 0]
+ encode = lambda text: tiny_tokenizer.encode(text, add_special_tokens = False)
+ eos = tiny_tokenizer.eos_token_id
+ build = dict(text = "chat\n<think>\n", reasoning_is_extracted = True) if in_reasoning else {}
+ grammar = lambda: _grammar_after_prompt(tiny_tokenizer, **build)
+
+ def plain(generated):
+ return _mask_at(grammar(), prompt + generated, width)
+
+ drafting = grammar()
+ committed = encode("ok" if in_reasoning else "{")
+ drafting.settle(mx.array(prompt + committed))
+ for drafted in (encode('"a":12}'), encode('"a":3'), encode('"a":45}')):
+ for j in range(len(drafted) + 1):
+ np.testing.assert_array_equal(
+ _mask_at(drafting, prompt + committed + drafted[:j], width),
+ plain(committed + drafted[:j]),
+ )
+ refused = [encode('"a":1}') + [eos] + encode("1")] + ([] if in_reasoning else [encode('"a"]1')])
+ for drafted in refused:
+ assert not np.isneginf(_mask_at(drafting, prompt + committed + drafted, width)).any()
+ committed += encode('"a":7')
+ drafting.settle(mx.array(prompt + committed))
+ np.testing.assert_array_equal(_mask_at(drafting, prompt + committed, width), plain(committed))
+ if not in_reasoning:
+ # Rows take tokens past an end token the decode loop stops on, which a plain advance ignores.
+ drafting.settle(mx.array(prompt + committed + encode("}") + [eos] + encode("1")))
+
+
+def test_a_drafted_state_the_schema_cannot_spell_fails_only_once_accepted():
+ pytest.importorskip("mlx_lm")
+ from mlx_lm.tokenizer_utils import TokenizerWrapper
+
+ mx = _metal_mx()
+ tokenizer, runtime_stop = _two_stop_tokenizer()
+ wrapped, width = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop]), len(tokenizer)
+ control = tokenizer.convert_ids_to_tokens(runtime_stop)
+ either = {"type": "json_schema", "schema": {"anyOf": [{"const": control}, {"type": "integer"}]}}
+ quote, digit = (tokenizer.encode(text, add_special_tokens = False)[0] for text in '"4')
+ for accepted in ([digit], [quote], [quote, digit]):
+ processor = _grammar_after_prompt(wrapped, either, prompt = [0])
+ processor.settle(mx.array([0]))
+ assert not np.isneginf(_mask_at(processor, [0, quote], width)).any()
+ if accepted[0] == quote:
+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):
+ processor.settle(mx.array([0, *accepted]))
+ else:
+ processor.settle(mx.array([0, digit]))
+ assert set(np.nonzero(~np.isneginf(_mask_at(processor, [0, digit], width)))[0]) >= {
+ digit,
+ runtime_stop,
+ }
+
+
def _uncopyable_naive_detokenizer(detokenizers):
"""The pre-0.6.0 naive detokenizer, whose copy raises; pinned by behaviour, not the installed wheel."""
@@ -10842,6 +11030,102 @@ def _video_vlm_backend(monkeypatch, streams):
return backend
+def test_a_quantized_speculative_load_batches_unless_its_caches_are_turboquant(monkeypatch):
+ from core.inference import mlx_inference
+
+ gaps = {"row": None, "drafted": "no drafted rows"}
+ monkeypatch.setattr(mlx_inference, "_row_quantized_cache_gap", lambda: gaps["row"])
+ monkeypatch.setattr(mlx_inference, "_speculative_quantized_row_gap", lambda: gaps["drafted"])
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._is_vlm, backend._kv_quant = True, {"kv_bits": 4}
+ assert backend._kv_policy_batch_reason(resident = True) is None
+ backend._speculative_draft = object()
+ assert backend._kv_policy_batch_reason(resident = True) == "no drafted rows"
+ gaps["drafted"] = None
+ assert backend._kv_policy_batch_reason(resident = True) is None
+ assert "does not carry" in backend._kv_policy_batch_reason()
+ backend._turboquant = True
+ assert "TurboQuant" in backend._kv_policy_batch_reason(resident = True)
+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": None}, 1
+ gaps["row"] = "no rows"
+ assert backend._kv_policy_batch_reason(resident = True) == "no rows"
+
+
+def test_a_drafted_batch_row_decodes_its_own_clip_and_resumes_no_cache(monkeypatch):
+ from contextlib import ExitStack
+
+ from core.inference import mlx_inference
+
+ engine = _batch_engine(monkeypatch)
+ monkeypatch.setattr(engine, "GenerationRequest", lambda **k: k, raising = False)
+ monkeypatch.setattr(engine, "SamplingParams", lambda **k: k, raising = False)
+ monkeypatch.setattr(engine, "BatchRowRefused", ValueError, raising = False)
+ clips = {"videos": ["clip.mp4"], "fps": 2.0}
+ monkeypatch.setattr(mlx_inference, "_batch_clips", lambda stack, plans, _p: [clips])
+ added, released = [], []
+ plan = SimpleNamespace(
+ videos = ("a",),
+ images = None,
+ prompt = "p",
+ max_tokens = 4,
+ sampling = {},
+ processors = None,
+ think_prefix = "",
+ )
+ session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)
+ session.backend = SimpleNamespace(
+ _plan_vlm_request = lambda *_a, **_k: plan,
+ _vlm_batch_row_cache = lambda *_a, **_k: pytest.fail("a clip's row resumes no cache"),
+ _kv_quant_bits = lambda: None,
+ _release_vlm_snapshots = lambda: released.append(True),
+ _vlm_batch_prompt = lambda prompt: {"prompt": prompt},
+ _processor = None,
+ _model = None,
+ )
+ session.stream = SimpleNamespace(add = lambda request: added.append(request) or 0)
+ session._adapter_state, session._decodes_ids, session._resumes_rows = None, False, True
+ session._retires_rows, session._speculative, session._held = False, object(), ExitStack()
+ session._rows, session._by_row = {}, {}
+ session.admit({}, 0)
+ assert added[0]["videos"] == ["clip.mp4"] and added[0]["fps"] == 2.0
+ assert "prompt_cache_state" not in added[0] and released == [True]
+
+
+def test_a_speculative_load_drafts_a_video_reply_on_its_expanded_prompt(monkeypatch):
+ from core.inference import mlx_inference
+
+ streams, prepared, drafted = [], [], []
+ backend = _video_vlm_backend(monkeypatch, streams)
+ backend._vlm_is_diffusion_model = lambda _model: False
+ monkeypatch.setattr(mlx_inference, "_video_sampling_fps", lambda *_a: 2.0)
+
+ class _Ids(list):
+ def tolist(self):
+ return list(self)
+
+ utils = types.ModuleType("mlx_vlm.utils")
+ utils.prepare_inputs = lambda *_a, **kwargs: prepared.append(kwargs) or {
+ "input_ids": [_Ids([7, 7, 8])],
+ "pixel_values_videos": "frames",
+ }
+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)
+ backend._speculative_draft = SimpleNamespace(
+ draft_kind = "mtp",
+ draft_n = 0,
+ draft_n_accepted = 0,
+ prepare = lambda ids, *_a: drafted.append(ids),
+ )
+ turn = [
+ {"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "what moves"}]}
+ ]
+ args = (turn, None, 0, 1, 0, 0, 1, 1, None)
+ assert list(backend._generate_vlm(*args, _adapter_state = False, videos = [_CLIP_B64])) == ["ok"]
+ ((_, stream_kwargs, _),) = streams
+ assert drafted == [[7, 7, 8]] and stream_kwargs["draft_model"] is backend._speculative_draft
+ assert stream_kwargs["pixel_values_videos"] == "frames" and prepared[0]["fps"] == 2.0
+ assert len(prepared[0]["videos"]) == 1
+
+
def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch):
"""A snapshot spans the video rows but is keyed by token ids the clip does not vary, so a
later clip would resume the earlier one's vision rows and its mRoPE grid."""
diff --git a/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts
index c8ecfb4dee7..881c889f9d9 100644
--- a/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts
+++ b/studio/frontend/src/features/chat/lib/mlx-spec-fallback.ts
@@ -30,8 +30,6 @@ export function mlxSpecFallbackMessage(
return `The drafter could not be confirmed to fit in memory beside this model, ${outcome}. When memory is the limit, a shorter context length or a smaller drafter can help.`;
case "runtime_error":
return `Speculative decoding could not start for this load, ${outcome}.`;
- case "kv_quant":
- return "Speculative decoding cannot run with KV cache quantization, which this load asked for. Setting KV Cache Dtype to Auto in this model's settings removes that obstacle.";
case "auto_context_cost":
return `Auto left out a drafter that does not fit beside this model at its full context length, ${outcome}. To try it anyway, choose its mode in this model's settings, with a shorter or automatic context length.`;
default:
diff --git a/studio/frontend/tests/mlx-spec-fallback-copy.test.ts b/studio/frontend/tests/mlx-spec-fallback-copy.test.ts
index 1a941c0a1c4..d8cc7e48a50 100644
--- a/studio/frontend/tests/mlx-spec-fallback-copy.test.ts
+++ b/studio/frontend/tests/mlx-spec-fallback-copy.test.ts
@@ -16,7 +16,7 @@ test("an MLX fallback notice says what runs in the passed-over drafter's place",
test("an MLX fallback notice claims no more than every path behind its code", () => {
// No code guarantees its remedy works, and drafter_no_memory also covers an unpriced fit.
- for (const code of ["drafter_not_found", "drafter_incompatible", "drafter_no_memory", "runtime_error", "kv_quant", "auto_context_cost"]) {
+ for (const code of ["drafter_not_found", "drafter_incompatible", "drafter_no_memory", "runtime_error", "auto_context_cost"]) {
assert.doesNotMatch(mlxSpecFallbackMessage(code, null) ?? "missing", /to use it|missing/);
}
assert.doesNotMatch(mlxSpecFallbackMessage("drafter_no_memory", null) ?? "", /does not fit/);