Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

feat(studio): batch KV-quantized and TurboQuant MLX loads with prompt snapshot reuse (#12343)

Long Yixing89641cc65fd6
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 486559598f6..08909de2780 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -2499,6 +2499,7 @@ def _fitted_context(
retains_history: bool,
kv_bits = None,
is_vlm = False,
+ eligibility = None,
):
"""The fit for a resident model and the eligibility verdict it was priced under."""
try:
@@ -2506,7 +2507,9 @@ def _fitted_context(
except Exception as exc:
logger.debug("MLX snapshot unavailable for %s: %s", model_name, exc)
return None, None
- verdict = None if kv_bits is None else _kv_quant_eligibility(model, is_vlm, kv_bits)
+ verdict = eligibility
+ if verdict is None and kv_bits is not None:
+ verdict = _kv_quant_eligibility(model, is_vlm, kv_bits)
fitted = mlx_fit_to_memory(
model_dir,
ceiling,
@@ -3689,6 +3692,14 @@ def _row_prompt_cache_gap():
return row_prompt_cache_unavailable_reason()
+def _row_quantized_cache_gap():
+ try:
+ from unsloth_zoo.mlx.generate import row_quantized_prompt_cache_unavailable_reason
+ except ImportError:
+ return "the installed unsloth-zoo cannot batch a quantized KV cache"
+ return row_quantized_prompt_cache_unavailable_reason()
+
+
class _VisionBatchSession:
def __init__(
self,
@@ -3719,6 +3730,8 @@ class _VisionBatchSession:
defaults = GenerationDefaults(
prefill_batch_size = 1,
completion_batch_size = width,
+ # Rows quantize their own caches where the single path's decode does.
+ **backend._kv_runtime_quant_kwargs(),
),
)
except BaseException:
@@ -3754,13 +3767,16 @@ class _VisionBatchSession:
if self._resumes_rows
else None
)
+ if state is None and backend._kv_quant_bits() is not None:
+ # mlx-vlm's own batch cache would be float.
+ raise RowRefused("this row cannot carry the load's quantized KV cache")
if state is None and plan.images:
backend._release_vlm_snapshots()
resume = {} if state is None else {"prompt_cache_state": state}
try:
row_number = self.stream.add(
GenerationRequest(
- prompt = plan.prompt,
+ prompt = backend._vlm_batch_prompt(plan.prompt),
image = plan.images[0] if plan.images else None,
max_tokens = plan.max_tokens,
sampling = SamplingParams(**plan.sampling),
@@ -4047,10 +4063,12 @@ class MLXInferenceBackend:
adapter_state,
images = None,
):
- """A batched row's ``prompt_cache_state``, or None while the store is off. Once it is on,
- every row gets one, so all rows' caches share the factory's layout."""
+ """A batched row's ``prompt_cache_state``, or None while the store is off on an unquantized
+ load. Otherwise every row gets one, so all rows' caches share the factory's layout."""
store = self._vlm_prompt_cache_store()
- if store is None or self._vlm_is_diffusion_model(self._model):
+ if (store is None and self._kv_quant_bits() is None) or _vlm_generation_is_diffusion(
+ self._model
+ ):
return None
media_ids = self._vlm_media_token_ids(getattr(self._model, "config", None))
try:
@@ -4058,7 +4076,9 @@ class MLXInferenceBackend:
except Exception as exc:
logger.info("MLX VLM prompt cache unavailable for a batched row (%s)", exc)
return None
- if images and (not media_ids or self._vlm_bidirectional_vision(self._model)):
+ if store is None:
+ pass
+ elif images and (not media_ids or self._vlm_bidirectional_vision(self._model)):
# Unplaceable, or prefilled by the single path as a media block: nothing serves it.
store.clear()
store = None
@@ -4445,7 +4465,17 @@ class MLXInferenceBackend:
if self._turboquant
else ""
)
- use_vlm = is_vision or (self._turboquant and not self._turboquant_refusal)
+ # mlx-lm cannot batch a quantized cache; mlx-vlm, which also ships text architectures, can.
+ batches_quantized_text = (
+ kv_bits is not None
+ and not (is_distributed or is_lora)
+ and _row_quantized_cache_gap() is None
+ )
+ use_vlm = (
+ is_vision
+ or (self._turboquant and not self._turboquant_refusal)
+ or batches_quantized_text
+ )
self._distributed_group = distributed_group
self._distributed_rank = distributed_rank
self._distributed_world_size = distributed_size
@@ -4525,10 +4555,22 @@ class MLXInferenceBackend:
if is_vision or not use_vlm or is_metal_queue_dead(exc):
raise
logger.warning(
- "TurboQuant load of %s through mlx-vlm failed (%s); serving through mlx-lm without it",
+ "Load of %s through mlx-vlm for its KV quantization failed (%s); serving through mlx-lm",
model_name,
exc,
)
+ _eligibility = None
+ if model is not None and batches_quantized_text and not (is_vision or self._turboquant):
+ _eligibility = _kv_quant_eligibility(model, True, _normalize_mlx_kv_bits(kv_bits))
+ verdict, reason, _retainable = _eligibility
+ if verdict not in ("full", "partial"):
+ # Unquantized, the load gains nothing from mlx-vlm's batch.
+ logger.info(
+ "MLX KV quantization not applied to %s (%s); serving through mlx-lm",
+ model_name,
+ reason,
+ )
+ model = tokenizer_or_processor = _eligibility = None
if model is None:
import gc
@@ -4536,7 +4578,8 @@ class MLXInferenceBackend:
gc.collect()
_drain_generation_streams(mx)
mx.clear_cache()
- self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD
+ if self._turboquant:
+ self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD
use_vlm = False
load_kwargs["text_only"] = True
model, tokenizer_or_processor = FastMLXModel.from_pretrained(
@@ -4572,7 +4615,7 @@ class MLXInferenceBackend:
# fit either at full width, which only ever under-promises.
_requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)
_fitted_ctx, _eligibility = (
- (None, None)
+ (None, _eligibility)
if not _priceable or _positive_int(max_seq_length) is not None
else _fitted_context(
self._model,
@@ -4582,6 +4625,7 @@ class MLXInferenceBackend:
retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),
kv_bits = _requested_bits,
is_vlm = use_vlm,
+ eligibility = _eligibility,
)
)
if _fitted_ctx:
@@ -5723,10 +5767,12 @@ class MLXInferenceBackend:
sequences = _mlx_stop_sequences(stop),
normalizer = normalizer,
)
- if max_new_tokens is None:
- max_new_tokens = self._unset_generation_budget(prompt)
- if images:
- max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)
+ max_new_tokens = self._generation_limit(
+ prompt,
+ max_new_tokens,
+ images = images,
+ cap = UNSET_GENERATION_BUDGET if images else None,
+ )
vlm_kwargs = dict(
max_tokens = max_new_tokens,
temperature = temperature,
@@ -6108,10 +6154,13 @@ class MLXInferenceBackend:
if stopped:
self._mark_stopped()
- def _kv_policy_batch_reason(self):
- if self._kv_quant_bits() is not None or getattr(self, "_kv_context_budget", None):
- return "the load quantizes or budgets its KV cache, which a batch does not carry"
- return None
+ def _kv_policy_batch_reason(self, resident = False):
+ if self._kv_quant_bits() is None and not getattr(self, "_kv_context_budget", None):
+ return None
+ # Only resident vision rows carry their own cache, where the quantization lives.
+ if resident and self._is_vlm:
+ return _row_quantized_cache_gap()
+ return "the load quantizes or budgets its KV cache, which a batch does not carry"
def batch_unavailable_reason(self, requests):
if self._model is None:
@@ -6140,7 +6189,7 @@ class MLXInferenceBackend:
reason = _request_batch_gap(request)
if reason is not None:
return reason
- reason = self._kv_policy_batch_reason()
+ reason = self._kv_policy_batch_reason(resident = True)
if reason is not None:
return reason
if self._is_vlm:
@@ -6256,6 +6305,20 @@ class MLXInferenceBackend:
finally:
session.close()
+ def _vlm_batch_prompt(self, prompt):
+ from unsloth_zoo.mlx.generate import vlm_batch_adds_special_tokens
+
+ bos = getattr(self._tokenizer, "bos_token", None)
+ if (
+ not self._reads_vision_input()
+ and bos
+ and prompt.startswith(bos)
+ and vlm_batch_adds_special_tokens(self._model, self._processor)
+ ):
+ # The single path encodes a text load's prompt itself, keeping only the template's BOS.
+ return prompt[len(bos) :]
+ return prompt
+
def _generate_vlm_batch(
self,
requests,
@@ -6275,7 +6338,7 @@ class MLXInferenceBackend:
plans = [self._plan_vlm_request(request) for request in requests]
batch = [
GenerationRequest(
- prompt = plan.prompt,
+ prompt = self._vlm_batch_prompt(plan.prompt),
image = plan.images[0] if plan.images else None,
max_tokens = plan.max_tokens,
sampling = SamplingParams(**plan.sampling),
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 5fa4f779f8f..08cbc509677 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -8,6 +8,7 @@ import copy
import json
import subprocess
import sys
+import threading
import types
from collections import Counter
from contextlib import contextmanager
@@ -3175,6 +3176,68 @@ def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypat
assert attempts == [(False, None)]
+def test_a_uniformly_quantized_text_load_goes_through_mlx_vlm_only_when_that_batches(monkeypatch):
+ pytest.importorskip("mlx.core")
+ from core.inference import mlx_inference
+
+ engine = _batch_engine(monkeypatch)
+ attempts, broken, probes = [], [], []
+
+ class _Loader:
+ @staticmethod
+ def from_pretrained(name, **kwargs):
+ attempts.append(kwargs["text_only"])
+ if broken and not kwargs["text_only"]:
+ raise ValueError("Model type llama4_text not supported.")
+ return SimpleNamespace(config = {}), SimpleNamespace()
+
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader.FastMLXModel = _Loader
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)
+ monkeypatch.setattr(
+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (2048, 2048, 2048)
+ )
+ text = SimpleNamespace(identifier = "org/text", is_vision = False, is_lora = False)
+
+ def routes(
+ kv_quant,
+ gap = None,
+ is_lora = False,
+ verdict = "full",
+ max_seq_length = 2048,
+ ):
+ attempts.clear()
+ probes.clear()
+ text.is_lora = is_lora
+ monkeypatch.setattr(
+ engine, "row_quantized_prompt_cache_unavailable_reason", lambda: gap, raising = False
+ )
+ monkeypatch.setattr(
+ mlx_inference,
+ "_kv_quant_eligibility",
+ lambda *a, **k: probes.append(a) or (verdict, "no cache", True),
+ )
+ backend = mlx_inference.MLXInferenceBackend()
+ backend.load_model(text, kv_quant = kv_quant, max_seq_length = max_seq_length)
+ return list(attempts), backend._turboquant_refusal
+
+ # The verdict that picked the route is the one the load's policy and fit use.
+ for max_seq_length in (2048, 0):
+ assert routes("4", max_seq_length = max_seq_length) == ([False], "")
+ assert len(probes) == 1
+ assert routes("4", gap = "old mlx-vlm") == ([True], "")
+ assert routes("auto") == ([True], "")
+ assert routes("4", is_lora = True) == ([True], "")
+ # Unquantized, it would only trade mlx-lm for mlx-vlm without batching.
+ assert routes("4", verdict = "partial") == ([False], "")
+ assert routes("4", verdict = "none") == ([False, True], "")
+ assert routes("tq-4", verdict = "none") == ([False], "")
+ # An architecture mlx-vlm lacks keeps the mlx-lm load, with no TurboQuant notice.
+ broken.append(True)
+ assert routes("4") == ([False, True], "")
+
+
def test_reload_comparison_and_response_carry_the_resolved_setting():
"""A load-time knob must force a reload and reach the client.
@@ -5604,6 +5667,8 @@ def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatch
monkeypatch.setattr(
mlx_inference, "_kv_quant_eligibility", lambda *a: verdicts.append(a) or FULL
)
+ # Text loads stay on mlx-lm here; routing through mlx-vlm has its own test.
+ monkeypatch.setattr(mlx_inference, "_row_quantized_cache_gap", lambda: "unavailable")
def load(
fitted = (24_576, FULL),
@@ -5641,7 +5706,17 @@ def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatch
backend, info = load(max_seq_length = 0, kv_quant = "4")
assert (verdicts, probes) == ([], [24_576])
assert asked == [
- ("fake/text", None, dict(load_in_4bit = True, retains_history = True, kv_bits = 4, is_vlm = False))
+ (
+ "fake/text",
+ None,
+ dict(
+ load_in_4bit = True,
+ retains_history = True,
+ kv_bits = 4,
+ is_vlm = False,
+ eligibility = None,
+ ),
+ )
]
assert (info["context_length"], info["context_length_fitted"]) == (24_576, 24_576)
assert (info["mlx_kv_bits"], info["mlx_context_budget"]) == (4, 24_576)
@@ -5690,6 +5765,7 @@ def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatch
retains_history = reserved,
kv_bits = None,
is_vlm = True,
+ eligibility = None,
),
)
]
@@ -5884,6 +5960,50 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto
), "a row the engine has already retired is not handed back again"
+def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatch):
+ engine = _batch_engine(monkeypatch)
+ monkeypatch.setattr(engine, "GenerationRequest", SimpleNamespace, raising = False)
+ monkeypatch.setattr(engine, "SamplingParams", dict, raising = False)
+ monkeypatch.setattr(engine, "BatchRowRefused", RuntimeError, raising = False)
+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: True, raising = False)
+ backend = _vlm_backend(monkeypatch, markers = None)
+ backend._tokenizer.bos_token = "<s>"
+ backend._plan_vlm_request = lambda request: SimpleNamespace(
+ prompt = "<s>hi",
+ images = None,
+ max_tokens = 4,
+ sampling = {},
+ processors = None,
+ think_prefix = "",
+ stream = None,
+ )
+ session = _open_vision_session([])
+ session.backend, session._adapter_state, session._resumes_rows = backend, None, False
+ added = []
+ session.stream = SimpleNamespace(
+ add = lambda request: added.append(request.prompt) or len(added) - 1
+ )
+
+ def stream_batch(model, processor, batch, defaults):
+ added.extend(request.prompt for request in batch)
+ raise LookupError
+
+ monkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)
+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)
+ backend._reads_vision, backend._generation_lock = False, threading.Lock()
+ with pytest.raises(LookupError):
+ next(backend._generate_vlm_batch([{}]))
+ for reads_vision in (False, True):
+ backend._reads_vision = reads_vision
+ session.admit({}, reads_vision)
+ # Where the batch adds no special tokens the template's BOS is the only one.
+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: False)
+ backend._reads_vision = False
+ session.admit({}, "gemma")
+ # A vision load's single path lets mlx-vlm tokenize, so its rows do too.
+ assert added == ["hi", "hi", "<s>hi", "<s>hi"]
+
+
def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(monkeypatch):
from core.inference import mlx_inference
@@ -5892,6 +6012,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
monkeypatch.setattr(engine, "SamplingParams", dict, raising = False)
monkeypatch.setattr(engine, "BatchRowRefused", RuntimeError, raising = False)
monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: None, raising = False)
+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: True, raising = False)
backend = _vlm_backend(monkeypatch, markers = None)
store = mlx_inference.VLMPromptSnapshotStore(max_bytes = 10**9)
for key in ("m", "m|img"):
@@ -5947,6 +6068,15 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
session.admit({"images": [object()]}, "old")
assert not hasattr(added[3], "prompt_cache_state") and len(store) == 0
+ # A quantized load puts every row on the factory's cache, the store off or not.
+ backend._kv_quant = {"kv_bits": 4}
+ with pytest.raises(mlx_inference.RowRefused):
+ session.admit({}, "unquantized")
+ session._resumes_rows = True
+ backend._vlm_prompt_cache_store = lambda: None
+ session.admit({"images": [object()]}, "quantized")
+ assert added[4].prompt_cache_state._store is None
+
def _batch_engine(monkeypatch):
"""The zoo batch engine, or a stand-in where the backend CI installs no unsloth-zoo."""
@@ -6023,6 +6153,74 @@ def test_a_quantized_or_budgeted_kv_cache_does_not_batch(monkeypatch):
assert backend.batch_unavailable_reason([{}, {}]) is not None
+def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ engine = _batch_engine(monkeypatch)
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ backend._model, backend._processor, backend._is_vlm = object(), object(), True
+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": 4}, 4096
+ backend._turboquant = False
+ backend._vlm_batch_unavailable_reason = lambda requests: None
+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)
+ capable = lambda: None
+ monkeypatch.setattr(
+ engine, "row_quantized_prompt_cache_unavailable_reason", capable, raising = False
+ )
+ assert backend.resident_unavailable_reason({}) is None
+ assert backend.batch_unavailable_reason([{}, {}]) is not None
+ backend._turboquant = True
+ assert backend.resident_unavailable_reason({}) is None
+ assert backend.batch_unavailable_reason([{}, {}]) is not None
+ monkeypatch.setattr(
+ engine, "row_quantized_prompt_cache_unavailable_reason", lambda: "old mlx-vlm"
+ )
+ assert backend.resident_unavailable_reason({}) == "old mlx-vlm"
+
+
+def test_a_turboquant_vision_batch_quantizes_rows_as_the_single_path_does(monkeypatch):
+ from core.inference import mlx_inference
+
+ engine = _batch_engine(monkeypatch)
+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)
+ monkeypatch.setattr(
+ engine, "BatchStream", lambda model, processor, defaults: defaults, raising = False
+ )
+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)
+ backend._model = backend._processor = object()
+ backend._kv_quant, backend._turboquant = {"kv_bits": 3.5}, True
+ defaults = mlx_inference._VisionBatchSession(backend, width = 2).stream
+ assert (defaults.kv_bits, defaults.kv_quant_scheme, defaults.quantized_kv_start) == (
+ 3.5,
+ "turboquant",
+ 0,
+ )
+ backend._kv_quant, backend._turboquant = {"kv_bits": 4}, False
+ assert not hasattr(mlx_inference._VisionBatchSession(backend, width = 2).stream, "kv_bits")
+
+
+def test_a_batched_vision_row_keeps_to_the_context_budget(monkeypatch):
+ from core.inference import context_refusal
+
+ backend = _vlm_backend(monkeypatch, markers = None)
+ backend._kv_context_budget = 1024
+ plan = lambda: backend._plan_vlm_row(
+ [{"role": "user", "content": "hi"}],
+ None,
+ temperature = 0.7,
+ top_p = 0.9,
+ top_k = 0,
+ min_p = 0.0,
+ max_new_tokens = 500,
+ repetition_penalty = 1.0,
+ )
+ backend._count_prompt_tokens = lambda *a, **k: 1000
+ assert plan().max_tokens == 24
+ backend._count_prompt_tokens = lambda *a, **k: 1024
+ with pytest.raises(context_refusal.ContextBudgetExceeded):
+ plan()
+
+
try:
import mlx.core as _mx
_METAL = _mx.metal.is_available()