Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
feat(studio): batch KV-quantized and TurboQuant MLX loads with prompt snapshot reuse (#12343)
Long Yixing89641cc65fd6
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 486559598f6..08909de2780 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -2499,6 +2499,7 @@ def _fitted_context(retains_history: bool,kv_bits = None,is_vlm = False,+ eligibility = None,):"""The fit for a resident model and the eligibility verdict it was priced under."""try:@@ -2506,7 +2507,9 @@ def _fitted_context(except Exception as exc:logger.debug("MLX snapshot unavailable for %s: %s", model_name, exc)return None, None- verdict = None if kv_bits is None else _kv_quant_eligibility(model, is_vlm, kv_bits)+ verdict = eligibility+ if verdict is None and kv_bits is not None:+ verdict = _kv_quant_eligibility(model, is_vlm, kv_bits)fitted = mlx_fit_to_memory(model_dir,ceiling,@@ -3689,6 +3692,14 @@ def _row_prompt_cache_gap():return row_prompt_cache_unavailable_reason()+def _row_quantized_cache_gap():+ try:+ from unsloth_zoo.mlx.generate import row_quantized_prompt_cache_unavailable_reason+ except ImportError:+ return "the installed unsloth-zoo cannot batch a quantized KV cache"+ return row_quantized_prompt_cache_unavailable_reason()++class _VisionBatchSession:def __init__(self,@@ -3719,6 +3730,8 @@ class _VisionBatchSession:defaults = GenerationDefaults(prefill_batch_size = 1,completion_batch_size = width,+ # Rows quantize their own caches where the single path's decode does.+ **backend._kv_runtime_quant_kwargs(),),)except BaseException:@@ -3754,13 +3767,16 @@ class _VisionBatchSession:if self._resumes_rowselse None)+ if state is None and backend._kv_quant_bits() is not None:+ # mlx-vlm's own batch cache would be float.+ raise RowRefused("this row cannot carry the load's quantized KV cache")if state is None and plan.images:backend._release_vlm_snapshots()resume = {} if state is None else {"prompt_cache_state": state}try:row_number = self.stream.add(GenerationRequest(- prompt = plan.prompt,+ prompt = backend._vlm_batch_prompt(plan.prompt),image = plan.images[0] if plan.images else None,max_tokens = plan.max_tokens,sampling = SamplingParams(**plan.sampling),@@ -4047,10 +4063,12 @@ class MLXInferenceBackend:adapter_state,images = None,):- """A batched row's ``prompt_cache_state``, or None while the store is off. Once it is on,- every row gets one, so all rows' caches share the factory's layout."""+ """A batched row's ``prompt_cache_state``, or None while the store is off on an unquantized+ load. Otherwise every row gets one, so all rows' caches share the factory's layout."""store = self._vlm_prompt_cache_store()- if store is None or self._vlm_is_diffusion_model(self._model):+ if (store is None and self._kv_quant_bits() is None) or _vlm_generation_is_diffusion(+ self._model+ ):return Nonemedia_ids = self._vlm_media_token_ids(getattr(self._model, "config", None))try:@@ -4058,7 +4076,9 @@ class MLXInferenceBackend:except Exception as exc:logger.info("MLX VLM prompt cache unavailable for a batched row (%s)", exc)return None- if images and (not media_ids or self._vlm_bidirectional_vision(self._model)):+ if store is None:+ pass+ elif images and (not media_ids or self._vlm_bidirectional_vision(self._model)):# Unplaceable, or prefilled by the single path as a media block: nothing serves it.store.clear()store = None@@ -4445,7 +4465,17 @@ class MLXInferenceBackend:if self._turboquantelse "")- use_vlm = is_vision or (self._turboquant and not self._turboquant_refusal)+ # mlx-lm cannot batch a quantized cache; mlx-vlm, which also ships text architectures, can.+ batches_quantized_text = (+ kv_bits is not None+ and not (is_distributed or is_lora)+ and _row_quantized_cache_gap() is None+ )+ use_vlm = (+ is_vision+ or (self._turboquant and not self._turboquant_refusal)+ or batches_quantized_text+ )self._distributed_group = distributed_groupself._distributed_rank = distributed_rankself._distributed_world_size = distributed_size@@ -4525,10 +4555,22 @@ class MLXInferenceBackend:if is_vision or not use_vlm or is_metal_queue_dead(exc):raiselogger.warning(- "TurboQuant load of %s through mlx-vlm failed (%s); serving through mlx-lm without it",+ "Load of %s through mlx-vlm for its KV quantization failed (%s); serving through mlx-lm",model_name,exc,)+ _eligibility = None+ if model is not None and batches_quantized_text and not (is_vision or self._turboquant):+ _eligibility = _kv_quant_eligibility(model, True, _normalize_mlx_kv_bits(kv_bits))+ verdict, reason, _retainable = _eligibility+ if verdict not in ("full", "partial"):+ # Unquantized, the load gains nothing from mlx-vlm's batch.+ logger.info(+ "MLX KV quantization not applied to %s (%s); serving through mlx-lm",+ model_name,+ reason,+ )+ model = tokenizer_or_processor = _eligibility = Noneif model is None:import gc@@ -4536,7 +4578,8 @@ class MLXInferenceBackend:gc.collect()_drain_generation_streams(mx)mx.clear_cache()- self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD+ if self._turboquant:+ self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOADuse_vlm = Falseload_kwargs["text_only"] = Truemodel, tokenizer_or_processor = FastMLXModel.from_pretrained(@@ -4572,7 +4615,7 @@ class MLXInferenceBackend:# fit either at full width, which only ever under-promises._requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)_fitted_ctx, _eligibility = (- (None, None)+ (None, _eligibility)if not _priceable or _positive_int(max_seq_length) is not Noneelse _fitted_context(self._model,@@ -4582,6 +4625,7 @@ class MLXInferenceBackend:retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),kv_bits = _requested_bits,is_vlm = use_vlm,+ eligibility = _eligibility,))if _fitted_ctx:@@ -5723,10 +5767,12 @@ class MLXInferenceBackend:sequences = _mlx_stop_sequences(stop),normalizer = normalizer,)- if max_new_tokens is None:- max_new_tokens = self._unset_generation_budget(prompt)- if images:- max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)+ max_new_tokens = self._generation_limit(+ prompt,+ max_new_tokens,+ images = images,+ cap = UNSET_GENERATION_BUDGET if images else None,+ )vlm_kwargs = dict(max_tokens = max_new_tokens,temperature = temperature,@@ -6108,10 +6154,13 @@ class MLXInferenceBackend:if stopped:self._mark_stopped()- def _kv_policy_batch_reason(self):- if self._kv_quant_bits() is not None or getattr(self, "_kv_context_budget", None):- return "the load quantizes or budgets its KV cache, which a batch does not carry"- return None+ def _kv_policy_batch_reason(self, resident = False):+ if self._kv_quant_bits() is None and not getattr(self, "_kv_context_budget", None):+ return None+ # Only resident vision rows carry their own cache, where the quantization lives.+ if resident and self._is_vlm:+ return _row_quantized_cache_gap()+ return "the load quantizes or budgets its KV cache, which a batch does not carry"def batch_unavailable_reason(self, requests):if self._model is None:@@ -6140,7 +6189,7 @@ class MLXInferenceBackend:reason = _request_batch_gap(request)if reason is not None:return reason- reason = self._kv_policy_batch_reason()+ reason = self._kv_policy_batch_reason(resident = True)if reason is not None:return reasonif self._is_vlm:@@ -6256,6 +6305,20 @@ class MLXInferenceBackend:finally:session.close()+ def _vlm_batch_prompt(self, prompt):+ from unsloth_zoo.mlx.generate import vlm_batch_adds_special_tokens++ bos = getattr(self._tokenizer, "bos_token", None)+ if (+ not self._reads_vision_input()+ and bos+ and prompt.startswith(bos)+ and vlm_batch_adds_special_tokens(self._model, self._processor)+ ):+ # The single path encodes a text load's prompt itself, keeping only the template's BOS.+ return prompt[len(bos) :]+ return prompt+def _generate_vlm_batch(self,requests,@@ -6275,7 +6338,7 @@ class MLXInferenceBackend:plans = [self._plan_vlm_request(request) for request in requests]batch = [GenerationRequest(- prompt = plan.prompt,+ prompt = self._vlm_batch_prompt(plan.prompt),image = plan.images[0] if plan.images else None,max_tokens = plan.max_tokens,sampling = SamplingParams(**plan.sampling),diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 5fa4f779f8f..08cbc509677 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -8,6 +8,7 @@ import copyimport jsonimport subprocessimport sys+import threadingimport typesfrom collections import Counterfrom contextlib import contextmanager@@ -3175,6 +3176,68 @@ def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypatassert attempts == [(False, None)]+def test_a_uniformly_quantized_text_load_goes_through_mlx_vlm_only_when_that_batches(monkeypatch):+ pytest.importorskip("mlx.core")+ from core.inference import mlx_inference++ engine = _batch_engine(monkeypatch)+ attempts, broken, probes = [], [], []++ class _Loader:+ @staticmethod+ def from_pretrained(name, **kwargs):+ attempts.append(kwargs["text_only"])+ if broken and not kwargs["text_only"]:+ raise ValueError("Model type llama4_text not supported.")+ return SimpleNamespace(config = {}), SimpleNamespace()++ loader = types.ModuleType("unsloth_zoo.mlx.loader")+ loader.FastMLXModel = _Loader+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)+ monkeypatch.setattr(+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (2048, 2048, 2048)+ )+ text = SimpleNamespace(identifier = "org/text", is_vision = False, is_lora = False)++ def routes(+ kv_quant,+ gap = None,+ is_lora = False,+ verdict = "full",+ max_seq_length = 2048,+ ):+ attempts.clear()+ probes.clear()+ text.is_lora = is_lora+ monkeypatch.setattr(+ engine, "row_quantized_prompt_cache_unavailable_reason", lambda: gap, raising = False+ )+ monkeypatch.setattr(+ mlx_inference,+ "_kv_quant_eligibility",+ lambda *a, **k: probes.append(a) or (verdict, "no cache", True),+ )+ backend = mlx_inference.MLXInferenceBackend()+ backend.load_model(text, kv_quant = kv_quant, max_seq_length = max_seq_length)+ return list(attempts), backend._turboquant_refusal++ # The verdict that picked the route is the one the load's policy and fit use.+ for max_seq_length in (2048, 0):+ assert routes("4", max_seq_length = max_seq_length) == ([False], "")+ assert len(probes) == 1+ assert routes("4", gap = "old mlx-vlm") == ([True], "")+ assert routes("auto") == ([True], "")+ assert routes("4", is_lora = True) == ([True], "")+ # Unquantized, it would only trade mlx-lm for mlx-vlm without batching.+ assert routes("4", verdict = "partial") == ([False], "")+ assert routes("4", verdict = "none") == ([False, True], "")+ assert routes("tq-4", verdict = "none") == ([False], "")+ # An architecture mlx-vlm lacks keeps the mlx-lm load, with no TurboQuant notice.+ broken.append(True)+ assert routes("4") == ([False, True], "")++def test_reload_comparison_and_response_carry_the_resolved_setting():"""A load-time knob must force a reload and reach the client.@@ -5604,6 +5667,8 @@ def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatchmonkeypatch.setattr(mlx_inference, "_kv_quant_eligibility", lambda *a: verdicts.append(a) or FULL)+ # Text loads stay on mlx-lm here; routing through mlx-vlm has its own test.+ monkeypatch.setattr(mlx_inference, "_row_quantized_cache_gap", lambda: "unavailable")def load(fitted = (24_576, FULL),@@ -5641,7 +5706,17 @@ def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatchbackend, info = load(max_seq_length = 0, kv_quant = "4")assert (verdicts, probes) == ([], [24_576])assert asked == [- ("fake/text", None, dict(load_in_4bit = True, retains_history = True, kv_bits = 4, is_vlm = False))+ (+ "fake/text",+ None,+ dict(+ load_in_4bit = True,+ retains_history = True,+ kv_bits = 4,+ is_vlm = False,+ eligibility = None,+ ),+ )]assert (info["context_length"], info["context_length_fitted"]) == (24_576, 24_576)assert (info["mlx_kv_bits"], info["mlx_context_budget"]) == (4, 24_576)@@ -5690,6 +5765,7 @@ def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatchretains_history = reserved,kv_bits = None,is_vlm = True,+ eligibility = None,),)]@@ -5884,6 +5960,50 @@ def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, sto), "a row the engine has already retired is not handed back again"+def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatch):+ engine = _batch_engine(monkeypatch)+ monkeypatch.setattr(engine, "GenerationRequest", SimpleNamespace, raising = False)+ monkeypatch.setattr(engine, "SamplingParams", dict, raising = False)+ monkeypatch.setattr(engine, "BatchRowRefused", RuntimeError, raising = False)+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: True, raising = False)+ backend = _vlm_backend(monkeypatch, markers = None)+ backend._tokenizer.bos_token = "<s>"+ backend._plan_vlm_request = lambda request: SimpleNamespace(+ prompt = "<s>hi",+ images = None,+ max_tokens = 4,+ sampling = {},+ processors = None,+ think_prefix = "",+ stream = None,+ )+ session = _open_vision_session([])+ session.backend, session._adapter_state, session._resumes_rows = backend, None, False+ added = []+ session.stream = SimpleNamespace(+ add = lambda request: added.append(request.prompt) or len(added) - 1+ )++ def stream_batch(model, processor, batch, defaults):+ added.extend(request.prompt for request in batch)+ raise LookupError++ monkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)+ backend._reads_vision, backend._generation_lock = False, threading.Lock()+ with pytest.raises(LookupError):+ next(backend._generate_vlm_batch([{}]))+ for reads_vision in (False, True):+ backend._reads_vision = reads_vision+ session.admit({}, reads_vision)+ # Where the batch adds no special tokens the template's BOS is the only one.+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: False)+ backend._reads_vision = False+ session.admit({}, "gemma")+ # A vision load's single path lets mlx-vlm tokenize, so its rows do too.+ assert added == ["hi", "hi", "<s>hi", "<s>hi"]++def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(monkeypatch):from core.inference import mlx_inference@@ -5892,6 +6012,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(momonkeypatch.setattr(engine, "SamplingParams", dict, raising = False)monkeypatch.setattr(engine, "BatchRowRefused", RuntimeError, raising = False)monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: None, raising = False)+ monkeypatch.setattr(engine, "vlm_batch_adds_special_tokens", lambda *a: True, raising = False)backend = _vlm_backend(monkeypatch, markers = None)store = mlx_inference.VLMPromptSnapshotStore(max_bytes = 10**9)for key in ("m", "m|img"):@@ -5947,6 +6068,15 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mosession.admit({"images": [object()]}, "old")assert not hasattr(added[3], "prompt_cache_state") and len(store) == 0+ # A quantized load puts every row on the factory's cache, the store off or not.+ backend._kv_quant = {"kv_bits": 4}+ with pytest.raises(mlx_inference.RowRefused):+ session.admit({}, "unquantized")+ session._resumes_rows = True+ backend._vlm_prompt_cache_store = lambda: None+ session.admit({"images": [object()]}, "quantized")+ assert added[4].prompt_cache_state._store is None+def _batch_engine(monkeypatch):"""The zoo batch engine, or a stand-in where the backend CI installs no unsloth-zoo."""@@ -6023,6 +6153,74 @@ def test_a_quantized_or_budgeted_kv_cache_does_not_batch(monkeypatch):assert backend.batch_unavailable_reason([{}, {}]) is not None+def test_a_quantized_vision_load_batches_only_as_resident_rows(monkeypatch):+ from core.inference.mlx_inference import MLXInferenceBackend++ engine = _batch_engine(monkeypatch)+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)+ backend._model, backend._processor, backend._is_vlm = object(), object(), True+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": 4}, 4096+ backend._turboquant = False+ backend._vlm_batch_unavailable_reason = lambda requests: None+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)+ capable = lambda: None+ monkeypatch.setattr(+ engine, "row_quantized_prompt_cache_unavailable_reason", capable, raising = False+ )+ assert backend.resident_unavailable_reason({}) is None+ assert backend.batch_unavailable_reason([{}, {}]) is not None+ backend._turboquant = True+ assert backend.resident_unavailable_reason({}) is None+ assert backend.batch_unavailable_reason([{}, {}]) is not None+ monkeypatch.setattr(+ engine, "row_quantized_prompt_cache_unavailable_reason", lambda: "old mlx-vlm"+ )+ assert backend.resident_unavailable_reason({}) == "old mlx-vlm"+++def test_a_turboquant_vision_batch_quantizes_rows_as_the_single_path_does(monkeypatch):+ from core.inference import mlx_inference++ engine = _batch_engine(monkeypatch)+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)+ monkeypatch.setattr(+ engine, "BatchStream", lambda model, processor, defaults: defaults, raising = False+ )+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)+ backend._model = backend._processor = object()+ backend._kv_quant, backend._turboquant = {"kv_bits": 3.5}, True+ defaults = mlx_inference._VisionBatchSession(backend, width = 2).stream+ assert (defaults.kv_bits, defaults.kv_quant_scheme, defaults.quantized_kv_start) == (+ 3.5,+ "turboquant",+ 0,+ )+ backend._kv_quant, backend._turboquant = {"kv_bits": 4}, False+ assert not hasattr(mlx_inference._VisionBatchSession(backend, width = 2).stream, "kv_bits")+++def test_a_batched_vision_row_keeps_to_the_context_budget(monkeypatch):+ from core.inference import context_refusal++ backend = _vlm_backend(monkeypatch, markers = None)+ backend._kv_context_budget = 1024+ plan = lambda: backend._plan_vlm_row(+ [{"role": "user", "content": "hi"}],+ None,+ temperature = 0.7,+ top_p = 0.9,+ top_k = 0,+ min_p = 0.0,+ max_new_tokens = 500,+ repetition_penalty = 1.0,+ )+ backend._count_prompt_tokens = lambda *a, **k: 1000+ assert plan().max_tokens == 24+ backend._count_prompt_tokens = lambda *a, **k: 1024+ with pytest.raises(context_refusal.ContextBudgetExceeded):+ plan()++try:import mlx.core as _mx_METAL = _mx.metal.is_available()