Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: TurboQuant KV cache option for MLX inference (#11170)
Long Yixingeadcab66e1de
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 851e14b36f1..63b2c828a6b 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -106,6 +106,12 @@ def _is_opaque_cache(value):return not hasattr(value, "__dict__") and hasattr(type(value), "state")+def _rebuilt_tuple(original, items):+ """Rebuild as the tuple's own type: TurboQuant dispatches on its state class."""+ rebuild = getattr(type(original), "_make", None)+ return rebuild(items) if rebuild is not None else tuple(items)++def _copy_value(value, mx):if isinstance(value, mx.array):# Safe to share: MLX writes in place only into an unreferenced buffer; slices are copied out.@@ -113,7 +119,7 @@ def _copy_value(value, mx):if isinstance(value, list):return [_copy_value(item, mx) for item in value]if isinstance(value, tuple):- return tuple(_copy_value(item, mx) for item in value)+ return _rebuilt_tuple(value, (_copy_value(item, mx) for item in value))if isinstance(value, dict):return {key: _copy_value(item, mx) for key, item in value.items()}if _is_cache(value):@@ -146,7 +152,7 @@ def _release_value(value, mx):if isinstance(value, list):value[:] = [_release_value(item, mx) for item in value]elif isinstance(value, tuple):- return tuple(_release_value(item, mx) for item in value)+ return _rebuilt_tuple(value, (_release_value(item, mx) for item in value))elif isinstance(value, dict):for key, item in list(value.items()):value[key] = _release_value(item, mx)@@ -1586,6 +1592,7 @@ PROMPT_CACHE_ENTRIES = 6# Every bit width mx.quantize supports; unrelated to llama.cpp's cache_type_kv names.MLX_KV_BITS_CHOICES = (8, 6, 5, 4, 3, 2)+MLX_TURBOQUANT_BITS_CHOICES = (4, 3.5, 3, 2)# Quantization group size; a head dim that is not a multiple makes mx.quantize raise.MLX_KV_GROUP_SIZE = 64# Surfaced with the resolved setting so an API client sees the reuse cost too.@@ -1593,6 +1600,63 @@ MLX_KV_QUANT_NO_REUSE = ("The installed mlx-lm cannot measure a quantized cache entry, so prompt-cache ""reuse across turns is disabled while this is on.")+MLX_TURBOQUANT_TEXT_LOAD = (+ "TurboQuant is not available for this model: it could not be loaded through the runtime "+ "TurboQuant requires. The model is served without it."+)+++def _turboquant_refusal(*, is_vision, is_distributed, is_lora):+ """Why a text model cannot take the mlx-vlm route TurboQuant lives on, else ""."""+ if is_vision:+ return ""+ if is_distributed:+ return "TurboQuant is not available under distributed inference. Run on one device, or turn it off."+ if is_lora:+ return "TurboQuant is not available for a LoRA adapter on a text model. Merge it first, or turn TurboQuant off."+ return ""+++MLX_KV_QUANT_AUTO = "auto"+# TurboQuant's widths repeat mx.quantize's, so the scheme rides in the value: "4" alone is ambiguous.+MLX_TURBOQUANT_PREFIX = "tq-"+++def _kv_quant_choice(bits):+ return str(int(bits)) if float(bits).is_integer() else str(float(bits))+++MLX_KV_QUANT_CHOICES = (+ (MLX_KV_QUANT_AUTO,)+ + tuple(_kv_quant_choice(bits) for bits in MLX_KV_BITS_CHOICES)+ + tuple(MLX_TURBOQUANT_PREFIX + _kv_quant_choice(bits) for bits in MLX_TURBOQUANT_BITS_CHOICES)+)+++def parse_mlx_kv_quant(value):+ """The (bit width, TurboQuant) pair for a choice. Total: an unknown one reads as auto."""++ if value is None:+ return None, False+ text = str(value).strip().lower()+ if text in ("", MLX_KV_QUANT_AUTO):+ return None, False+ if text not in MLX_KV_QUANT_CHOICES:+ logger.warning(+ "MLX kv quantization %r unsupported (choose %s); ignoring",+ value,+ " or ".join(MLX_KV_QUANT_CHOICES),+ )+ return None, False+ turboquant = text.startswith(MLX_TURBOQUANT_PREFIX)+ width = float(text.removeprefix(MLX_TURBOQUANT_PREFIX))+ return (int(width) if width.is_integer() else width), turboquant+++def encode_mlx_kv_quant(bits, turboquant = False):+ if bits is None:+ return MLX_KV_QUANT_AUTO+ return (MLX_TURBOQUANT_PREFIX if turboquant else "") + _kv_quant_choice(bits)def _kv_entry_nbytes(entry):@@ -2222,6 +2286,97 @@ def _kv_window_enforced(model, is_vlm, window):return None+def _turboquant_reserved_positions(count, bits):+ """Entry positions mlx-vlm reserves, learned from a control cache rather than a fixed number."""++ from mlx_vlm.generate.common import maybe_quantize_kv_cache+ from mlx_vlm.models.cache import KVCache+ from mlx_vlm.turboquant import TurboQuantKVCache++ control = [KVCache() for _ in range(count)]+ maybe_quantize_kv_cache(control, 0, MLX_KV_GROUP_SIZE, bits, kv_quant_scheme = "turboquant")+ return {+ index for index, entry in enumerate(control) if not isinstance(entry, TurboQuantKVCache)+ }+++def _turboquant_status(+ model,+ bits,+ refusal = "",+):+ import mlx.core as mx++ status = dict(requested_kv_bits = bits, kv_bits = None, eligibility = "none", reason = "", note = "")+ if refusal:+ status["eligibility"] = "refused"+ status["reason"] = refusal+ logger.info("MLX TurboQuant not applied: %s", refusal)+ return status++ try:+ from mlx_vlm.generate.common import maybe_quantize_kv_cache+ except ImportError:+ # TurboQuant predates generate.common (mlx-vlm 0.6.0).+ status["eligibility"] = "refused"+ status["reason"] = "TurboQuant requires a newer mlx-vlm. Update the MLX packages and retry."+ return status+ from mlx_vlm.models.cache import make_prompt_cache+ from mlx_vlm.turboquant import TurboQuantKVCache++ language_model = getattr(model, "language_model", model)+ try:+ entries = make_prompt_cache(language_model)+ except Exception as exc:+ if is_metal_queue_dead(exc):+ raise+ logger.warning("MLX TurboQuant eligibility probe failed: %s", exc)+ status["reason"] = "this model's KV cache layout could not be inspected"+ return status+ if not entries:+ status["reason"] = "this model builds no KV cache to quantize"+ return status++ rng_key = _mlx_rng_key_words()+ try:+ reserved = _turboquant_reserved_positions(len(entries), bits)+ maybe_quantize_kv_cache(entries, 0, MLX_KV_GROUP_SIZE, bits, kv_quant_scheme = "turboquant")+ converted = declined = 0+ for index, entry in enumerate(entries):+ leaves = list(_flatten_kv_entries([entry]))+ quantized = sum(isinstance(leaf, TurboQuantKVCache) for leaf in leaves)+ converted += quantized+ if index not in reserved:+ declined += len(leaves) - quantized+ if not converted:+ status["reason"] = (+ "This model's cache keeps its native format; no layers can use TurboQuant."+ )+ return status+ for _ in range(2):+ result = language_model(mx.array([[0]]), cache = entries)+ mx.eval(getattr(result, "logits", result), [entry.state for entry in entries])+ status["kv_bits"] = bits+ status["eligibility"] = "partial" if declined else "full"+ if declined:+ status["note"] = "Some layers keep their native cache format."+ return status+ except Exception as exc:+ if is_metal_queue_dead(exc):+ raise+ logger.warning("MLX TurboQuant eligibility probe failed: %s", exc)+ status["eligibility"] = "refused"+ status["reason"] = f"this model's KV cache cannot use TurboQuant ({type(exc).__name__})"+ return status+ finally:+ _restore_mlx_rng_key(rng_key)+ # After the probe's own locals are gone, or the pages stay in the allocator.+ entries.clear()+ del entries+ _drain_generation_streams(mx)+ mx.clear_cache()++def _kv_quant_status(requested_bits, model, is_vlm):"""Resolve a requested bit width against this model into a status dict."""status = {@@ -3383,6 +3538,8 @@ class MLXInferenceBackend:self._tokenizer = Noneself._processor = Noneself._is_vlm = False+ self._reads_vision = None+ self._turboquant_refusal = ""self._multi_image_marker = Noneself._config = {}self._distributed_group = None@@ -3642,11 +3799,21 @@ class MLXInferenceBackend:def _prepare_kv_entries(self, entries):bits = self._kv_quant_bits()- return entries if bits is None else _quantize_kv_entries(entries, bits)+ # TurboQuant converts in mlx-vlm: mx.quantize cannot express its fractional widths.+ if bits is None or getattr(self, "_turboquant", False):+ return entries+ return _quantize_kv_entries(entries, bits)++ def _kv_runtime_quant_kwargs(self):+ """TurboQuant's width for mlx-vlm's converter; start 0, since its default leaves 5000 tokens unquantized."""+ bits = self._kv_quant_bits()+ if bits is None or not getattr(self, "_turboquant", False):+ return {}+ return {"kv_bits": bits, "kv_quant_scheme": "turboquant", "quantized_kv_start": 0}def _kv_quant_generate_kwargs(self):"""A pre-quantized cache, not kv_bits: both runtimes would convert a rotating entry."""- if self._kv_quant_bits() is None:+ if self._kv_quant_bits() is None or getattr(self, "_turboquant", False):return {}return {"prompt_cache": self._prepare_kv_entries(_make_kv_entries(self._model, self._is_vlm))@@ -3696,6 +3863,13 @@ class MLXInferenceBackend:))+ def _reads_vision_input(self):+ """Whether this model reads images, which mlx-vlm serving it no longer implies."""+ reads = getattr(self, "_reads_vision", None)+ if reads is None:+ return bool(getattr(self, "_is_vlm", False))+ return bool(reads)+def _count_media_prompt_tokens(self,prompt,@@ -3745,7 +3919,7 @@ class MLXInferenceBackend:videos = None,):"""Prompt length as generation tokenizes it; vision models follow mlx_vlm's marker rule."""- if not self._is_vlm:+ if not self._reads_vision_input():return len(self._encode_prompt(prompt))expanded = self._count_media_prompt_tokens(prompt, images, audio, videos)if expanded is not None:@@ -3832,11 +4006,12 @@ class MLXInferenceBackend:# two stay apart so a client can tell them apart.confirmed = self._kv_cache_window_enforceable(served)enforceable = confirmed is True- quant = _kv_quant_status(- _normalize_mlx_kv_bits(kv_bits),- self._model,- is_vlm,- )+ if getattr(self, "_turboquant", False) and kv_bits is not None:+ quant = _turboquant_status(+ self._model, kv_bits, getattr(self, "_turboquant_refusal", "")+ )+ else:+ quant = _kv_quant_status(_normalize_mlx_kv_bits(kv_bits), self._model, is_vlm)if quant["kv_bits"] is None and enforceable:logger.info("MLX KV cache limited to %d tokens", int(served))return quant, int(served), True, None@@ -3857,7 +4032,7 @@ class MLXInferenceBackend:dtype = None,parallel_mode = None,distributed_group = None,- kv_bits = None,+ kv_quant = None,chat_template_override = None,) -> bool:import mlx.core as mx@@ -3866,8 +4041,24 @@ class MLXInferenceBackend:self._hf_token = hf_tokenmodel_name = config.identifier if hasattr(config, "identifier") else str(config)is_vision = getattr(config, "is_vision", False)+ kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)+ self._turboquant = turboquant+ if self._turboquant:+ try:+ from mlx_vlm.turboquant import TurboQuantKVCache+ except ImportError as exc:+ raise RuntimeError(+ "TurboQuant requires a newer mlx-vlm. Update the MLX packages and retry."+ ) from excdistributed_rank, distributed_size = _mlx_distributed_rank_size(distributed_group)is_distributed = distributed_group is not None and distributed_size > 1+ is_lora = getattr(config, "is_lora", False)+ self._turboquant_refusal = (+ _turboquant_refusal(is_vision = is_vision, is_distributed = is_distributed, is_lora = is_lora)+ if self._turboquant+ else ""+ )+ use_vlm = is_vision or (self._turboquant and not self._turboquant_refusal)self._distributed_group = distributed_groupself._distributed_rank = distributed_rankself._distributed_world_size = distributed_size@@ -3890,12 +4081,10 @@ class MLXInferenceBackend:os.environ["HF_TOKEN"] = hf_tokenself._configure_memory_limits()- is_lora = getattr(config, "is_lora", False)-logger.info("Loading %s via %s (is_lora=%s, distributed=%s, rank=%s/%s, mode=%s)",model_name,- "mlx-vlm" if is_vision else "mlx-lm",+ "mlx-vlm" if use_vlm else "mlx-lm",is_lora,is_distributed,distributed_rank,@@ -3928,7 +4117,7 @@ class MLXInferenceBackend:"load_in_4bit": load_in_4bit,"token": hf_token,"trust_remote_code": trust_remote_code,- "text_only": False if is_vision else True,+ "text_only": not use_vlm,}if is_distributed:if parallel_mode == "pipeline":@@ -3939,12 +4128,37 @@ class MLXInferenceBackend:# Freed before the replacement weights are allocated, for headroom.self._model_fusion.close()self._clear_prompt_cache()- model, tokenizer_or_processor = FastMLXModel.from_pretrained(- model_name,- **load_kwargs,- )+ model = None+ try:+ model, tokenizer_or_processor = FastMLXModel.from_pretrained(+ model_name,+ **load_kwargs,+ )+ except Exception as exc:+ if is_vision or not use_vlm or is_metal_queue_dead(exc):+ raise+ logger.warning(+ "TurboQuant load of %s through mlx-vlm failed (%s); serving through mlx-lm without it",+ model_name,+ exc,+ )+ if model is None:+ import gc++ # Outside the except: its traceback keeps the failed load's weights alive.+ gc.collect()+ _drain_generation_streams(mx)+ mx.clear_cache()+ self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD+ use_vlm = False+ load_kwargs["text_only"] = True+ model, tokenizer_or_processor = FastMLXModel.from_pretrained(+ model_name,+ **load_kwargs,+ )- if is_vision:+ self._reads_vision = bool(is_vision)+ if use_vlm:processor = tokenizer_or_processorself._model = modelself._processor = processor@@ -3973,7 +4187,7 @@ class MLXInferenceBackend:self._kv_cache_window,_ctx_enforced,self._kv_context_budget,- ) = self._resolve_kv_policy(is_vision, kv_bits, max_seq_length, _served_ctx)+ ) = self._resolve_kv_policy(use_vlm, kv_bits, max_seq_length, _served_ctx)if self._kv_quant["kv_bits"] is not None:logger.info("MLX KV cache quantization: %s-bit (%s eligibility)",@@ -3996,7 +4210,7 @@ class MLXInferenceBackend:chat_template_override,self._tokenizer,self._processor,- lambda: self._render_template_probe(is_vision),+ lambda: self._render_template_probe(use_vlm),)if native_marks_audio:_revoke_override_that_drops_audio(self._template_override, self._processor, self._model)@@ -4057,6 +4271,10 @@ class MLXInferenceBackend:"mlx_context_budget": self._kv_context_budget,"mlx_kv_bits": self._kv_quant["kv_bits"],"mlx_kv_bits_requested": self._kv_quant["requested_kv_bits"],+ "mlx_kv_quant": encode_mlx_kv_quant(self._kv_quant["kv_bits"], self._turboquant),+ "mlx_kv_quant_requested": encode_mlx_kv_quant(+ self._kv_quant["requested_kv_bits"], self._turboquant+ ),"mlx_kv_quant_eligibility": self._kv_quant["eligibility"],"mlx_kv_quant_reason": self._kv_quant["reason"],"mlx_kv_quant_note": self._kv_quant["note"],@@ -4131,7 +4349,7 @@ class MLXInferenceBackend:)if isinstance(_proc_tpl, (str, dict, list, tuple)) and _proc_tpl:info["processor_template"] = _proc_tpl- info["renders_image"] = _proc is not None and bool(getattr(self, "_is_vlm", False))+ info["renders_image"] = _proc is not None and self._reads_vision_input()info["accepts_multiple_images"] = info["renders_image"] and bool(getattr(self, "_multi_image_marker", None))@@ -4286,10 +4504,8 @@ class MLXInferenceBackend:full_messages = self._with_system_prompt(messages, system_prompt)if self._is_vlm:- # Through the processor, which is what a vision generation renders with; the text renderer would not- # recover the template failures it recovers from. images=None: an image anywhere in the conversation makes- # the structured-item check raise, and the caller declines rather than pricing a prompt without it.- prompt, _ = self._render_vlm_prompt(+ # images=None: an image anywhere makes the structured-item check raise.+ prompt, _target, _markers = self._render_vlm_prompt(full_messages,None,tools = tools,@@ -4346,7 +4562,7 @@ class MLXInferenceBackend:if video is not None:if not _mlx_vlm_decodes_video():raise RuntimeError("The installed mlx-vlm does not read video.")- if not self._is_vlm:+ if not self._reads_vision_input():raise RuntimeError("The loaded model does not read video.")# Reset so a failed run cannot surface stale stats.@@ -4357,7 +4573,7 @@ class MLXInferenceBackend:# History first, the attachment last: MLX binds pixels to markers in order.attached = list(images or []) + ([image] if image is not None else [])# Shared with the transformers vision path so both render the same turns (#10092).- if self._is_vlm and (attached or video is not None):+ if self._reads_vision_input() and (attached or video is not None):# Processor templates want part lists, the tokenizer fallback wants strings.from core.inference.chat_template_helpers import (chat_render_target as _chat_render_target,@@ -4405,7 +4621,7 @@ class MLXInferenceBackend:if self._is_vlm:stream = self._generate_vlm(full_messages,- attached,+ attached if self._reads_vision_input() else None,temperature,top_p,top_k,@@ -5004,7 +5220,31 @@ class MLXInferenceBackend:raise RuntimeError(f"VLM chat template marked {marked} image(s) for {attached_images} attached.")- return prompt, chat_target++ markers = detect_reasoning_channel_markers(chat_target, tools = tools)+ if not self._reads_vision_input():+ # No images means no image tokens to place, so skipping this would drop the tool schema.+ from core.inference.chat_template_helpers import (+ render_with_native_template_fallback,+ )++ model_info = self.models.get(self.active_model_name, {})+ recovered = render_with_native_template_fallback(+ formatted_prompt = prompt,+ tokenizer = self._tokenizer,+ model_info = model_info,+ active_model_name = self.active_model_name,+ messages = messages,+ tools = tools,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ continue_final_message = continue_final_message,+ hf_token = model_info.get("hf_token"),+ return_metadata = True,+ )+ prompt, markers = recovered.prompt, recovered.reasoning_channel_markers+ return prompt, chat_target, markersdef _plan_vlm_request(self, request) -> "_VLMRowPlan":image = request.get("image")@@ -5037,7 +5277,7 @@ class MLXInferenceBackend:stop = None,) -> "_VLMRowPlan":images = [image] if image is not None else None- prompt, chat_target = self._render_vlm_prompt(+ prompt, chat_target, markers = self._render_vlm_prompt(messages,images,tools = tools,@@ -5052,7 +5292,6 @@ class MLXInferenceBackend:think_prefix = detect_think_prefill(prompt, getattr(chat_target, "all_special_tokens", None))- markers = detect_reasoning_channel_markers(chat_target, tools = tools)normalizer = (make_reasoning_normalizer(markers,@@ -5149,7 +5388,7 @@ class MLXInferenceBackend:from mlx_vlm import stream_generate as vlm_streamimages = list(images) if images else None- prompt, chat_target = self._render_vlm_prompt(+ prompt, chat_target, vlm_reasoning_markers = self._render_vlm_prompt(messages,images,videos = [video] if video is not None else None,@@ -5163,7 +5402,6 @@ class MLXInferenceBackend:from core.inference.chat_template_helpers import detect_think_prefill# Detected once: the decoder keeps the delimiters the normalizer below consumes.- vlm_reasoning_markers = detect_reasoning_channel_markers(chat_target, tools = tools)prefill = detect_think_prefill(prompt,getattr(chat_target, "all_special_tokens", None),@@ -5299,9 +5537,15 @@ class MLXInferenceBackend:else None)+ if not self._reads_vision_input():+ # mlx-vlm adds special tokens by family, doubling a BOS the template already emitted.+ import mlx.core as mx+ vlm_kwargs["input_ids"] = mx.array([self._encode_prompt(prompt)])+session = self._vlm_prompt_cache_session(_adapter_state, images, prompt, has_video = video is not None)+ vlm_kwargs.update(self._kv_runtime_quant_kwargs())if session is not None:vlm_kwargs["prompt_cache"] = session.cachevlm_kwargs["prompt_cache_state"] = session@@ -5830,6 +6074,7 @@ class MLXInferenceBackend:max_tokens = max_new_tokens,# Greedy; the knobs below are load-time state, not caller kwargs.temperature = 0.0,+ **self._kv_runtime_quant_kwargs(),**self._kv_quant_generate_kwargs(),**self._kv_window_generate_kwargs(),)diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 71fcf282611..9137f128cef 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -244,6 +244,8 @@ def _audio_generation_timeout(_MLX_RUNTIME_MIRROR_FIELDS = ("mlx_kv_bits","mlx_kv_bits_requested",+ "mlx_kv_quant",+ "mlx_kv_quant_requested","mlx_kv_quant_eligibility","mlx_kv_quant_reason","mlx_kv_quant_note",@@ -1955,7 +1957,7 @@ class InferenceOrchestrator:subject: Optional[str] = None,tensor_parallel: bool = False,mlx_distributed: bool = False,- mlx_kv_bits: Optional[int] = None,+ mlx_kv_quant: Optional[str] = None,chat_template_override: Optional[str] = None,load_cancel_event: Optional[threading.Event] = None,post_handoff_expected_free_gb: Optional[dict[int, float]] = None,@@ -1999,7 +2001,7 @@ class InferenceOrchestrator:"mlx_parallel_mode": ("tensor" if tensor_parallel else "pipeline")if mlx_distributedelse None,- "mlx_kv_bits": mlx_kv_bits,+ "mlx_kv_quant": mlx_kv_quant,"chat_template_override": chat_template_override,# Read in the worker, which hides the accelerators before detection."audio_device": audio_device,diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 705c54e631c..4c67f8433bf 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -616,7 +616,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:if getattr(backend, "device", None) == "mlx":load_kwargs["parallel_mode"] = config.get("mlx_parallel_mode")load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")- load_kwargs["kv_bits"] = config.get("mlx_kv_bits")+ load_kwargs["kv_quant"] = config.get("mlx_kv_quant")load_kwargs["chat_template_override"] = config.get("chat_template_override")success = backend.load_model(**load_kwargs)finally:@@ -675,6 +675,8 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:"context_unbounded_when_batched","mlx_kv_bits","mlx_kv_bits_requested",+ "mlx_kv_quant",+ "mlx_kv_quant_requested","mlx_kv_quant_eligibility","mlx_kv_quant_reason","mlx_kv_quant_note",diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 2f37b0369e0..53123838982 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -126,16 +126,29 @@ class LoadRequest(BaseModel):"(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32')"),)- mlx_kv_bits: Optional[int] = Field(+ mlx_kv_quant: Optional[str] = Field(None,description = (- "MLX KV cache quantization bit width (8, 6, 5, 4, 3 or 2). MLX takes a bit "- "width rather than a llama.cpp dtype name, so this is separate from "- "cache_type_kv. Omit for an unquantized cache. Ignored by non-MLX "- "backends; a model whose cache layout cannot be quantized reports "- "the reason instead of applying it."+ "MLX KV cache quantization: 'auto' for an unquantized cache, '8'/'6'/'5'/'4'/'3'/'2' "+ "for an mx.quantize width, or 'tq-4'/'tq-3.5'/'tq-3'/'tq-2' for TurboQuant. MLX names a "+ "width rather than a llama.cpp dtype, so this is separate from cache_type_kv. Ignored "+ "by non-MLX backends; a model whose cache layout cannot be quantized reports the reason "+ "instead of applying it."),)+ mlx_kv_bits: Optional[float] = Field(+ None, description = "Superseded by mlx_kv_quant; read only when mlx_kv_quant is omitted."+ )++ @model_validator(mode = "after")+ def derive_mlx_kv_quant(self):+ """A bare width only ever meant mx.quantize; null is how a client spells Auto."""++ if "mlx_kv_quant" not in self.model_fields_set and self.mlx_kv_bits is not None:+ from core.inference.mlx_inference import encode_mlx_kv_quant+ self.mlx_kv_quant = encode_mlx_kv_quant(self.mlx_kv_bits)+ return self+gpu_ids: Optional[List[int]] = Field(None,description = (@@ -1309,17 +1322,22 @@ class _InferenceRuntimeFields(BaseModel):False,description = "Whether the active model runs on the AMD Ryzen AI NPU (FastFlowLM through Lemonade)",)- mlx_kv_bits: Optional[int] = Field(- None, description = "MLX KV quantization bit width actually applied, if any"+ mlx_kv_quant: Optional[str] = Field(+ None, description = "MLX KV cache quantization actually applied, in mlx_kv_quant's vocabulary")- mlx_kv_bits_requested: Optional[int] = Field(+ mlx_kv_quant_requested: Optional[str] = Field(None,description = (- "MLX KV quantization bit width the load asked for. Differs from "- "mlx_kv_bits when the model could not honor it, which is exactly "- "when the reason matters."+ "MLX KV cache quantization the load asked for. Differs from mlx_kv_quant when the model "+ "could not honor it, which is exactly when the reason matters."),)+ mlx_kv_bits: Optional[float] = Field(+ None, description = "mlx_kv_quant as a bare width, kept for clients reading the older field"+ )+ mlx_kv_bits_requested: Optional[float] = Field(+ None, description = "mlx_kv_quant_requested as a bare width, kept for the same reason"+ )chat_template_override: Optional[str] = Field(None,description = "Chat template override this model was loaded with. llama.cpp reports what it applied; MLX reports what was requested, since a refusal applies nothing and the reason accompanies it",diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.pyindex 361dd44cb48..9c6f014a1fa 100644--- a/studio/backend/routes/chat_history.py+++ b/studio/backend/routes/chat_history.py@@ -411,6 +411,10 @@ class ChatPresetLoadConfig(BaseModel):customContextLength: Optional[int] = Field(default = None, gt = 0)maxSeqLength: Optional[float] = NonekvCacheDtype: Optional[str] = None+ mlxKvQuant: Optional[+ Literal["auto", "8", "6", "5", "4", "3", "2", "tq-4", "tq-3.5", "tq-3", "tq-2"]+ ] = None+ # Declared only because extra="forbid" would 400 the whole save for a client still sending it.mlxKvBits: Optional[Literal[8, 6, 5, 4, 3, 2]] = NonespeculativeType: Optional[str] = NonespecDraftNMax: Optional[int] = Field(default = None, ge = 1, le = 16)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 0514be2f75e..62f1b9ae9a6 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -7622,6 +7622,8 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:# Not MLX, so the MLX runtime fields report as absent.is_mlx = False,is_npu = False,+ mlx_kv_quant = None,+ mlx_kv_quant_requested = None,mlx_kv_bits = None,mlx_kv_bits_requested = None,mlx_kv_quant_eligibility = None,@@ -15710,11 +15712,12 @@ def _mlx_runtime_settings_match(backend, request) -> bool:stored None on every request and reload forever."""entry = backend.models.get(backend.active_model_name, {}) or {}- if "mlx_kv_bits_requested" not in entry:+ if "mlx_kv_quant_requested" not in entry:return True- from core.inference.mlx_inference import _normalize_mlx_kv_bits+ from core.inference.mlx_inference import encode_mlx_kv_quant, parse_mlx_kv_quant- return entry["mlx_kv_bits_requested"] == _normalize_mlx_kv_bits(request.mlx_kv_bits) and (+ requested = encode_mlx_kv_quant(*parse_mlx_kv_quant(getattr(request, "mlx_kv_quant", None)))+ return entry["mlx_kv_quant_requested"] == requested and (entry.get("chat_template_override_requested") or None) == (request.chat_template_override or None)@@ -16653,6 +16656,8 @@ async def _load_model_impl(has_audio_input = _model_info.get("has_audio_input", False),has_video_input = _model_info.get("has_video_input", False),is_mlx = bool(_model_info.get("is_mlx", False)),+ mlx_kv_quant = _model_info.get("mlx_kv_quant"),+ mlx_kv_quant_requested = _model_info.get("mlx_kv_quant_requested"),**_unsloth_serving_fields(_model_info),mlx_kv_bits = _model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),@@ -17334,7 +17339,7 @@ async def _load_model_impl(else placement.requested_gpu_ids),subject = current_subject,- mlx_kv_bits = request.mlx_kv_bits,+ mlx_kv_quant = request.mlx_kv_quant,chat_template_override = request.chat_template_override,load_cancel_event = load_cancel_event,on_prior_worker_released = _release_chat_after_teardown,@@ -17467,6 +17472,8 @@ async def _load_model_impl(has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),has_video_input = _model_info.get("has_video_input", False),is_mlx = bool(_model_info.get("is_mlx", False)),+ mlx_kv_quant = _model_info.get("mlx_kv_quant"),+ mlx_kv_quant_requested = _model_info.get("mlx_kv_quant_requested"),**_unsloth_serving_fields(_model_info),mlx_kv_bits = _model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),@@ -19911,6 +19918,8 @@ async def get_status(current_subject: str):has_audio_input = has_audio_input,has_video_input = has_video_input,is_mlx = bool(model_info.get("is_mlx", False)),+ mlx_kv_quant = model_info.get("mlx_kv_quant"),+ mlx_kv_quant_requested = model_info.get("mlx_kv_quant_requested"),**_unsloth_serving_fields(model_info),mlx_kv_bits = model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = model_info.get("mlx_kv_bits_requested"),diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex e2f2b9c2db1..428b9c201f9 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -25,6 +25,7 @@ from pydantic import (StrictInt,ValidationError,field_validator,+ model_validator,)from auth.authentication import (@@ -916,7 +917,19 @@ class ModelOverridePayload(BaseModel):custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576)kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32)# A discrete set, enforced by the normalizer; these bounds only block absurd values.- mlx_kv_bits: Optional[int] = Field(default = None, ge = 2, le = 8)+ mlx_kv_quant: Optional[str] = Field(default = None, max_length = 16)+ mlx_kv_bits: Optional[float] = Field(default = None, ge = 2, le = 8)++ @model_validator(mode = "after")+ def derive_mlx_kv_quant(self):+ """Fold the pair into the field storage keeps; null is how a client spells Auto."""++ if "mlx_kv_quant" not in self.model_fields_set and self.mlx_kv_bits is not None:+ from core.inference.mlx_inference import encode_mlx_kv_quant+ self.mlx_kv_quant = encode_mlx_kv_quant(self.mlx_kv_bits)+ self.mlx_kv_bits = None+ return self+speculative_type: Optional[str] = Field(default = None, max_length = 32)spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16)# Parallel decode slots (llama-server --parallel), GGUF-only; None follows the server default.@@ -2272,7 +2285,7 @@ def update_openai_auto_switch_override(max_seq_length = max_seq_length,custom_context_length = custom_context_length,kv_cache_dtype = payload.kv_cache_dtype,- mlx_kv_bits = payload.mlx_kv_bits,+ mlx_kv_quant = payload.mlx_kv_quant,speculative_type = payload.speculative_type,spec_draft_n_max = payload.spec_draft_n_max,n_parallel = payload.n_parallel,diff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.pyindex 42171f520b5..3c7a66d7fce 100644--- a/studio/backend/tests/test_chat_history_routes.py+++ b/studio/backend/tests/test_chat_history_routes.py@@ -608,12 +608,13 @@ def test_chat_preset_load_config_covers_frontend_persisted_fields():assert persisted, "no PresetLoadConfig keys parsed"backend = set(chat_history.ChatPresetLoadConfig.model_fields)+ backend -= {"mlxKvBits"}assert (persisted == backend), f"schema drift: frontend-only {persisted - backend}, backend-only {backend - persisted}"-def test_chat_settings_payload_accepts_mlx_kv_bits():+def test_chat_settings_payload_accepts_mlx_kv_quant():from pydantic import ValidationError# extra="forbid" rejects the whole settings write on an undeclared key.@@ -623,14 +624,15 @@ def test_chat_settings_payload_accepts_mlx_kv_bits():{"name": "MLX preset","params": {"temperature": 0.7},- "loadConfig": {"mlxKvBits": 8},+ "loadConfig": {"mlxKvQuant": "tq-3.5"},},],})dumped = payload.model_dump(exclude_unset = True)- assert dumped["customPresets"][0]["loadConfig"]["mlxKvBits"] == 8+ assert dumped["customPresets"][0]["loadConfig"]["mlxKvQuant"] == "tq-3.5"+ chat_history.ChatPresetLoadConfig.model_validate({"mlxKvQuant": "auto"})for width in (4, None):chat_history.ChatPresetLoadConfig.model_validate({"mlxKvBits": width})# Only the widths MLX supports.diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex a1f9494acbc..553eda6dfca 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1053,6 +1053,8 @@ class TestLoadHubDownloadExclusion:"parallel_slots","is_mlx","is_npu",+ "mlx_kv_quant",+ "mlx_kv_quant_requested","mlx_kv_bits","mlx_kv_bits_requested","mlx_kv_quant_eligibility",@@ -1077,7 +1079,7 @@ class TestLoadHubDownloadExclusion:fields = route._llama_runtime_fields(backend)assert fields["is_mlx"] is False- assert fields["mlx_kv_bits_requested"] is None+ assert fields["mlx_kv_quant_requested"] is Nonedef test_in_flight_marker_counts_and_normalizes_case(self):assert not hf_gguf_load_in_flight(REPO)diff --git a/studio/backend/tests/test_metal_turboquant.py b/studio/backend/tests/test_metal_turboquant.pynew file mode 100644index 00000000000..981cc137ddc--- /dev/null+++ b/studio/backend/tests/test_metal_turboquant.py@@ -0,0 +1,122 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""What TurboQuant does to a real cache, on the installed mlx-vlm rather than the fake-mlx shim."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++mx = pytest.importorskip("mlx.core", reason = "needs a real MLX runtime")+pytest.importorskip("mlx_vlm.turboquant", reason = "needs an mlx-vlm with TurboQuant")++import mlx.nn as nn # noqa: E402++from core.inference.mlx_inference import ( # noqa: E402+ MLX_KV_GROUP_SIZE,+ _copy_value,+ _turboquant_status,+)+++def _llama(layer_types = None, sliding = None):+ from mlx_vlm.models.llama.config import ModelConfig+ from mlx_vlm.models.llama.language import Model++ model = Model(+ ModelConfig(+ model_type = "llama",+ hidden_size = 64,+ num_hidden_layers = 4,+ intermediate_size = 128,+ num_attention_heads = 4,+ num_key_value_heads = 4,+ rms_norm_eps = 1e-5,+ vocab_size = 128,+ layer_types = layer_types,+ sliding_window = sliding,+ )+ )+ mx.eval(model.parameters())+ return model++[email protected]("bits", [2, 3, 3.5, 4])+def test_every_offered_width_converts_and_decodes(bits):+ status = _turboquant_status(_llama(), bits)+ assert (status["eligibility"], status["kv_bits"], status["note"]) == ("full", bits, "")+++def test_a_model_holding_layers_native_reports_partial_and_says_so():+ sliding = _llama(layer_types = ["sliding_attention"] + ["full_attention"] * 3, sliding = 64)+ status = _turboquant_status(sliding, 4)+ assert (status["eligibility"], status["kv_bits"]) == ("partial", 4) and status["note"]+++class FailsOnDecode(nn.Module):+ def __init__(+ self,+ inner,+ error = "no decode here",+ ):+ super().__init__()+ self.inner = inner+ self.error = error+ self.calls = 0++ @property+ def layers(self):+ return self.inner.layers++ def __call__(self, *args, **kwargs):+ self.calls += 1+ if self.calls > 1:+ raise RuntimeError(self.error)+ return self.inner(*args, **kwargs)+++def test_a_model_that_cannot_decode_is_refused_rather_than_raising():+ model = FailsOnDecode(_llama())+ status = _turboquant_status(model, 4)+ assert model.calls == 2+ assert status["eligibility"] == "refused" and status["kv_bits"] is None+ assert "RuntimeError" in status["reason"]+++def test_a_dead_metal_queue_is_raised_not_reported_as_a_refusal():+ # The orchestrator retires the worker on this marker; a refusal would keep serving on a dead queue.+ model = FailsOnDecode(_llama(), "[METAL] Command buffer execution failed: GPU Timeout Error")+ with pytest.raises(RuntimeError, match = "GPU Timeout"):+ _turboquant_status(model, 4)+++def test_a_copied_turboquant_cache_can_still_be_extended():+ from mlx_vlm.generate.common import maybe_quantize_kv_cache+ from mlx_vlm.models.cache import make_prompt_cache+ from mlx_vlm.turboquant import TurboQuantKVCache++ model = _llama()+ entries = make_prompt_cache(model)+ maybe_quantize_kv_cache(entries, 0, MLX_KV_GROUP_SIZE, 4, kv_quant_scheme = "turboquant")+ result = model(mx.array([[1, 2, 3, 4]]), cache = entries)+ mx.eval(getattr(result, "logits", result), [entry.state for entry in entries])++ at = next(index for index, e in enumerate(entries) if isinstance(e, TurboQuantKVCache))+ original = entries[at].state+ copied = _copy_value(entries, mx)++ assert [type(half) for half in copied[at].state] == [type(half) for half in original]++ served = entries[at].offset+ follow_on = model(mx.array([[5]]), cache = entries)+ mx.eval(getattr(follow_on, "logits", follow_on), [entry.state for entry in entries])+ assert copied[at].offset == served < entries[at].offset++ grown = model(mx.array([[6]]), cache = copied)+ mx.eval(getattr(grown, "logits", grown), [entry.state for entry in copied])+ assert copied[at].offset == served + 1diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex dfd327f88f9..10eb382295b 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -277,6 +277,8 @@ def test_mlx_vlm_generation_survives_every_fusion_refusing(monkeypatch, mlx_infebackend = MLXInferenceBackend()backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})backend._processor = SimpleNamespace(chat_template = "template")+ backend._is_vlm = True+ backend._reads_vision = Trueargs = ([{"role": "user", "content": [{"type": "image"}]}],[object()],@@ -1123,6 +1125,7 @@ def test_mlx_vlm_reemits_think_prefill_inside_adapter_context(backend = MLXInferenceBackend()backend._model = SimpleNamespace(config = {"model_type": "deepseek_vl_v2"})backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())+ backend._reads_vision = Truemonkeypatch.setattr(backend, "_release_vlm_snapshots", lambda: order.append("snapshots_released"))@@ -1201,6 +1204,7 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):)backend = MLXInferenceBackend()backend._model = SimpleNamespace(config = {"model_type": "deepseek_vl_v2"})+ backend._reads_vision = Truebackend._processor = SimpleNamespace(tokenizer = SimpleNamespace())args = ([{"role": "user", "content": [{"type": "image"}]}], [object()], 0, 1, 0, 0, 1, 1, None)tools = [{"function": {"name": "search"}}]@@ -1232,6 +1236,13 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):tool_history = args[0] + [{"role": "assistant", "tool_calls": [{"id": "call-1"}]}]with pytest.raises(RuntimeError, match = "tool-call history"):list(backend._generate_vlm(*((tool_history,) + args[1:]), tools = tools))+ state["generic"] = "generic prompt"+ backend._kv_quant = {"kv_bits": 3.5}+ backend._turboquant = True+ list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools))+ assert calls["stream"][-1][1]["kv_bits"] == 3.5+ assert calls["stream"][-1][1]["kv_quant_scheme"] == "turboquant"+state["generic"] = ValueError("generic rendering failed")state["model"] = f"User: {args[0][0]['content']}"with pytest.raises(ValueError, match = "generic rendering failed"):@@ -1262,6 +1273,7 @@ def test_mlx_vlm_image_injection_reuses_media_aliases(monkeypatch):backend = MLXInferenceBackend()backend._model = object()backend._is_vlm = True+ backend._reads_vision = Truecaptured = []backend._generate_vlm = lambda messages, *_args, **_kwargs: (captured.append(messages) or iter(())@@ -1821,6 +1833,7 @@ def test_mlx_vlm_normalizes_native_reasoning_channels(monkeypatch):apply_chat_template = lambda *_args, **_kwargs: "prompt",)backend._is_vlm = True+ backend._reads_vision = Trueassert list(backend.generate_chat_response(@@ -1867,6 +1880,7 @@ def test_mlx_vlm_post_tool_prompt_opens_reasoning_channel(monkeypatch):apply_chat_template = lambda *_args, **_kwargs: post_tool_prompt,)backend._is_vlm = True+ backend._reads_vision = Truesnapshots = list(backend.generate_chat_response(@@ -2983,6 +2997,25 @@ def test_kv_quant_status_applies_only_when_eligible(monkeypatch):assert mlx_inference._normalize_mlx_kv_bits(8) == 8# Every width the runtime accepts is offered; mlx-lm adds no domain of its own.assert [mlx_inference._normalize_mlx_kv_bits(b) for b in (2, 3, 5, 6)] == [2, 3, 5, 6]+ assert [mlx_inference.parse_mlx_kv_quant(c) for c in ("auto", "4", "tq-4", "tq-3.5")] == [+ (None, False),+ (4, False),+ (4, True),+ (3.5, True),+ ]++ refuse = mlx_inference._turboquant_refusal+ assert refuse(is_vision = False, is_distributed = False, is_lora = True)+ assert refuse(is_vision = False, is_distributed = True, is_lora = False)+ assert not refuse(is_vision = False, is_distributed = False, is_lora = False)+ assert not refuse(is_vision = True, is_distributed = True, is_lora = True)++ turbo = mlx_inference.MLXInferenceBackend()+ turbo._kv_quant = {"kv_bits": 3.5}+ turbo._turboquant = True+ width = {"kv_bits": 3.5, "kv_quant_scheme": "turboquant", "quantized_kv_start": 0}+ assert turbo._kv_runtime_quant_kwargs() == width+ assert turbo._kv_quant_generate_kwargs() == {}text = mlx_inference._kv_quant_status(8, object(), False)vlm = mlx_inference._kv_quant_status(8, object(), True)@@ -3028,6 +3061,57 @@ def _tiny_lm(return _LM()+def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypatch):+ pytest.importorskip("mlx.core")+ from core.inference import mlx_inference++ attempts = []++ class _Loader:+ @staticmethod+ def from_pretrained(name, **kwargs):+ # The retry must not run under the failed load's traceback, which pins its weights.+ attempts.append((kwargs["text_only"], sys.exc_info()[0]))+ if not kwargs["text_only"]:+ raise ValueError("Expected shape (262144, 640) but received shape (262144, 80)")+ return SimpleNamespace(config = {}), SimpleNamespace()++ loader = types.ModuleType("unsloth_zoo.mlx.loader")+ loader.FastMLXModel = _Loader+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)+ monkeypatch.setattr(+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (2048, 2048, 2048)+ )++ backend = mlx_inference.MLXInferenceBackend()+ config = SimpleNamespace(identifier = "org/prequantized-text", is_vision = False, is_lora = False)+ backend.load_model(config, kv_quant = "tq-4")++ assert attempts == [(False, None), (True, None)]+ assert backend._turboquant_refusal == mlx_inference.MLX_TURBOQUANT_TEXT_LOAD+ assert backend._is_vlm is False+ assert backend._kv_quant["kv_bits"] is None++ attempts.clear()+ vision = SimpleNamespace(identifier = "org/vlm", is_vision = True, is_lora = False)+ with pytest.raises(ValueError, match = "Expected shape"):+ mlx_inference.MLXInferenceBackend().load_model(vision, kv_quant = "tq-4")+ assert attempts == [(False, None)]++ class _DeadQueue(_Loader):+ @staticmethod+ def from_pretrained(name, **kwargs):+ attempts.append((kwargs["text_only"], None))+ raise RuntimeError("[METAL] Command buffer execution failed: GPU Timeout Error")++ attempts.clear()+ loader.FastMLXModel = _DeadQueue+ with pytest.raises(RuntimeError, match = "GPU Timeout"):+ mlx_inference.MLXInferenceBackend().load_model(config, kv_quant = "tq-4")+ assert attempts == [(False, None)]++def test_reload_comparison_and_response_carry_the_resolved_setting():"""A load-time knob must force a reload and reach the client.@@ -3041,19 +3125,23 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():def req(**knobs):return LoadRequest(model = "m", model_path = "m", **knobs)- be = SimpleNamespace(active_model_name = "m", models = {"m": {"mlx_kv_bits_requested": 8}})+ be = SimpleNamespace(active_model_name = "m", models = {"m": {"mlx_kv_quant_requested": "8"}})+ assert _mlx_runtime_settings_match(be, req(mlx_kv_quant = "8"))+ assert not _mlx_runtime_settings_match(be, req(mlx_kv_quant = "4"))assert _mlx_runtime_settings_match(be, req(mlx_kv_bits = 8))- assert not _mlx_runtime_settings_match(be, req(mlx_kv_bits = 4))- be.models["m"] = {"mlx_kv_bits_requested": None}- assert _mlx_runtime_settings_match(be, req(mlx_kv_bits = 7)) # both normalize away+ assert req(mlx_kv_quant = None, mlx_kv_bits = 8).mlx_kv_quant is None+ be.models["m"] = {"mlx_kv_quant_requested": "4"}+ assert not _mlx_runtime_settings_match(be, req(mlx_kv_quant = "tq-4"))+ be.models["m"] = {"mlx_kv_quant_requested": "auto"}+ assert _mlx_runtime_settings_match(be, req(mlx_kv_quant = "tq-6")) # both normalize awayassert _mlx_runtime_settings_match(SimpleNamespace(active_model_name = "m", models = {"m": {}}),- req(mlx_kv_bits = 8),+ req(mlx_kv_quant = "8"),)# Compared against the REQUESTED value, or a refused template reloads every request.be.models["m"] = {- "mlx_kv_bits_requested": None,+ "mlx_kv_quant_requested": "auto","chat_template_override_requested": "{{ custom }}",}assert _mlx_runtime_settings_match(be, req(chat_template_override = "{{ custom }}"))@@ -3061,7 +3149,7 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():assert not _mlx_runtime_settings_match(be, req())# A refusal records the request, so the same request still matches.be.models["m"] = {- "mlx_kv_bits_requested": None,+ "mlx_kv_quant_requested": "auto","chat_template_override_requested": "{{ refused }}","chat_template_override_reason": "it could not render a conversation",}@@ -3072,11 +3160,11 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():model = "m",display_name = "m",inference = {},- mlx_kv_bits = 8,+ mlx_kv_quant = "8",mlx_kv_quant_eligibility = "full",mlx_kv_quant_note = "n",)- assert resp.mlx_kv_bits == 8 and resp.mlx_kv_quant_note == "n"+ assert resp.mlx_kv_quant == "8" and resp.mlx_kv_quant_note == "n"def test_kv_quant_probe_reports_what_the_runtime_would_really_do(monkeypatch):@@ -3131,8 +3219,8 @@ def test_parent_mirror_omits_runtime_fields_a_backend_never_reported():assert _mlx_runtime_mirror_fields({"is_vision": False}) == {}assert _mlx_runtime_mirror_fields(- {"mlx_kv_bits": 8, "mlx_kv_bits_requested": 8, "other": 1}- ) == {"mlx_kv_bits": 8, "mlx_kv_bits_requested": 8}+ {"mlx_kv_quant": "8", "mlx_kv_quant_requested": "8", "other": 1}+ ) == {"mlx_kv_quant": "8", "mlx_kv_quant_requested": "8"}def test_chat_template_override_installs_only_where_one_already_exists():@@ -3350,6 +3438,8 @@ def _vlm_backend(chat_template = "t", tokenizer = backend._tokenizer, apply_chat_template = lambda *a, **k: "")backend._model = object()+ backend._is_vlm = True+ backend._reads_vision = Truereturn backend@@ -4007,6 +4097,7 @@ def test_vlm_seed_rides_on_the_sampler_not_a_seed_kwarg(monkeypatch):backend = MLXInferenceBackend()backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})backend._processor = SimpleNamespace(chat_template = "template")+ backend._reads_vision = Trueargs = ([{"role": "user", "content": [{"type": "image"}]}],[object()],@@ -5079,6 +5170,7 @@ def test_the_window_reaches_the_runtime_on_every_generation_route(monkeypatch):vlm = MLXInferenceBackend()vlm._kv_cache_window = 4096vlm._model = SimpleNamespace(config = {"model_type": "m"})+ vlm._reads_vision = Truevlm._processor = SimpleNamespace(tokenizer = SimpleNamespace())next(vlm._generate_vlm(@@ -6049,6 +6141,7 @@ def _run_spm_vlm_turn(tokenizer = turn, chat_template = turn.chat_template, detokenizer = turn.detokenizer)backend._tokenizer = turn+ backend._reads_vision = Truereturn list(backend._generate_vlm([{"role": "user", "content": [{"type": "image"}]}],@@ -6926,6 +7019,7 @@ def _run_vlm_budget(backend = _budget_backend(monkeypatch, served = served, marker_tokens = marker_tokens)backend._is_vlm = True+ backend._reads_vision = Truebackend._model = SimpleNamespace()backend._processor = SimpleNamespace(tokenizer = backend._tokenizer)args = (messages, [image] if image is not None else None, 0, 1, 0, 0, max_new_tokens, 1, None)@@ -7083,6 +7177,7 @@ def _video_vlm_backend(monkeypatch, streams):backend._model = SimpleNamespace(config = {"model_type": "qwen3_5"})backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())backend._is_vlm = True+ backend._reads_vision = Truereturn backend@@ -7367,6 +7462,7 @@ def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_clibackend = MLXInferenceBackend()backend._model = object()backend._is_vlm = True+ backend._reads_vision = Truecaptured = []backend._generate_vlm = lambda messages, *_args, **kwargs: (captured.append((messages, kwargs)) or iter(())@@ -7378,11 +7474,12 @@ def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_cliassert messages[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]assert kwargs["video"] == _CLIP_B64- backend._is_vlm = False+ backend._reads_vision = Falsewith pytest.raises(RuntimeError, match = "loaded model does not read video"):list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))backend._is_vlm = True+ backend._reads_vision = Truemonkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)with pytest.raises(RuntimeError, match = "installed mlx-vlm does not read video"):list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))@@ -7859,6 +7956,8 @@ def _constrained_vlm_reply(backend = MLXInferenceBackend()backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})backend._processor = SimpleNamespace(chat_template = "template", all_special_tokens = specials)+ backend._is_vlm = True+ backend._reads_vision = Trueargs = ([{"role": "user", "content": "hi"}], [], 0.7, 0.9, 40, 0.01, 64, 1.0, None)return list(backend._generate_vlm(*args, response_format = {"type": "json_object"}))@@ -8111,3 +8210,90 @@ def test_the_grammar_ends_on_the_stop_the_decode_loop_names():_allowed(constraint, len(tokenizer))constraint.advance(int(token_id))assert _allowed(constraint, len(tokenizer)) == {runtime_stop}+++def test_turboquant_leaves_every_cache_for_the_runtime_to_build():+ lm_cache = pytest.importorskip("mlx_lm.models.cache")++ from core.inference.mlx_inference import MLXInferenceBackend++ backend = MLXInferenceBackend()+ kinds = lambda: [+ type(entry).__name__+ for entry in backend._prepare_kv_entries([lm_cache.KVCache(), lm_cache.KVCache()])+ ]++ backend._kv_quant = {"kv_bits": 3.5}+ backend._turboquant = True+ assert kinds() == ["KVCache"] * 2+ backend._kv_quant = {"kv_bits": 4}+ assert kinds() == ["KVCache"] * 2++ backend._turboquant = False+ assert kinds() == ["QuantizedKVCache"] * 2+++def test_turboquant_sends_its_width_on_both_vlm_cache_branches(monkeypatch):+ """No pre-converted entry carries the width, so a reused session must still send it."""+ from core.inference.mlx_inference import MLXInferenceBackend++ backend = MLXInferenceBackend()+ backend._reads_vision = True+ backend._kv_quant = {"kv_bits": 3.5}+ backend._turboquant = True++ class _Session(contextlib.AbstractContextManager):+ cache, media_block, step, produced_tokens, produced_seconds = [object()], None, 8, 0, 0.0++ def __exit__(self, *_exc):+ return False++ def finish(self):+ return None++ session = _Session()+ fresh = dict(_drive_vlm_generation(backend, monkeypatch))+ reused = dict(_drive_vlm_generation(backend, monkeypatch, session = session))+ for kwargs in (fresh, reused):+ assert (kwargs["kv_bits"], kwargs["kv_quant_scheme"], kwargs["quantized_kv_start"]) == (+ 3.5,+ "turboquant",+ 0,+ )+ assert "prompt_cache" in reused and "prompt_cache" not in fresh+++def test_a_turboquant_load_turns_a_pin_into_a_request_budget(monkeypatch):+ # Same rule as mx.quantize since #11084: an enforceable pin budgets requests, it does not refuse.+ pytest.importorskip("mlx_lm.models.cache")+ from core.inference import mlx_inference+ from core.inference.mlx_inference import MLXInferenceBackend++ def status(+ _model,+ bits,+ refusal = "",+ ):+ if refusal:+ return {"kv_bits": None, "eligibility": "refused", "reason": refusal}+ return {"kv_bits": bits, "eligibility": "full", "reason": ""}++ monkeypatch.setattr(mlx_inference, "_turboquant_status", status)+ honours = SimpleNamespace(layers = [object(), object()])++ def policy(+ requested,+ served,+ refusal = "",+ ):+ backend = MLXInferenceBackend()+ backend._model = honours+ backend._is_vlm = True+ backend._turboquant = True+ backend._turboquant_refusal = refusal+ quant, window, enforced, budget = backend._resolve_kv_policy(True, 3.5, requested, served)+ return quant["kv_bits"], window, enforced, budget++ assert policy(8192, 8192) == (3.5, None, False, 8192)+ assert policy(0, 262144) == (3.5, None, False, None)+ assert policy(8192, 8192, "refused") == (None, 8192, True, None)diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 4d7aa9386e6..966f7a59ba1 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -1415,6 +1415,12 @@ def _put(model_id, **fields):)+def test_an_older_clients_cache_width_survives_the_override_route(monkeypatch):+ _mock_override_store(monkeypatch)+ _put("org/m", mlx_kv_bits = 8)+ assert settings.get_model_overrides()["org/m"]["mlx_kv_quant"] == "8"++def test_model_override_roundtrip(monkeypatch):_mock_override_store(monkeypatch)@@ -8588,6 +8594,7 @@ def test_fill_absent_fields_carries_the_browser_only_settings_into_a_legacy_entrmax_seq_length = 8192,)settings_route.update_openai_auto_switch_override(legacy, "tester")+ store[settings.MODEL_OVERRIDES_SETTING_KEY]["unsloth/B-GGUF:Q4_K_M"]["mlx_kv_bits"] = 8backfill = settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF:Q4_K_M",@@ -8595,6 +8602,7 @@ def test_fill_absent_fields_carries_the_browser_only_settings_into_a_legacy_entrmax_seq_length = 2048,custom_context_length = 32768,kv_cache_dtype = "q8_0",+ mlx_kv_quant = "tq-4",speculative_type = "ngram",gpu_ids = [0, 1],fill_absent_fields = True,@@ -8609,6 +8617,7 @@ def test_fill_absent_fields_carries_the_browser_only_settings_into_a_legacy_entrassert entry["kv_cache_dtype"] == "q8_0"assert entry["speculative_type"] == "ngram"assert entry["gpu_ids"] == [0, 1]+ assert settings.model_override_load_kwargs(entry, is_gguf = False)["mlx_kv_quant"] == "8"# One entry, not two: the fill resolves onto the key a load reads.assert list(store[settings.MODEL_OVERRIDES_SETTING_KEY]) == ["unsloth/B-GGUF:Q4_K_M"]@@ -9064,22 +9073,44 @@ def test_two_spellings_of_one_cached_quant_do_not_delete_each_others_save(monkeyassert stored[list(stored)[0]]["max_seq_length"] in (4096, 8192)-def test_mlx_kv_bits_survives_the_whole_override_projection():+def test_mlx_kv_quant_survives_the_whole_override_projection():# Dropped here, an API auto-switch would load a remembered MLX model at full- # precision while the picker honored the width.- for bits in (8, 6, 5, 4, 3, 2):- assert settings.normalize_model_override({"mlx_kv_bits": bits}) == {"mlx_kv_bits": bits}+ # precision while the picker honored the setting.+ for quant in ("8", "6", "5", "4", "3", "2", "tq-4", "tq-3.5", "tq-3", "tq-2"):+ assert settings.normalize_model_override({"mlx_kv_quant": quant}) == {"mlx_kv_quant": quant}++ for rejected in ("7", "3.5", "tq-8", "tq-6", "auto", 4, True, None):+ assert settings.normalize_model_override({"mlx_kv_quant": rejected}) == {}+ assert settings.normalize_model_override({"mlx_kv_bits": 8}) == {"mlx_kv_quant": "8"}+ # A hand-edited width drops alone instead of aborting the whole override.+ for stored in (7, "invalid", "4", True, [4]):+ assert settings.normalize_model_override({"mlx_kv_bits": stored}) == {}+ assert settings.model_override_load_kwargs({"mlx_kv_bits": stored}, is_gguf = False) == {}+ assert settings.model_override_load_kwargs({"mlx_kv_bits": 8}, is_gguf = False) == {+ "mlx_kv_quant": "8"+ }+ assert (+ settings.model_override_load_kwargs({"mlx_kv_quant": None, "mlx_kv_bits": 8}, is_gguf = False)+ == {}+ )+ both = {"mlx_kv_quant": "auto", "mlx_kv_bits": 4}+ assert settings.normalize_model_override(both) == {}+ assert settings.model_override_load_kwargs(both, is_gguf = False) == {}+ assert LoadRequest(model_path = "unsloth/A", **both).mlx_kv_quant == "auto"++ def _folded(**kw):+ payload = settings_route.ModelOverridePayload(model_id = "m", **kw)+ return payload.mlx_kv_quant, payload.mlx_kv_bits- # A discrete set, so an in-range width can still be one mx.quantize rejects.- # bool is an int subclass, and a string width would reach LoadRequest untyped.- for rejected in (7, 1, 0, 9, True, False, "4", 4.5, None):- assert settings.normalize_model_override({"mlx_kv_bits": rejected}) == {}+ assert _folded(mlx_kv_bits = 8) == ("8", None)+ assert _folded(mlx_kv_quant = None, mlx_kv_bits = 8) == (None, None)+ assert _folded(mlx_kv_quant = "tq-4", mlx_kv_bits = 8) == ("tq-4", None)# Ungated on is_gguf, matching the picker's own load payload.for is_gguf in (True, False):- kwargs = settings.model_override_load_kwargs({"mlx_kv_bits": 4}, is_gguf = is_gguf)- assert kwargs["mlx_kv_bits"] == 4- assert LoadRequest(model_path = "unsloth/A", **kwargs).mlx_kv_bits == 4+ kwargs = settings.model_override_load_kwargs({"mlx_kv_quant": "tq-4"}, is_gguf = is_gguf)+ assert kwargs["mlx_kv_quant"] == "tq-4"+ assert LoadRequest(model_path = "unsloth/A", **kwargs).mlx_kv_quant == "tq-4"def _idle_backend(kw, monkeypatch, *, user_loaded):diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.pyindex 7aabfdd22af..576713f8dd7 100644--- a/studio/backend/utils/openai_auto_switch_settings.py+++ b/studio/backend/utils/openai_auto_switch_settings.py@@ -365,8 +365,8 @@ CTX_CHECKPOINTS_MAX = 256CACHE_RAM_MIN_MIB = -1CACHE_RAM_MAX_MIB = 1024 * 1024VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"})-# Mirrors MLX_KV_BITS_CHOICES in core/inference/mlx_inference.py; a set, not a range.-VALID_MLX_KV_BITS = frozenset({8, 6, 5, 4, 3, 2})+# Mirrors MLX_KV_QUANT_CHOICES in core/inference/mlx_inference.py; a set, not a range.+VALID_MLX_KV_QUANT = frozenset({"8", "6", "5", "4", "3", "2", "tq-4", "tq-3.5", "tq-3", "tq-2"})# Mirrors PARALLEL_MIN/MAX in llama_server_args.py.PARALLEL_SLOTS_MIN = 1@@ -386,6 +386,20 @@ VALID_GPU_INDEX_KINDS = frozenset({"physical", "vulkan"})LEGACY_GPU_INDEX_KIND = "physical"+def _mlx_kv_quant_of(entry: dict[str, Any]) -> Optional[str]:+ """This entry's cache quantization, reading the width it superseded when that is all it+ holds. Only an entry omitting the field predates the setting."""+ if "mlx_kv_quant" in entry:+ return _clean_str(entry["mlx_kv_quant"], VALID_MLX_KV_QUANT)+ bits = entry.get("mlx_kv_bits")+ # Only a number: a hand-edited string or a bool must drop the width, not abort the whole override.+ if isinstance(bits, bool) or not isinstance(bits, (int, float)):+ return None+ from core.inference.mlx_inference import encode_mlx_kv_quant++ return _clean_str(encode_mlx_kv_quant(bits), VALID_MLX_KV_QUANT)++def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]:if not isinstance(value, str):return None@@ -428,10 +442,9 @@ def normalize_model_override(if kv_cache_dtype:entry["kv_cache_dtype"] = kv_cache_dtype- # MLX quantizes by bit width, not by a llama.cpp dtype name, so it is its own field.- mlx_kv_bits = payload.get("mlx_kv_bits")- if not isinstance(mlx_kv_bits, bool) and mlx_kv_bits in VALID_MLX_KV_BITS:- entry["mlx_kv_bits"] = int(mlx_kv_bits)+ mlx_kv_quant = _mlx_kv_quant_of(payload)+ if mlx_kv_quant:+ entry["mlx_kv_quant"] = mlx_kv_quantspeculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES)if speculative_type:@@ -581,7 +594,6 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> difor source, target in (("llama_extra_args", "llama_extra_args"),("kv_cache_dtype", "cache_type_kv"),- ("mlx_kv_bits", "mlx_kv_bits"),("n_parallel", "n_parallel"),("speculative_type", "speculative_type"),("spec_draft_n_max", "spec_draft_n_max"),@@ -594,6 +606,10 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> diif override.get(source) is not None:kwargs[target] = override[source]+ mlx_kv_quant = _mlx_kv_quant_of(override)+ if mlx_kv_quant:+ kwargs["mlx_kv_quant"] = mlx_kv_quant+if is_gguf:if override.get("n_batch") is not None:kwargs["n_batch"] = override["n_batch"]@@ -878,8 +894,11 @@ def set_model_override(model_id.strip(),entry or None,fill_absent_fields = fill_absent_fields,- # The pin and its index space are one value: filling the qualifier onto ids this browser did not write relabels them.- coupled_fields = (("gpu_ids", "gpu_index_kind"),),+ coupled_fields = (+ # The pin and its index space are one value: filling the qualifier onto ids this browser did not write relabels them.+ ("gpu_ids", "gpu_index_kind"),+ ("mlx_kv_quant", "mlx_kv_bits"),+ ),)_invalidate(MODEL_OVERRIDES_SETTING_KEY)return entrydiff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 30bcee4f0ff..6a0d5dba518 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -2405,8 +2405,8 @@ const VISIBLE_MODEL_RUNTIME_KEYS = ["chatTemplateOverrideReason",// Or a background autoload leaves its width and verdict on the restored model.// The rest of the group mlxRuntimeStateFrom writes.- "mlxKvBits",- "loadedMlxKvBitsRequested",+ "mlxKvQuant",+ "loadedMlxKvQuantRequested","mlxKvQuantReason","mlxKvQuantNote","loadedContextBudget",@@ -3451,7 +3451,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{trust_remote_code: trustRemoteCode,chat_template_override: effectiveChatTemplateOverride,cache_type_kv: config.kvCacheDtype,- mlx_kv_bits: config.mlxKvBits ?? null,+ mlx_kv_quant: config.mlxKvQuant ?? null,speculative_type: effectiveSpeculativeType,spec_draft_n_max: effectiveSpecDraftNMax,reasoning_budget:diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsxindex ca771dd0a05..3652ad28d40 100644--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx@@ -551,7 +551,7 @@ export function ChatSettingsPanel({);const customContextLength = useChatRuntimeStore((s) => s.customContextLength);const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);- const mlxKvBits = useChatRuntimeStore((s) => s.mlxKvBits);+ const mlxKvQuant = useChatRuntimeStore((s) => s.mlxKvQuant);const gpuMemoryMode = useChatRuntimeStore((s) => s.gpuMemoryMode);const gpuLayers = useChatRuntimeStore((s) => s.gpuLayers);const nCpuMoe = useChatRuntimeStore((s) => s.nCpuMoe);@@ -747,7 +747,7 @@ export function ChatSettingsPanel({customContextLength,loadedContextLength,kvCacheDtype,- mlxKvBits,+ mlxKvQuant,gpuMemoryMode,gpuLayers,nCpuMoe,@@ -772,7 +772,7 @@ export function ChatSettingsPanel({customContextLength,loadedContextLength,kvCacheDtype,- mlxKvBits,+ mlxKvQuant,gpuMemoryMode,gpuLayers,nCpuMoe,diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex 2dfe699584e..030e20868a0 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -1901,10 +1901,9 @@ export function useChatModelRuntime() {const previousServerTuning: ServerTuningValues =rollbackConfig ?? useChatRuntimeStore.getState();// Same reason: the rollback echo would overwrite an edit staged against it.- const previousMlxKvBits =- rollbackConfig- ? (rollbackConfig.mlxKvBits ?? null)- : useChatRuntimeStore.getState().mlxKvBits;+ const previousMlxKvQuant = rollbackConfig+ ? (rollbackConfig.mlxKvQuant ?? null)+ : useChatRuntimeStore.getState().mlxKvQuant;if (isGguf && isDiffusion === undefined) {// Prepare the token exactly as validateModel/loadModel do: the Hub rejects an invalid// Authorization header with 401 even for a public repo, so sending the raw stored token here would@@ -2000,8 +1999,10 @@ export function useChatModelRuntime() {const loadKvCacheDtype =pendingLoadConfig?.kvCacheDtype ?? stateBeforeUnload.kvCacheDtype;// Per-model, not a standing preference: eligibility is decided per model.- let loadMlxKvBits =- pendingLoadConfig?.mlxKvBits ?? stateBeforeUnload.mlxKvBits;+ let loadMlxKvQuant =+ pendingLoadConfig+ ? pendingLoadConfig.mlxKvQuant ?? null+ : stateBeforeUnload.mlxKvQuant;// gpuMemoryMode is a standing preference; the rest are per-model knobs the reset below clears, so// they are re-baselined there in lock-step with the store. A GGUF native context can exceed// maxSeqLength, so sizing on raw maxSeqLength could pass, unload, then have /load refuse it. A@@ -2370,9 +2371,7 @@ export function useChatModelRuntime() {ctxCheckpoints: pendingLoadConfig?.ctxCheckpoints ?? null,cacheRam: pendingLoadConfig?.cacheRam ?? null,};- // Both payload-only. The store keeps its values: a width is dormant preset state off MLX, and a- // completed load rewrites both anyway.- loadMlxKvBits = pendingLoadConfig?.mlxKvBits ?? null;+ loadMlxKvQuant = pendingLoadConfig?.mlxKvQuant ?? null;loadChatTemplateOverride =pendingLoadConfig?.chatTemplateOverride?.trim()? pendingLoadConfig.chatTemplateOverride@@ -2469,7 +2468,7 @@ export function useChatModelRuntime() {approved_remote_code_fingerprint: approvedRemoteCodeFingerprint,chat_template_override: effectiveChatTemplateOverride,cache_type_kv: loadKvCacheDtype,- mlx_kv_bits: loadMlxKvBits ?? null,+ mlx_kv_quant: loadMlxKvQuant ?? null,speculative_type: loadSpeculativeType,spec_draft_n_max: loadSpecDraftNMax,n_parallel: loadNParallel,@@ -2842,7 +2841,7 @@ export function useChatModelRuntime() {chat_template_override:rollbackState.loadedChatTemplateOverride,cache_type_kv: rollbackState.loadedKvCacheDtype,- mlx_kv_bits: rollbackState.loadedMlxKvBitsRequested,+ mlx_kv_quant: rollbackState.loadedMlxKvQuantRequested,speculative_type:rollbackState.loadedSpeculativeType,spec_draft_n_max:@@ -2946,7 +2945,7 @@ export function useChatModelRuntime() {...mlxRuntimeStateFrom(rollbackResponse),// After the spread, which seeds the control from the echo; the control keeps its intent, like// nParallel above.- mlxKvBits: previousMlxKvBits,+ mlxKvQuant: previousMlxKvQuant,loadedChatTemplateOverride:rollbackState.loadedChatTemplateOverride,...loadedGpuMemoryFields(rollbackResponse),diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex c528c2a6ca9..e4dc6ca9a75 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -10,6 +10,7 @@ import {savedContextPin,} from "@/features/model-picker";// eslint-disable-next-line no-restricted-imports -- Avoid the hub barrel's React and download-manager exports.+import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";import { modelDisplayName } from "@/features/hub/lib/model-identity";import { getInferenceStatus } from "../api/chat-api";import { isSpeechOnlyStatus } from "./speech-only-status";@@ -533,11 +534,11 @@ export function applyActiveModelStatusToStore(// and request move together; a late reply can overwrite a newer one....(seedLoadParams &&hydratingExistingModel &&- status.mlx_kv_bits !== undefined &&+ status.mlx_kv_quant_requested !== undefined &&(status.is_mlx === true? {- mlxKvBits: status.mlx_kv_bits_requested ?? null,- loadedMlxKvBitsRequested: status.mlx_kv_bits_requested ?? null,+ mlxKvQuant: normalizeMlxKvQuant(status.mlx_kv_quant_requested),+ loadedMlxKvQuantRequested: normalizeMlxKvQuant(status.mlx_kv_quant_requested),mlxKvQuantReason: status.mlx_kv_quant_reason ?? null,chatTemplateOverrideReason:status.chat_template_override_reason ?? null,@@ -545,7 +546,7 @@ export function applyActiveModelStatusToStore(}: {// The verdict retires; the editable width is dormant, not wrong.- loadedMlxKvBitsRequested: null,+ loadedMlxKvQuantRequested: null,mlxKvQuantReason: null,chatTemplateOverrideReason: null,mlxKvQuantNote: null,@@ -555,13 +556,13 @@ export function applyActiveModelStatusToStore(...(seedLoadParams &&!hydratingExistingModel &&status.is_mlx === true &&- status.mlx_kv_bits !== undefined &&- prevState.mlxKvBits === null &&- prevState.loadedMlxKvBitsRequested === null &&+ status.mlx_kv_quant_requested !== undefined &&+ prevState.mlxKvQuant === null &&+ prevState.loadedMlxKvQuantRequested === null &&prevState.mlxKvQuantReason === null &&prevState.chatTemplateOverrideReason === null && {- mlxKvBits: status.mlx_kv_bits_requested ?? null,- loadedMlxKvBitsRequested: status.mlx_kv_bits_requested ?? null,+ mlxKvQuant: normalizeMlxKvQuant(status.mlx_kv_quant_requested),+ loadedMlxKvQuantRequested: normalizeMlxKvQuant(status.mlx_kv_quant_requested),mlxKvQuantReason: status.mlx_kv_quant_reason ?? null,chatTemplateOverrideReason: status.chat_template_override_reason ?? null,mlxKvQuantNote: status.mlx_kv_quant_note ?? null,diff --git a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts b/studio/frontend/src/features/chat/lib/mlx-runtime-state.tsindex d4706e24051..97ef5d9ca62 100644--- a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts+++ b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts@@ -1,39 +1,43 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import {+ normalizeMlxKvQuant,+ type MlxKvQuant,+} from "@/features/model-picker/model-config/per-model-config";import type { LoadModelResponse } from "../types/api";type MlxRuntimeResponse = Pick<LoadModelResponse,| "is_mlx"- | "mlx_kv_bits_requested"+ | "mlx_kv_quant_requested"| "mlx_kv_quant_reason"| "chat_template_override_reason"| "mlx_kv_quant_note">;/** MLX KV-quantization state a load response establishes. A non-MLX response retires the verdict but- * omits mlxKvBits: the width is dormant there, not wrong, and a preset carrying it must survive+ * omits mlxKvQuant: the setting is dormant there, not wrong, and a preset carrying it must survive* the round-trip. */export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {- mlxKvBits?: number | null;- loadedMlxKvBitsRequested: number | null;+ mlxKvQuant?: MlxKvQuant | null;+ loadedMlxKvQuantRequested: MlxKvQuant | null;mlxKvQuantReason: string | null;chatTemplateOverrideReason: string | null;mlxKvQuantNote: string | null;} {if (resp.is_mlx !== true) {return {- loadedMlxKvBitsRequested: null,+ loadedMlxKvQuantRequested: null,mlxKvQuantReason: null,chatTemplateOverrideReason: null,mlxKvQuantNote: null,};}+ const requested = normalizeMlxKvQuant(resp.mlx_kv_quant_requested);return {- // Requested, not applied: a refusal applies no width but still has a reason.- mlxKvBits: resp.mlx_kv_bits_requested ?? null,- loadedMlxKvBitsRequested: resp.mlx_kv_bits_requested ?? null,+ mlxKvQuant: requested,+ loadedMlxKvQuantRequested: requested,mlxKvQuantReason: resp.mlx_kv_quant_reason ?? null,chatTemplateOverrideReason: resp.chat_template_override_reason ?? null,mlxKvQuantNote: resp.mlx_kv_quant_note ?? null,diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex eecb3bc01b2..b34e0b869db 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import type { PerModelConfig } from "@/features/model-picker";+import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";import {parseGpuLayersOverride,@@ -17,7 +18,7 @@ type ResidentRuntime = Pick<InferenceStatusResponse,| "requested_context_length"| "cache_type_kv"- | "mlx_kv_bits_requested"+ | "mlx_kv_quant_requested"| "speculative_type"| "spec_draft_n_max"| "requested_parallel_slots"@@ -260,7 +261,7 @@ const SETTING_CHECKS: SettingCheck[] = [mlxComparable: true,pinned: () => true,agrees: (c, s) =>- (c.mlxKvBits ?? null) === (s.mlx_kv_bits_requested ?? null),+ (c.mlxKvQuant ?? null) === normalizeMlxKvQuant(s.mlx_kv_quant_requested),},{// Always pinned: an unset mode resolves to the standing preference and the load sends it. Readingdiff --git a/studio/frontend/src/features/chat/presets/preset-load-config.ts b/studio/frontend/src/features/chat/presets/preset-load-config.tsindex 3191798e18b..007d092fe5a 100644--- a/studio/frontend/src/features/chat/presets/preset-load-config.ts+++ b/studio/frontend/src/features/chat/presets/preset-load-config.ts@@ -12,7 +12,8 @@ import {MAX_SEQ_LENGTH_MAX,DEFAULT_MAX_SEQ_LENGTH,KV_CACHE_DTYPES,- MLX_KV_BITS,+ mlxKvQuantLabel,+ normalizeMlxKvQuant,N_BATCH_MAX,N_BATCH_MIN,N_PARALLEL_MAX,@@ -45,7 +46,7 @@ export type PresetLoadConfig = Pick<| "customContextLength"| "maxSeqLength"| "kvCacheDtype"- | "mlxKvBits"+ | "mlxKvQuant"| "speculativeType"| "specDraftNMax"| "nParallel"@@ -71,7 +72,7 @@ export const EMPTY_PRESET_LOAD_CONFIG: PresetLoadConfig = {customContextLength: null,maxSeqLength: null,kvCacheDtype: null,- mlxKvBits: null,+ mlxKvQuant: null,speculativeType: null,specDraftNMax: null,nParallel: null,@@ -150,11 +151,7 @@ export function normalizePresetLoadConfig(const normalized: PresetLoadConfig = {customContextLength: requestableContextLength(partial.customContextLength),maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength as number | null),- mlxKvBits:- typeof partial.mlxKvBits === "number" &&- MLX_KV_BITS.includes(partial.mlxKvBits)- ? partial.mlxKvBits- : null,+ mlxKvQuant: normalizeMlxKvQuant(partial.mlxKvQuant, partial.mlxKvBits),kvCacheDtype:typeof partial.kvCacheDtype === "string" &&VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype)@@ -263,7 +260,7 @@ export function capturePresetLoadConfig(): PresetLoadConfig | undefined {customContextLength: effectiveContextLength ?? null,maxSeqLength: isMlx ? null : normalizeMaxSeqLength(snapshot.maxSeqLength),kvCacheDtype: snapshot.kvCacheDtype ?? null,- mlxKvBits: snapshot.mlxKvBits ?? null,+ mlxKvQuant: snapshot.mlxKvQuant ?? null,speculativeType: normalizeSpeculativeType(snapshot.speculativeType),specDraftNMax: snapshot.specDraftNMax ?? null,nParallel: snapshot.nParallel ?? null,@@ -329,7 +326,7 @@ export function applyPresetLoadConfig(config?: PresetLoadConfig | null): void {maxSeqLength: normalizeMaxSeqLength(config.maxSeqLength) ?? DEFAULT_MAX_SEQ_LENGTH,customContextLength: config.customContextLength ?? null,kvCacheDtype: config.kvCacheDtype ?? null,- mlxKvBits: config.mlxKvBits ?? null,+ mlxKvQuant: config.mlxKvQuant ?? null,speculativeType: config.speculativeType ?? null,specDraftNMax: config.specDraftNMax ?? null,nParallel: config.nParallel ?? null,@@ -367,8 +364,8 @@ export function formatPresetLoadConfigSummary(if (config.kvCacheDtype) {parts.push(`KV ${config.kvCacheDtype}`);}- if (config.mlxKvBits) {- parts.push(`MLX KV ${config.mlxKvBits}-bit`);+ if (config.mlxKvQuant) {+ parts.push(`MLX KV ${mlxKvQuantLabel(config.mlxKvQuant)}`);}if (config.speculativeType && config.speculativeType !== "auto") {parts.push(`Spec ${config.speculativeType}`);diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex d672d22ee02..068a6e26737 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -1662,7 +1662,7 @@ export function SharedComposer({approved_remote_code_fingerprint: approvedRemoteCodeFingerprint,chat_template_override: effectiveChatTemplateOverride,cache_type_kv: ownConfig.kvCacheDtype ?? null,- mlx_kv_bits: ownConfig.mlxKvBits ?? null,+ mlx_kv_quant: ownConfig.mlxKvQuant ?? null,speculative_type: effectiveSpeculativeType,spec_draft_n_max: effectiveSpecDraftNMax,reasoning_budget:diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex 9dbaaf27440..e280b5122d3 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -11,7 +11,10 @@ import {pinnedReasoningEffort,useModelReasoningEffortStore,} from "@/features/model-picker/components/model-selector/model-reasoning-effort";-import { loadedContextFields } from "@/features/model-picker/model-config/per-model-config";+import {+ loadedContextFields,+ type MlxKvQuant,+} from "@/features/model-picker/model-config/per-model-config";import {cachedPinnableGpuIndexKind,reconcileCachedGpuSelection,@@ -2394,9 +2397,8 @@ type ChatRuntimeStore = {maxToolCallsPerMessage: number;toolCallTimeout: number;kvCacheDtype: string | null;- mlxKvBits: number | null;- /** Width the backend was last asked for; the verdict belongs beside it. */- loadedMlxKvBitsRequested: number | null;+ mlxKvQuant: MlxKvQuant | null;+ loadedMlxKvQuantRequested: MlxKvQuant | null;mlxKvQuantReason: string | null;chatTemplateOverrideReason: string | null;mlxKvQuantNote: string | null;@@ -4178,8 +4180,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({maxToolCallsPerMessage: 25,toolCallTimeout: 5,kvCacheDtype: null,- mlxKvBits: null,- loadedMlxKvBitsRequested: null,+ mlxKvQuant: null,+ loadedMlxKvQuantRequested: null,mlxKvQuantReason: null,chatTemplateOverrideReason: null,mlxKvQuantNote: null,@@ -5126,8 +5128,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({toolFullOutput: {},activeDiffusionCanvasByThreadId: {},kvCacheDtype: null,- mlxKvBits: null,- loadedMlxKvBitsRequested: null,+ mlxKvQuant: null,+ loadedMlxKvQuantRequested: null,mlxKvQuantReason: null,chatTemplateOverrideReason: null,mlxKvQuantNote: null,diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 6927e7a5c3f..aac56bc45f8 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -66,7 +66,7 @@ export interface LoadModelRequest {approved_remote_code_fingerprint?: string | null;chat_template_override?: string | null;cache_type_kv?: string | null;- mlx_kv_bits?: number | null;+ mlx_kv_quant?: string | null;/** Speculative decoding mode for GGUF models: "auto" (platform-aware DSpark/DFlash when the model* ships that sidecar, else MTP on MTP GGUFs, ngram-mod for sub-3B), "mtp", "dspark",* "dflash", "ngram", "mtp+ngram", "off". The legacy spellings are still accepted. */@@ -264,8 +264,8 @@ export interface LoadModelResponse {preserve_thinking_default?: boolean;supports_tools?: boolean;cache_type_kv?: string | null;- mlx_kv_bits?: number | null;- mlx_kv_bits_requested?: number | null;+ mlx_kv_quant?: string | null;+ mlx_kv_quant_requested?: string | null;mlx_kv_quant_eligibility?: string | null;mlx_kv_quant_reason?: string | null;chat_template_override_reason?: string | null;@@ -377,8 +377,8 @@ export interface InferenceStatusResponse {context_length_enforced?: boolean | null;context_unbounded_when_batched?: boolean;cache_type_kv?: string | null;- mlx_kv_bits?: number | null;- mlx_kv_bits_requested?: number | null;+ mlx_kv_quant?: string | null;+ mlx_kv_quant_requested?: string | null;mlx_kv_quant_eligibility?: string | null;mlx_kv_quant_reason?: string | null;chat_template_override_reason?: string | null;diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.tsindex c240c99d5fe..91b6273c01f 100644--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts@@ -15,6 +15,7 @@ import {} from "../model-config/model-identity";import {DEFAULT_PER_MODEL_CONFIG,+ normalizeMlxKvQuant,type PerModelConfig,deletePerModelConfigsForOverrideKeys,normalizePerModelConfig,@@ -33,6 +34,7 @@ export interface ApiModelOverride {// biome-ignore lint/style/useNamingConvention: API schemakv_cache_dtype?: string;// biome-ignore lint/style/useNamingConvention: API schema+ mlx_kv_quant?: string;mlx_kv_bits?: number;// biome-ignore lint/style/useNamingConvention: API schemaspeculative_type?: string;@@ -309,6 +311,8 @@ export function fromApiOverride(// auto-switch max_seq_length first. So a row stating either field owns both.const serverStatesPin =override.custom_context_length != null || override.max_seq_length != null;+ const serverStatesKvQuant =+ "mlx_kv_quant" in override || "mlx_kv_bits" in override;const normalized = normalizePerModelConfig({...DEFAULT_PER_MODEL_CONFIG,customContextLength: serverStatesPin@@ -318,7 +322,9 @@ export function fromApiOverride(? (override.max_seq_length ?? null): local.maxSeqLength,kvCacheDtype: override.kv_cache_dtype ?? local.kvCacheDtype,- mlxKvBits: override.mlx_kv_bits ?? local.mlxKvBits,+ mlxKvQuant: serverStatesKvQuant+ ? normalizeMlxKvQuant(override.mlx_kv_quant, override.mlx_kv_bits)+ : (local.mlxKvQuant ?? null),speculativeType: override.speculative_type ?? local.speculativeType,specDraftNMax: override.spec_draft_n_max ?? local.specDraftNMax,specDraftCacheDtype:@@ -373,8 +379,8 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {payload.kv_cache_dtype = config.kvCacheDtype;}// Travels beside kv_cache_dtype, or an API auto-switch loads a remembered MLX model at full precision.- if (config.mlxKvBits != null) {- payload.mlx_kv_bits = config.mlxKvBits;+ if (config.mlxKvQuant) {+ payload.mlx_kv_quant = config.mlxKvQuant;}if (config.speculativeType) {payload.speculative_type = config.speculativeType;diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex f14861a1b54..3c8b56d3ca1 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -146,7 +146,9 @@ import {MAX_SEQ_LENGTH_MAX,MAX_SEQ_LENGTH_MIN,MAX_SEQ_LENGTH_STEP,- MLX_KV_BITS,+ MLX_KV_QUANTS,+ mlxKvQuantLabel,+ type MlxKvQuant,N_BATCH_LLAMA_DEFAULT,N_BATCH_MAX,N_BATCH_MIN,@@ -1047,7 +1049,7 @@ function GpuMemorySettings({);}-const MLX_KV_BITS_AUTO = "auto";+const MLX_KV_QUANT_AUTO = "auto";function AdvancedSettingsToggle({checked,@@ -1162,12 +1164,15 @@ function MlxAdvancedSettings({<InfoHint>Lower KV cache precision to save memory, at some cost to quality.Auto keeps full precision; 8-bit is the safest step down.+ TurboQuant holds quality better at the low widths and adds 3.5-bit,+ which uses 3-bit keys beside 4-bit values; sliding-window and+ recurrent layers keep their native cache under it.</InfoHint></div><Select- value={config.mlxKvBits ? String(config.mlxKvBits) : MLX_KV_BITS_AUTO}+ value={config.mlxKvQuant ?? MLX_KV_QUANT_AUTO}onValueChange={(v) =>- update({ mlxKvBits: v === MLX_KV_BITS_AUTO ? null : Number(v) })+ update({ mlxKvQuant: v === MLX_KV_QUANT_AUTO ? null : (v as MlxKvQuant) })}><SelectTrigger@@ -1179,10 +1184,10 @@ function MlxAdvancedSettings({<SelectValue /></SelectTrigger><SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">- <SelectItem value={MLX_KV_BITS_AUTO}>Auto</SelectItem>- {MLX_KV_BITS.map((bits) => (- <SelectItem key={bits} value={String(bits)}>- {bits}-bit+ <SelectItem value={MLX_KV_QUANT_AUTO}>Auto</SelectItem>+ {MLX_KV_QUANTS.map((quant) => (+ <SelectItem key={quant} value={quant}>+ {mlxKvQuantLabel(quant)}</SelectItem>))}</SelectContent>@@ -2014,8 +2019,8 @@ export function ModelConfigPage({useShallow(selectResidentEstimateSettings),);const mlxKvQuantNote = useChatRuntimeStore((s) => s.mlxKvQuantNote);- const loadedMlxKvBitsRequested = useChatRuntimeStore(- (s) => s.loadedMlxKvBitsRequested,+ const loadedMlxKvQuantRequested = useChatRuntimeStore(+ (s) => s.loadedMlxKvQuantRequested,);const isActiveModel = loadedConfig != null;const hfToken = useChatRuntimeStore((s) => s.hfToken);@@ -2156,7 +2161,7 @@ export function ModelConfigPage({: null;const mlxKvQuantOutcome =isActiveModel &&- (configState.mlxKvBits ?? null) === (loadedMlxKvBitsRequested ?? null)+ (configState.mlxKvQuant ?? null) === (loadedMlxKvQuantRequested ?? null)? // Both, not either: dropping the note promises savings before the offset where quantization actually starts.[mlxKvQuantReason, mlxKvQuantNote].filter(Boolean).join(". ") || null: null;@@ -2178,12 +2183,12 @@ export function ModelConfigPage({);// Frozen like the rest of the auto-open decision, so editing the width does not reopen the// section the user just closed.- const [initialMlxKvBits] = useState(() => configState.mlxKvBits ?? null);+ const [initialMlxKvQuant] = useState(() => configState.mlxKvQuant ?? null);// Applicability stays live, unlike the snapshot above: MLX can become available after mount,// and a width that starts applying then has to surface.- const autoOpenForMlxKvBits = servedByMlx && initialMlxKvBits != null;+ const autoOpenForMlxKvQuant = servedByMlx && initialMlxKvQuant != null;const showAdvanced =- advancedPreference ?? (autoOpenAdvanced || autoOpenForMlxKvBits);+ advancedPreference ?? (autoOpenAdvanced || autoOpenForMlxKvQuant);const toggleAdvanced = saveAdvancedSettingsOpen;const contextInputRef = useRef<NumericValueInputHandle>(null);const maxSeqLengthInputRef = useRef<NumericValueInputHandle>(null);diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.tsindex 45cd02c9971..bcd9e36259d 100644--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts@@ -27,7 +27,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {);const customContextLength = useChatRuntimeStore((s) => s.customContextLength);const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);- const mlxKvBits = useChatRuntimeStore((s) => s.mlxKvBits);+ const mlxKvQuant = useChatRuntimeStore((s) => s.mlxKvQuant);const speculativeType = useChatRuntimeStore((s) => s.speculativeType);const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax);const nParallel = useChatRuntimeStore((s) => s.nParallel);@@ -81,7 +81,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {// Off-backend this stays null, or the model compares unequal to its own defaults// over a field it cannot show.- const effectiveMlxKvBits = isMlx ? (mlxKvBits ?? null) : null;+ const effectiveMlxKvQuant = isMlx ? (mlxKvQuant ?? null) : null;const config = useMemo<PerModelConfig | null>(() => {if (!checkpoint || isExternalModelId(checkpoint)) {@@ -94,7 +94,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {// pin every reload to whatever the first load happened to get.maxSeqLength: isGguf || isMlx ? null : maxSeqLength,kvCacheDtype: kvCacheDtype ?? null,- mlxKvBits: effectiveMlxKvBits,+ mlxKvQuant: effectiveMlxKvQuant,speculativeType: speculativeType ?? "auto",specDraftNMax: specDraftNMax ?? null,nParallel: nParallel ?? null,@@ -129,7 +129,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {maxSeqLength,customContextLength,kvCacheDtype,- effectiveMlxKvBits,+ effectiveMlxKvQuant,speculativeType,specDraftNMax,nParallel,diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.tsindex e9ec9af341e..897147131bc 100644--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts@@ -58,7 +58,7 @@ export function applyPerModelConfigToRuntime(: { ids: null, indexKind: null };useChatRuntimeStore.setState({customContextLength: config.customContextLength ?? null,- mlxKvBits: config.mlxKvBits ?? null,+ mlxKvQuant: config.mlxKvQuant ?? null,kvCacheDtype: config.kvCacheDtype ?? null,speculativeType:normalizeSpeculativeType(config.speculativeType) ??@@ -123,7 +123,7 @@ export function currentRuntimePerModelConfig(? normalizeMaxSeqLength(s.params.maxSeqLength): null,kvCacheDtype: s.kvCacheDtype ?? null,- mlxKvBits: s.mlxKvBits ?? null,+ mlxKvQuant: s.mlxKvQuant ?? null,speculativeType: normalizeSpeculativeType(s.speculativeType),specDraftNMax: s.specDraftNMax ?? null,specDraftCacheDtype: s.specDraftCacheDtype ?? null,@@ -164,7 +164,7 @@ export function perModelConfigsEqual(normalizeMaxSeqLength(a.maxSeqLength) ===normalizeMaxSeqLength(b.maxSeqLength) &&(a.kvCacheDtype ?? null) === (b.kvCacheDtype ?? null) &&- (a.mlxKvBits ?? null) === (b.mlxKvBits ?? null) &&+ (a.mlxKvQuant ?? null) === (b.mlxKvQuant ?? null) &&normalizeSpeculativeType(a.speculativeType) ===normalizeSpeculativeType(b.speculativeType) &&(a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) &&diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.tsindex c9506c44bb7..240f291fec9 100644--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts@@ -51,7 +51,7 @@ export function loadedConfigSignature(config.customContextLength ?? "",config.maxSeqLength ?? "",config.kvCacheDtype ?? "",- config.mlxKvBits ?? "",+ config.mlxKvQuant ?? "",config.speculativeType ?? "",config.specDraftNMax ?? "",config.specDraftCacheDtype ?? "",diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex 8b2bbdbb9af..dc1497f0b7f 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -22,8 +22,7 @@ export interface PerModelConfig {customContextLength: number | null;maxSeqLength: number | null;kvCacheDtype: string | null;- /** MLX KV cache quantization width. Optional so older blobs still parse. */- mlxKvBits?: number | null;+ mlxKvQuant?: MlxKvQuant | null;speculativeType: string | null;specDraftNMax: number | null;/** KV cache dtype for the DRAFT context, sized and quantized independently of kvCacheDtype.@@ -65,7 +64,7 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {customContextLength: null,maxSeqLength: null,kvCacheDtype: null,- mlxKvBits: null,+ mlxKvQuant: null,speculativeType: null,specDraftNMax: null,specDraftCacheDtype: null,@@ -280,8 +279,40 @@ export const KV_CACHE_DTYPES = ["f32",] as const;-// Every width mx.quantize supports. By bit width, not a dtype name, hence separate from KV_CACHE_DTYPES.-export const MLX_KV_BITS: readonly number[] = [8, 6, 5, 4, 3, 2];+export const MLX_KV_QUANTS = [+ "8",+ "6",+ "5",+ "4",+ "3",+ "2",+ "tq-4",+ "tq-3.5",+ "tq-3",+ "tq-2",+] as const;+export type MlxKvQuant = (typeof MLX_KV_QUANTS)[number];+const VALID_MLX_KV_QUANTS = new Set<string>(MLX_KV_QUANTS);++export function mlxKvQuantLabel(quant: string): string {+ return quant.startsWith("tq-") ? `TurboQuant ${quant.slice(3)}-bit` : `${quant}-bit`;+}++/** A bare width only ever meant mx.quantize; null is how a saved Auto spells itself. */+export function normalizeMlxKvQuant(+ value: unknown,+ supersededBits?: unknown,+): MlxKvQuant | null {+ if (typeof value === "string") {+ // Trimmed and lower-cased to match the backend's reader, or one row means two settings.+ const named = value.trim().toLowerCase();+ return VALID_MLX_KV_QUANTS.has(named) ? (named as MlxKvQuant) : null;+ }+ if (value !== undefined) return null;+ if (typeof supersededBits !== "number" || !Number.isFinite(supersededBits)) return null;+ const name = String(supersededBits);+ return VALID_MLX_KV_QUANTS.has(name) ? (name as MlxKvQuant) : null;+}const VALID_KV_CACHE_DTYPES = new Set<string>(KV_CACHE_DTYPES);// llama-server's --load-mode enum in --help order. "auto" is the default: the UI shows it, storage keeps null and@@ -322,9 +353,10 @@ const LEGACY_STORAGE_KEY = "unsloth_load_settings";const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";// would normalize the unknown field straight back out of the record.// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group-// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair; a-// client from before any of them-const STORAGE_SCHEMA_VERSION = 6;+// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,+// v7 mlxKvQuant+const STORAGE_SCHEMA_VERSION = 7;+const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;const PRE_VISION_SCHEMA_VERSION = 3;@@ -357,14 +389,14 @@ type StoredPerModelConfig = PerModelConfig & {version: number;};type StoredMap = Record<string, PerModelConfig | StoredPerModelConfig>;-type RawConfig = Partial<PerModelConfig> & { version?: unknown };+type RawConfig = Partial<PerModelConfig> & { version?: unknown; mlxKvBits?: unknown };const STORED_CONFIG_FIELDS = new Set(["version","customContextLength","maxSeqLength","kvCacheDtype",- "mlxKvBits",+ "mlxKvQuant","speculativeType","specDraftNMax","specDraftCacheDtype",@@ -960,11 +992,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {? Math.max(CONTEXT_LENGTH_MIN, Math.floor(partial.customContextLength)): null,maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength),- mlxKvBits:- typeof partial.mlxKvBits === "number" &&- MLX_KV_BITS.includes(partial.mlxKvBits)- ? partial.mlxKvBits- : null,+ mlxKvQuant: normalizeMlxKvQuant(partial.mlxKvQuant, partial.mlxKvBits),kvCacheDtype:typeof partial.kvCacheDtype === "string" &&VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype)@@ -1047,10 +1075,13 @@ function normalize(raw: unknown): PerModelConfig {* client reconstructs anyway, and stamping every record v4 would put the whole store out of reach.* The tuning group and the reasoning pair follow the same rule. */function storedSchemaVersion(normalized: PerModelConfig): number {+ if (normalized.mlxKvQuant != null) {+ return STORAGE_SCHEMA_VERSION;+ }const hasReasoningBudget =normalized.reasoningBudget !== -1 || normalized.reasoningBudgetMessage !== "";if (hasReasoningBudget) {- return STORAGE_SCHEMA_VERSION;+ return PRE_MLX_KV_QUANT_SCHEMA_VERSION;}const hasServerTuning =normalized.loadMode != null ||@@ -1218,7 +1249,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean {config.customContextLength == null &&config.maxSeqLength == null &&(config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype &&- (config.mlxKvBits ?? null) === DEFAULT_PER_MODEL_CONFIG.mlxKvBits &&+ (config.mlxKvQuant ?? null) === DEFAULT_PER_MODEL_CONFIG.mlxKvQuant &&config.speculativeType === DEFAULT_PER_MODEL_CONFIG.speculativeType &&config.specDraftNMax == null &&config.nParallel == null &&diff --git a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts b/studio/frontend/tests/hydration-clears-stale-local-record.test.tsindex 7fb5f9bf571..d03a6709487 100644--- a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts+++ b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts@@ -37,6 +37,18 @@ const {const MODEL = "unsloth/Model-GGUF";const VARIANT = "q4_k_m";+test("a server row holding the superseded cache width hydrates as the single choice", () => {+ assert.equal(fromApiOverride({ mlx_kv_bits: 8 }).mlxKvQuant, "8");+ assert.equal(fromApiOverride({ mlx_kv_quant: "tq-4", mlx_kv_bits: 8 }).mlxKvQuant, "tq-4");+ const local = { ...DEFAULT_PER_MODEL_CONFIG, mlxKvQuant: "tq-3.5" as const };+ assert.equal(fromApiOverride({ mlx_kv_quant: "auto", mlx_kv_bits: 4 }, local).mlxKvQuant, null);+ assert.equal(+ fromApiOverride({ mlx_kv_quant: null } as never, local).mlxKvQuant,+ null,+ );+ assert.equal(fromApiOverride({}, local).mlxKvQuant, "tq-3.5");+});+test("an explicit server clear leaves a config the panel must still persist", () => {// What this browser remembers: one flag, nothing else.const stored = { ...DEFAULT_PER_MODEL_CONFIG, llamaExtraArgs: ["--flash-attn"] };diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.tsindex 1ce40749ce4..96de26b75f3 100644--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts@@ -204,12 +204,12 @@ const ROWS: Row[] = [mlxRequest: 32768,transformersRequest: 32768,note:- "The task called v4 the 'future' record. It is not: STORAGE_SCHEMA_VERSION is 6 " +- "in this tree, not 3, so v4 is a v4-client record and reads normally.",+ "Not a future record: STORAGE_SCHEMA_VERSION is 7, so v4 is a v4-client record " ++ "and reads normally.",},{- name: "version 7 (genuinely future)",- raw: { version: 7, customContextLength: 32768 },+ name: "version 8 (genuinely future)",+ raw: { version: 8, customContextLength: 32768 },normalizedPin: null,rawPin: 32768,isDefault: true,@@ -330,27 +330,34 @@ test("a patched pin round-trips through storage on both backends", () => {test("both pin shapes are stamped version 1, so neither is distinguishable by version", () => {assert.equal(stampedVersion({ customContextLength: 32768 }), 1);assert.equal(stampedVersion({ maxSeqLength: 32768 }), 1);- // The old client's only forwards guard is `version > 5`, and v1 invites any client// back to v1 to rewrite the record.assert.equal(stage({ version: 1, customContextLength: 32768 }).remembered,true,);assert.equal(- stage({ version: 5, customContextLength: 32768 }).remembered,+ stage({ version: 6, customContextLength: 32768 }).remembered,true,);assert.equal(- stage({ version: 6, customContextLength: 32768 }).remembered,+ stage({ version: 7, customContextLength: 32768 }).remembered,true,);assert.equal(- stage({ version: 7, customContextLength: 32768 }).remembered,+ stage({ version: 8, customContextLength: 32768 }).remembered,false,);});+test("a stored mlxKvQuant stamps a version older builds refuse", () => {+ assert.equal(stampedVersion({ mlxKvQuant: "tq-4" }), 7);+ assert.equal(stampedVersion({ mlxKvQuant: "4" }), 7);+ assert.equal(stampedVersion({ loadMode: "mmap" }), 5);+ assert.equal(stampedVersion({ mlxKvQuant: "tq-4", loadMode: "mmap" }), 7);+});++/*** The old model-config page's read rule for a NON-GGUF target, transcribed from* `studio/frontend/src/features/model-picker/components/model-config-page.tsx` on main:@@ -472,3 +479,15 @@ test("BACKWARDS COMPAT: an old record still pins under the new code, on either b// And it is not reported as default, so it survives savePerModelConfig's delete-if-default.assert.equal(isDefaultConfig(config), false);});++test("the fractional TurboQuant choice survives the saved model record", () => {+ store.clear();+ savePerModelConfig(MODEL, null, { ...DEFAULT_PER_MODEL_CONFIG, mlxKvQuant: "tq-3.5" });+ assert.equal(resolveInitialConfig(MODEL, null).config.mlxKvQuant, "tq-3.5");+});++test("a record holding the superseded width reads back as the single choice", () => {+ assert.equal(stage({ mlxKvBits: 8 }).config.mlxKvQuant, "8");+ assert.equal(stage({ mlxKvQuant: " TQ-4 ", mlxKvBits: 8 }).config.mlxKvQuant, "tq-4");+ assert.equal(stage({ mlxKvQuant: null, mlxKvBits: 8 }).config.mlxKvQuant, null);+});diff --git a/studio/frontend/tests/model-config-instance-key.test.ts b/studio/frontend/tests/model-config-instance-key.test.tsindex 06428a738dd..44815cc33cf 100644--- a/studio/frontend/tests/model-config-instance-key.test.ts+++ b/studio/frontend/tests/model-config-instance-key.test.ts@@ -107,7 +107,7 @@ test("every mirrored setting moves the instance key", () => {{ ...LIVE, customContextLength: 8192 },{ ...LIVE, maxSeqLength: 4096 },{ ...LIVE, kvCacheDtype: "f16" },- { ...LIVE, mlxKvBits: 4 },+ { ...LIVE, mlxKvQuant: "4" },{ ...LIVE, speculativeType: "off" },{ ...LIVE, specDraftNMax: 4 },{ ...LIVE, nParallel: 1 },diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.tsindex 68ea65f21d7..9844e99ca1c 100644--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts@@ -63,7 +63,7 @@ const BLANK = {customContextLength: null,maxSeqLength: null,kvCacheDtype: null,- mlxKvBits: null,+ mlxKvQuant: null,speculativeType: null,specDraftNMax: null,nParallel: null,@@ -113,7 +113,7 @@ const ACCELERATORS: Record<string, Record<string, unknown>> = {// default load leaves the width unrequested; a pinned width is swept below like any other// field, and is not part of the base for the same reason placement is not."apple-mlx": {- mlx_kv_bits_requested: null,+ mlx_kv_quant_requested: null,},};@@ -150,10 +150,10 @@ const FIELDS: FieldCase[] = [different: "f16",},{- key: "mlxKvBits",- statusKey: "mlx_kv_bits_requested",- same: 8,- different: 4,+ key: "mlxKvQuant",+ statusKey: "mlx_kv_quant_requested",+ same: "8",+ different: "tq-4",},{key: "speculativeType",diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.tsindex 93d50341747..b5793a6617b 100644--- a/studio/frontend/tests/resident-config-match.test.ts+++ b/studio/frontend/tests/resident-config-match.test.ts@@ -26,7 +26,7 @@ const DEFAULT_ISH = {customContextLength: null,maxSeqLength: null,kvCacheDtype: null,- mlxKvBits: null,+ mlxKvQuant: null,speculativeType: null,specDraftNMax: null,nParallel: null,@@ -43,7 +43,7 @@ const BLANK = {customContextLength: null,maxSeqLength: null,kvCacheDtype: null,- mlxKvBits: null,+ mlxKvQuant: null,speculativeType: null,specDraftNMax: null,nParallel: null,@@ -143,10 +143,11 @@ const FIELDS: {differs: { cache_type_kv: "f16" },},{- name: "MLX KV bits",- config: { mlxKvBits: 4 },- same: { mlx_kv_bits_requested: 4 },- differs: { mlx_kv_bits_requested: 8 },+ name: "MLX KV quantization",+ config: { mlxKvQuant: "4" },+ same: { mlx_kv_quant_requested: "4" },+ differs: { mlx_kv_quant_requested: "tq-4" },+},{name: "speculative mode",@@ -245,6 +246,11 @@ for (const field of FIELDS) {});}+test("Auto adopts a resident model the backend reported as Auto", () => {+ assert.equal(matches({ mlx_kv_quant_requested: "auto" }, { ...BLANK, mlxKvQuant: null }), true);+ assert.equal(matches({ mlx_kv_quant_requested: "8" }, { ...BLANK, mlxKvQuant: null }), false);+});+/** Ordering is the backend's to choose: it narrows and reorders placement at fit time. */test("GPU placement compares as an order, not as a set", () => {// The picker hands the list to the backend in order and position decides which@@ -739,7 +745,7 @@ test("a non-GGUF resident is not judged on a GGUF invocation field", () => {true,);assert.equal(- matches({ ...DEFAULTS, is_gguf: false, mlx_kv_bits_requested: 4 }, BLANK),+ matches({ ...DEFAULTS, is_gguf: false, mlx_kv_quant_requested: "4" }, BLANK),false,);assert.equal(@@ -1444,7 +1450,7 @@ test("unset nullable settings ask for the default, not for the resident value",...DEFAULTS,requested_context_length: 8192,cache_type_kv: "q8_0",- mlx_kv_bits_requested: 4,+ mlx_kv_quant_requested: "4",requested_parallel_slots: 4,requested_n_batch: 2048,requested_n_ubatch: 512,@@ -1455,7 +1461,7 @@ test("unset nullable settings ask for the default, not for the resident value",for (const [key, value] of Object.entries({requested_context_length: 8192,cache_type_kv: "q8_0",- mlx_kv_bits_requested: 4,+ mlx_kv_quant_requested: "4",requested_parallel_slots: 4,requested_n_batch: 2048,requested_n_ubatch: 512,diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.tsindex 127ec617676..eac1cfdc97b 100644--- a/studio/frontend/tests/server-tuning-settings.test.ts+++ b/studio/frontend/tests/server-tuning-settings.test.ts@@ -113,10 +113,15 @@ test("a record only claims the new schema version when it carries one", () => {// toStoredConfig stamps the OLDEST version that understands every field// present, so an older client can still rewrite a record it fully knows.const source = readSrc("features/model-picker/model-config/per-model-config.ts");- assert.match(source, /const STORAGE_SCHEMA_VERSION = 6;/);+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 7;/);+ assert.match(source, /const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;/);assert.match(source, /const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;/);assert.match(source, /const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;/);- assert.match(source, /hasReasoningBudget\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);+ assert.match(source, /mlxKvQuant != null\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);+ assert.match(+ source,+ /hasReasoningBudget\)\s*\{\s*return PRE_MLX_KV_QUANT_SCHEMA_VERSION;/,+ );assert.match(source,/hasServerTuning\)\s*\{\s*return PRE_REASONING_BUDGET_SCHEMA_VERSION;/,diff --git a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.tsindex e93d86e9d0b..3c451ee82ef 100644--- a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts+++ b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts@@ -356,16 +356,16 @@ test("the backfill offers non-GGUF weights, keyed by repo id, after the v2 passconst snapshot = "/hf/models--mlx-community--Model-4bit/snapshots/abc";const template = { chatTemplateOverride: "{{ messages }}" };assert.ok(savePerModelConfig(folder, null, config(template)));- assert.ok(savePerModelConfig(snapshot, null, config({ mlxKvBits: 8 })));+ assert.ok(savePerModelConfig(snapshot, null, config({ mlxKvQuant: "8" })));// A newer build's record under the repo id makes adoption decline, and the path stays put.const declined = "/hf/models--org--Newer/snapshots/def";- assert.ok(savePerModelConfig("org/Newer", null, config({ mlxKvBits: 4 })));+ assert.ok(savePerModelConfig("org/Newer", null, config({ mlxKvQuant: "4" })));const map = readMap();const newer = Object.keys(map).find((key) => key.includes("org/newer"));assert.ok(newer);map[newer].version = 99;writeMap(map);- assert.ok(savePerModelConfig(declined, null, config({ mlxKvBits: 4 })));+ assert.ok(savePerModelConfig(declined, null, config({ mlxKvQuant: "4" })));// Identities the resolver never keys.const link = "/home/u/.ollama/.studio_links/ab12/model-latest.gguf";assert.ok(savePerModelConfig(link, null, config(template)));@@ -387,7 +387,7 @@ test("the backfill offers non-GGUF weights, keyed by repo id, after the v2 passnew Set(puts),new Set([folder, "mlx-community/model-4bit"]),);- assert.equal(resolveInitialConfig(declined, null).config.mlxKvBits, 4);+ assert.equal(resolveInitialConfig(declined, null).config.mlxKvQuant, "4");assert.equal(store.get(BACKFILL_FLAG), "1");});diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.pyindex fa1d8eae543..8c7ed9c3c8a 100644--- a/tests/studio/test_chat_autoload_failure_gate.py+++ b/tests/studio/test_chat_autoload_failure_gate.py@@ -353,20 +353,20 @@ let STORE: any = makeStore();// rather than complete-by-luck.// Imported by chat-adapter.ts from ../lib/mlx-runtime-state. Mirrors the real// one rather than returning {}: a non-MLX response retires the verdict but-// leaves mlxKvBits absent, and a scenario that saw an empty object would keep+// leaves mlxKvQuant absent, and a scenario that saw an empty object would keep// a stale width alive and pass for the wrong reason.function mlxRuntimeStateFrom(resp: any) {if (resp?.is_mlx !== true) {return {- loadedMlxKvBitsRequested: null,+ loadedMlxKvQuantRequested: null,mlxKvQuantReason: null,chatTemplateOverrideReason: null,mlxKvQuantNote: null,};}return {- mlxKvBits: resp.mlx_kv_bits_requested ?? null,- loadedMlxKvBitsRequested: resp.mlx_kv_bits_requested ?? null,+ mlxKvQuant: resp.mlx_kv_quant_requested ?? null,+ loadedMlxKvQuantRequested: resp.mlx_kv_quant_requested ?? null,mlxKvQuantReason: resp.mlx_kv_quant_reason ?? null,chatTemplateOverrideReason: resp.chat_template_override_reason ?? null,mlxKvQuantNote: resp.mlx_kv_quant_note ?? null,diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.pyindex f4e1b5edb0d..5d8d780d328 100644--- a/tests/studio/test_cli_attach_load_knobs.py+++ b/tests/studio/test_cli_attach_load_knobs.py@@ -926,6 +926,28 @@ def test_the_requested_mlx_kv_width_survives_a_reload(monkeypatch):assert server.loads[0]["mlx_kv_bits"] == 4+def test_a_turboquant_resident_reloads_as_turboquant(monkeypatch):+ """tq-4 and mx.quantize 4 share a width, so only mlx_kv_quant keeps the scheme."""+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "load_in_4bit": True,+ "mlx_kv_quant": "tq-4",+ "mlx_kv_quant_requested": "tq-4",+ "mlx_kv_bits": 4,+ "mlx_kv_bits_requested": 4,+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert server.loads[0]["mlx_kv_quant"] == "tq-4"++def test_a_proven_no_op_skips_the_preload_gate(monkeypatch):"""A proven no-op skips the gate."""server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.pyindex 6eeaf620609..0e4764413a1 100644--- a/unsloth_cli/commands/start.py+++ b/unsloth_cli/commands/start.py@@ -1789,6 +1789,8 @@ _RESIDENT_RUNTIME_FIELDS = {"speculative_type": "speculative_type","spec_draft_n_max": "spec_draft_n_max",# The applied value is null when the runtime refused the request.+ # Scheme and width together; a bare width reads back as mx.quantize, so TurboQuant would reload as it. The width stays for servers without mlx_kv_quant.+ "mlx_kv_quant_requested": "mlx_kv_quant","mlx_kv_bits_requested": "mlx_kv_bits",# LoadRequest defaults this to True, so omitting it would reload a full-precision model in 4-bit. Null on GGUF, which has no such setting."load_in_4bit": "load_in_4bit",