Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: TurboQuant KV cache option for MLX inference (#11170)

Long Yixingeadcab66e1de
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 851e14b36f1..63b2c828a6b 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -106,6 +106,12 @@ def _is_opaque_cache(value):
return not hasattr(value, "__dict__") and hasattr(type(value), "state")
+def _rebuilt_tuple(original, items):
+ """Rebuild as the tuple's own type: TurboQuant dispatches on its state class."""
+ rebuild = getattr(type(original), "_make", None)
+ return rebuild(items) if rebuild is not None else tuple(items)
+
+
def _copy_value(value, mx):
if isinstance(value, mx.array):
# Safe to share: MLX writes in place only into an unreferenced buffer; slices are copied out.
@@ -113,7 +119,7 @@ def _copy_value(value, mx):
if isinstance(value, list):
return [_copy_value(item, mx) for item in value]
if isinstance(value, tuple):
- return tuple(_copy_value(item, mx) for item in value)
+ return _rebuilt_tuple(value, (_copy_value(item, mx) for item in value))
if isinstance(value, dict):
return {key: _copy_value(item, mx) for key, item in value.items()}
if _is_cache(value):
@@ -146,7 +152,7 @@ def _release_value(value, mx):
if isinstance(value, list):
value[:] = [_release_value(item, mx) for item in value]
elif isinstance(value, tuple):
- return tuple(_release_value(item, mx) for item in value)
+ return _rebuilt_tuple(value, (_release_value(item, mx) for item in value))
elif isinstance(value, dict):
for key, item in list(value.items()):
value[key] = _release_value(item, mx)
@@ -1586,6 +1592,7 @@ PROMPT_CACHE_ENTRIES = 6
# Every bit width mx.quantize supports; unrelated to llama.cpp's cache_type_kv names.
MLX_KV_BITS_CHOICES = (8, 6, 5, 4, 3, 2)
+MLX_TURBOQUANT_BITS_CHOICES = (4, 3.5, 3, 2)
# Quantization group size; a head dim that is not a multiple makes mx.quantize raise.
MLX_KV_GROUP_SIZE = 64
# Surfaced with the resolved setting so an API client sees the reuse cost too.
@@ -1593,6 +1600,63 @@ MLX_KV_QUANT_NO_REUSE = (
"The installed mlx-lm cannot measure a quantized cache entry, so prompt-cache "
"reuse across turns is disabled while this is on."
)
+MLX_TURBOQUANT_TEXT_LOAD = (
+ "TurboQuant is not available for this model: it could not be loaded through the runtime "
+ "TurboQuant requires. The model is served without it."
+)
+
+
+def _turboquant_refusal(*, is_vision, is_distributed, is_lora):
+ """Why a text model cannot take the mlx-vlm route TurboQuant lives on, else ""."""
+ if is_vision:
+ return ""
+ if is_distributed:
+ return "TurboQuant is not available under distributed inference. Run on one device, or turn it off."
+ if is_lora:
+ return "TurboQuant is not available for a LoRA adapter on a text model. Merge it first, or turn TurboQuant off."
+ return ""
+
+
+MLX_KV_QUANT_AUTO = "auto"
+# TurboQuant's widths repeat mx.quantize's, so the scheme rides in the value: "4" alone is ambiguous.
+MLX_TURBOQUANT_PREFIX = "tq-"
+
+
+def _kv_quant_choice(bits):
+ return str(int(bits)) if float(bits).is_integer() else str(float(bits))
+
+
+MLX_KV_QUANT_CHOICES = (
+ (MLX_KV_QUANT_AUTO,)
+ + tuple(_kv_quant_choice(bits) for bits in MLX_KV_BITS_CHOICES)
+ + tuple(MLX_TURBOQUANT_PREFIX + _kv_quant_choice(bits) for bits in MLX_TURBOQUANT_BITS_CHOICES)
+)
+
+
+def parse_mlx_kv_quant(value):
+ """The (bit width, TurboQuant) pair for a choice. Total: an unknown one reads as auto."""
+
+ if value is None:
+ return None, False
+ text = str(value).strip().lower()
+ if text in ("", MLX_KV_QUANT_AUTO):
+ return None, False
+ if text not in MLX_KV_QUANT_CHOICES:
+ logger.warning(
+ "MLX kv quantization %r unsupported (choose %s); ignoring",
+ value,
+ " or ".join(MLX_KV_QUANT_CHOICES),
+ )
+ return None, False
+ turboquant = text.startswith(MLX_TURBOQUANT_PREFIX)
+ width = float(text.removeprefix(MLX_TURBOQUANT_PREFIX))
+ return (int(width) if width.is_integer() else width), turboquant
+
+
+def encode_mlx_kv_quant(bits, turboquant = False):
+ if bits is None:
+ return MLX_KV_QUANT_AUTO
+ return (MLX_TURBOQUANT_PREFIX if turboquant else "") + _kv_quant_choice(bits)
def _kv_entry_nbytes(entry):
@@ -2222,6 +2286,97 @@ def _kv_window_enforced(model, is_vlm, window):
return None
+def _turboquant_reserved_positions(count, bits):
+ """Entry positions mlx-vlm reserves, learned from a control cache rather than a fixed number."""
+
+ from mlx_vlm.generate.common import maybe_quantize_kv_cache
+ from mlx_vlm.models.cache import KVCache
+ from mlx_vlm.turboquant import TurboQuantKVCache
+
+ control = [KVCache() for _ in range(count)]
+ maybe_quantize_kv_cache(control, 0, MLX_KV_GROUP_SIZE, bits, kv_quant_scheme = "turboquant")
+ return {
+ index for index, entry in enumerate(control) if not isinstance(entry, TurboQuantKVCache)
+ }
+
+
+def _turboquant_status(
+ model,
+ bits,
+ refusal = "",
+):
+ import mlx.core as mx
+
+ status = dict(requested_kv_bits = bits, kv_bits = None, eligibility = "none", reason = "", note = "")
+ if refusal:
+ status["eligibility"] = "refused"
+ status["reason"] = refusal
+ logger.info("MLX TurboQuant not applied: %s", refusal)
+ return status
+
+ try:
+ from mlx_vlm.generate.common import maybe_quantize_kv_cache
+ except ImportError:
+ # TurboQuant predates generate.common (mlx-vlm 0.6.0).
+ status["eligibility"] = "refused"
+ status["reason"] = "TurboQuant requires a newer mlx-vlm. Update the MLX packages and retry."
+ return status
+ from mlx_vlm.models.cache import make_prompt_cache
+ from mlx_vlm.turboquant import TurboQuantKVCache
+
+ language_model = getattr(model, "language_model", model)
+ try:
+ entries = make_prompt_cache(language_model)
+ except Exception as exc:
+ if is_metal_queue_dead(exc):
+ raise
+ logger.warning("MLX TurboQuant eligibility probe failed: %s", exc)
+ status["reason"] = "this model's KV cache layout could not be inspected"
+ return status
+ if not entries:
+ status["reason"] = "this model builds no KV cache to quantize"
+ return status
+
+ rng_key = _mlx_rng_key_words()
+ try:
+ reserved = _turboquant_reserved_positions(len(entries), bits)
+ maybe_quantize_kv_cache(entries, 0, MLX_KV_GROUP_SIZE, bits, kv_quant_scheme = "turboquant")
+ converted = declined = 0
+ for index, entry in enumerate(entries):
+ leaves = list(_flatten_kv_entries([entry]))
+ quantized = sum(isinstance(leaf, TurboQuantKVCache) for leaf in leaves)
+ converted += quantized
+ if index not in reserved:
+ declined += len(leaves) - quantized
+ if not converted:
+ status["reason"] = (
+ "This model's cache keeps its native format; no layers can use TurboQuant."
+ )
+ return status
+ for _ in range(2):
+ result = language_model(mx.array([[0]]), cache = entries)
+ mx.eval(getattr(result, "logits", result), [entry.state for entry in entries])
+ status["kv_bits"] = bits
+ status["eligibility"] = "partial" if declined else "full"
+ if declined:
+ status["note"] = "Some layers keep their native cache format."
+ return status
+ except Exception as exc:
+ if is_metal_queue_dead(exc):
+ raise
+ logger.warning("MLX TurboQuant eligibility probe failed: %s", exc)
+ status["eligibility"] = "refused"
+ status["reason"] = f"this model's KV cache cannot use TurboQuant ({type(exc).__name__})"
+ return status
+ finally:
+ _restore_mlx_rng_key(rng_key)
+ # After the probe's own locals are gone, or the pages stay in the allocator.
+ entries.clear()
+ del entries
+ _drain_generation_streams(mx)
+ mx.clear_cache()
+
+
def _kv_quant_status(requested_bits, model, is_vlm):
"""Resolve a requested bit width against this model into a status dict."""
status = {
@@ -3383,6 +3538,8 @@ class MLXInferenceBackend:
self._tokenizer = None
self._processor = None
self._is_vlm = False
+ self._reads_vision = None
+ self._turboquant_refusal = ""
self._multi_image_marker = None
self._config = {}
self._distributed_group = None
@@ -3642,11 +3799,21 @@ class MLXInferenceBackend:
def _prepare_kv_entries(self, entries):
bits = self._kv_quant_bits()
- return entries if bits is None else _quantize_kv_entries(entries, bits)
+ # TurboQuant converts in mlx-vlm: mx.quantize cannot express its fractional widths.
+ if bits is None or getattr(self, "_turboquant", False):
+ return entries
+ return _quantize_kv_entries(entries, bits)
+
+ def _kv_runtime_quant_kwargs(self):
+ """TurboQuant's width for mlx-vlm's converter; start 0, since its default leaves 5000 tokens unquantized."""
+ bits = self._kv_quant_bits()
+ if bits is None or not getattr(self, "_turboquant", False):
+ return {}
+ return {"kv_bits": bits, "kv_quant_scheme": "turboquant", "quantized_kv_start": 0}
def _kv_quant_generate_kwargs(self):
"""A pre-quantized cache, not kv_bits: both runtimes would convert a rotating entry."""
- if self._kv_quant_bits() is None:
+ if self._kv_quant_bits() is None or getattr(self, "_turboquant", False):
return {}
return {
"prompt_cache": self._prepare_kv_entries(_make_kv_entries(self._model, self._is_vlm))
@@ -3696,6 +3863,13 @@ class MLXInferenceBackend:
)
)
+ def _reads_vision_input(self):
+ """Whether this model reads images, which mlx-vlm serving it no longer implies."""
+ reads = getattr(self, "_reads_vision", None)
+ if reads is None:
+ return bool(getattr(self, "_is_vlm", False))
+ return bool(reads)
+
def _count_media_prompt_tokens(
self,
prompt,
@@ -3745,7 +3919,7 @@ class MLXInferenceBackend:
videos = None,
):
"""Prompt length as generation tokenizes it; vision models follow mlx_vlm's marker rule."""
- if not self._is_vlm:
+ if not self._reads_vision_input():
return len(self._encode_prompt(prompt))
expanded = self._count_media_prompt_tokens(prompt, images, audio, videos)
if expanded is not None:
@@ -3832,11 +4006,12 @@ class MLXInferenceBackend:
# two stay apart so a client can tell them apart.
confirmed = self._kv_cache_window_enforceable(served)
enforceable = confirmed is True
- quant = _kv_quant_status(
- _normalize_mlx_kv_bits(kv_bits),
- self._model,
- is_vlm,
- )
+ if getattr(self, "_turboquant", False) and kv_bits is not None:
+ quant = _turboquant_status(
+ self._model, kv_bits, getattr(self, "_turboquant_refusal", "")
+ )
+ else:
+ quant = _kv_quant_status(_normalize_mlx_kv_bits(kv_bits), self._model, is_vlm)
if quant["kv_bits"] is None and enforceable:
logger.info("MLX KV cache limited to %d tokens", int(served))
return quant, int(served), True, None
@@ -3857,7 +4032,7 @@ class MLXInferenceBackend:
dtype = None,
parallel_mode = None,
distributed_group = None,
- kv_bits = None,
+ kv_quant = None,
chat_template_override = None,
) -> bool:
import mlx.core as mx
@@ -3866,8 +4041,24 @@ class MLXInferenceBackend:
self._hf_token = hf_token
model_name = config.identifier if hasattr(config, "identifier") else str(config)
is_vision = getattr(config, "is_vision", False)
+ kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)
+ self._turboquant = turboquant
+ if self._turboquant:
+ try:
+ from mlx_vlm.turboquant import TurboQuantKVCache
+ except ImportError as exc:
+ raise RuntimeError(
+ "TurboQuant requires a newer mlx-vlm. Update the MLX packages and retry."
+ ) from exc
distributed_rank, distributed_size = _mlx_distributed_rank_size(distributed_group)
is_distributed = distributed_group is not None and distributed_size > 1
+ is_lora = getattr(config, "is_lora", False)
+ self._turboquant_refusal = (
+ _turboquant_refusal(is_vision = is_vision, is_distributed = is_distributed, is_lora = is_lora)
+ if self._turboquant
+ else ""
+ )
+ use_vlm = is_vision or (self._turboquant and not self._turboquant_refusal)
self._distributed_group = distributed_group
self._distributed_rank = distributed_rank
self._distributed_world_size = distributed_size
@@ -3890,12 +4081,10 @@ class MLXInferenceBackend:
os.environ["HF_TOKEN"] = hf_token
self._configure_memory_limits()
- is_lora = getattr(config, "is_lora", False)
-
logger.info(
"Loading %s via %s (is_lora=%s, distributed=%s, rank=%s/%s, mode=%s)",
model_name,
- "mlx-vlm" if is_vision else "mlx-lm",
+ "mlx-vlm" if use_vlm else "mlx-lm",
is_lora,
is_distributed,
distributed_rank,
@@ -3928,7 +4117,7 @@ class MLXInferenceBackend:
"load_in_4bit": load_in_4bit,
"token": hf_token,
"trust_remote_code": trust_remote_code,
- "text_only": False if is_vision else True,
+ "text_only": not use_vlm,
}
if is_distributed:
if parallel_mode == "pipeline":
@@ -3939,12 +4128,37 @@ class MLXInferenceBackend:
# Freed before the replacement weights are allocated, for headroom.
self._model_fusion.close()
self._clear_prompt_cache()
- model, tokenizer_or_processor = FastMLXModel.from_pretrained(
- model_name,
- **load_kwargs,
- )
+ model = None
+ try:
+ model, tokenizer_or_processor = FastMLXModel.from_pretrained(
+ model_name,
+ **load_kwargs,
+ )
+ except Exception as exc:
+ if is_vision or not use_vlm or is_metal_queue_dead(exc):
+ raise
+ logger.warning(
+ "TurboQuant load of %s through mlx-vlm failed (%s); serving through mlx-lm without it",
+ model_name,
+ exc,
+ )
+ if model is None:
+ import gc
+
+ # Outside the except: its traceback keeps the failed load's weights alive.
+ gc.collect()
+ _drain_generation_streams(mx)
+ mx.clear_cache()
+ self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD
+ use_vlm = False
+ load_kwargs["text_only"] = True
+ model, tokenizer_or_processor = FastMLXModel.from_pretrained(
+ model_name,
+ **load_kwargs,
+ )
- if is_vision:
+ self._reads_vision = bool(is_vision)
+ if use_vlm:
processor = tokenizer_or_processor
self._model = model
self._processor = processor
@@ -3973,7 +4187,7 @@ class MLXInferenceBackend:
self._kv_cache_window,
_ctx_enforced,
self._kv_context_budget,
- ) = self._resolve_kv_policy(is_vision, kv_bits, max_seq_length, _served_ctx)
+ ) = self._resolve_kv_policy(use_vlm, kv_bits, max_seq_length, _served_ctx)
if self._kv_quant["kv_bits"] is not None:
logger.info(
"MLX KV cache quantization: %s-bit (%s eligibility)",
@@ -3996,7 +4210,7 @@ class MLXInferenceBackend:
chat_template_override,
self._tokenizer,
self._processor,
- lambda: self._render_template_probe(is_vision),
+ lambda: self._render_template_probe(use_vlm),
)
if native_marks_audio:
_revoke_override_that_drops_audio(self._template_override, self._processor, self._model)
@@ -4057,6 +4271,10 @@ class MLXInferenceBackend:
"mlx_context_budget": self._kv_context_budget,
"mlx_kv_bits": self._kv_quant["kv_bits"],
"mlx_kv_bits_requested": self._kv_quant["requested_kv_bits"],
+ "mlx_kv_quant": encode_mlx_kv_quant(self._kv_quant["kv_bits"], self._turboquant),
+ "mlx_kv_quant_requested": encode_mlx_kv_quant(
+ self._kv_quant["requested_kv_bits"], self._turboquant
+ ),
"mlx_kv_quant_eligibility": self._kv_quant["eligibility"],
"mlx_kv_quant_reason": self._kv_quant["reason"],
"mlx_kv_quant_note": self._kv_quant["note"],
@@ -4131,7 +4349,7 @@ class MLXInferenceBackend:
)
if isinstance(_proc_tpl, (str, dict, list, tuple)) and _proc_tpl:
info["processor_template"] = _proc_tpl
- info["renders_image"] = _proc is not None and bool(getattr(self, "_is_vlm", False))
+ info["renders_image"] = _proc is not None and self._reads_vision_input()
info["accepts_multiple_images"] = info["renders_image"] and bool(
getattr(self, "_multi_image_marker", None)
)
@@ -4286,10 +4504,8 @@ class MLXInferenceBackend:
full_messages = self._with_system_prompt(messages, system_prompt)
if self._is_vlm:
- # Through the processor, which is what a vision generation renders with; the text renderer would not
- # recover the template failures it recovers from. images=None: an image anywhere in the conversation makes
- # the structured-item check raise, and the caller declines rather than pricing a prompt without it.
- prompt, _ = self._render_vlm_prompt(
+ # images=None: an image anywhere makes the structured-item check raise.
+ prompt, _target, _markers = self._render_vlm_prompt(
full_messages,
None,
tools = tools,
@@ -4346,7 +4562,7 @@ class MLXInferenceBackend:
if video is not None:
if not _mlx_vlm_decodes_video():
raise RuntimeError("The installed mlx-vlm does not read video.")
- if not self._is_vlm:
+ if not self._reads_vision_input():
raise RuntimeError("The loaded model does not read video.")
# Reset so a failed run cannot surface stale stats.
@@ -4357,7 +4573,7 @@ class MLXInferenceBackend:
# History first, the attachment last: MLX binds pixels to markers in order.
attached = list(images or []) + ([image] if image is not None else [])
# Shared with the transformers vision path so both render the same turns (#10092).
- if self._is_vlm and (attached or video is not None):
+ if self._reads_vision_input() and (attached or video is not None):
# Processor templates want part lists, the tokenizer fallback wants strings.
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
@@ -4405,7 +4621,7 @@ class MLXInferenceBackend:
if self._is_vlm:
stream = self._generate_vlm(
full_messages,
- attached,
+ attached if self._reads_vision_input() else None,
temperature,
top_p,
top_k,
@@ -5004,7 +5220,31 @@ class MLXInferenceBackend:
raise RuntimeError(
f"VLM chat template marked {marked} image(s) for {attached_images} attached."
)
- return prompt, chat_target
+
+ markers = detect_reasoning_channel_markers(chat_target, tools = tools)
+ if not self._reads_vision_input():
+ # No images means no image tokens to place, so skipping this would drop the tool schema.
+ from core.inference.chat_template_helpers import (
+ render_with_native_template_fallback,
+ )
+
+ model_info = self.models.get(self.active_model_name, {})
+ recovered = render_with_native_template_fallback(
+ formatted_prompt = prompt,
+ tokenizer = self._tokenizer,
+ model_info = model_info,
+ active_model_name = self.active_model_name,
+ messages = messages,
+ tools = tools,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ continue_final_message = continue_final_message,
+ hf_token = model_info.get("hf_token"),
+ return_metadata = True,
+ )
+ prompt, markers = recovered.prompt, recovered.reasoning_channel_markers
+ return prompt, chat_target, markers
def _plan_vlm_request(self, request) -> "_VLMRowPlan":
image = request.get("image")
@@ -5037,7 +5277,7 @@ class MLXInferenceBackend:
stop = None,
) -> "_VLMRowPlan":
images = [image] if image is not None else None
- prompt, chat_target = self._render_vlm_prompt(
+ prompt, chat_target, markers = self._render_vlm_prompt(
messages,
images,
tools = tools,
@@ -5052,7 +5292,6 @@ class MLXInferenceBackend:
think_prefix = detect_think_prefill(
prompt, getattr(chat_target, "all_special_tokens", None)
)
- markers = detect_reasoning_channel_markers(chat_target, tools = tools)
normalizer = (
make_reasoning_normalizer(
markers,
@@ -5149,7 +5388,7 @@ class MLXInferenceBackend:
from mlx_vlm import stream_generate as vlm_stream
images = list(images) if images else None
- prompt, chat_target = self._render_vlm_prompt(
+ prompt, chat_target, vlm_reasoning_markers = self._render_vlm_prompt(
messages,
images,
videos = [video] if video is not None else None,
@@ -5163,7 +5402,6 @@ class MLXInferenceBackend:
from core.inference.chat_template_helpers import detect_think_prefill
# Detected once: the decoder keeps the delimiters the normalizer below consumes.
- vlm_reasoning_markers = detect_reasoning_channel_markers(chat_target, tools = tools)
prefill = detect_think_prefill(
prompt,
getattr(chat_target, "all_special_tokens", None),
@@ -5299,9 +5537,15 @@ class MLXInferenceBackend:
else None
)
+ if not self._reads_vision_input():
+ # mlx-vlm adds special tokens by family, doubling a BOS the template already emitted.
+ import mlx.core as mx
+ vlm_kwargs["input_ids"] = mx.array([self._encode_prompt(prompt)])
+
session = self._vlm_prompt_cache_session(
_adapter_state, images, prompt, has_video = video is not None
)
+ vlm_kwargs.update(self._kv_runtime_quant_kwargs())
if session is not None:
vlm_kwargs["prompt_cache"] = session.cache
vlm_kwargs["prompt_cache_state"] = session
@@ -5830,6 +6074,7 @@ class MLXInferenceBackend:
max_tokens = max_new_tokens,
# Greedy; the knobs below are load-time state, not caller kwargs.
temperature = 0.0,
+ **self._kv_runtime_quant_kwargs(),
**self._kv_quant_generate_kwargs(),
**self._kv_window_generate_kwargs(),
)
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 71fcf282611..9137f128cef 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -244,6 +244,8 @@ def _audio_generation_timeout(
_MLX_RUNTIME_MIRROR_FIELDS = (
"mlx_kv_bits",
"mlx_kv_bits_requested",
+ "mlx_kv_quant",
+ "mlx_kv_quant_requested",
"mlx_kv_quant_eligibility",
"mlx_kv_quant_reason",
"mlx_kv_quant_note",
@@ -1955,7 +1957,7 @@ class InferenceOrchestrator:
subject: Optional[str] = None,
tensor_parallel: bool = False,
mlx_distributed: bool = False,
- mlx_kv_bits: Optional[int] = None,
+ mlx_kv_quant: Optional[str] = None,
chat_template_override: Optional[str] = None,
load_cancel_event: Optional[threading.Event] = None,
post_handoff_expected_free_gb: Optional[dict[int, float]] = None,
@@ -1999,7 +2001,7 @@ class InferenceOrchestrator:
"mlx_parallel_mode": ("tensor" if tensor_parallel else "pipeline")
if mlx_distributed
else None,
- "mlx_kv_bits": mlx_kv_bits,
+ "mlx_kv_quant": mlx_kv_quant,
"chat_template_override": chat_template_override,
# Read in the worker, which hides the accelerators before detection.
"audio_device": audio_device,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 705c54e631c..4c67f8433bf 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -616,7 +616,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
if getattr(backend, "device", None) == "mlx":
load_kwargs["parallel_mode"] = config.get("mlx_parallel_mode")
load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")
- load_kwargs["kv_bits"] = config.get("mlx_kv_bits")
+ load_kwargs["kv_quant"] = config.get("mlx_kv_quant")
load_kwargs["chat_template_override"] = config.get("chat_template_override")
success = backend.load_model(**load_kwargs)
finally:
@@ -675,6 +675,8 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"context_unbounded_when_batched",
"mlx_kv_bits",
"mlx_kv_bits_requested",
+ "mlx_kv_quant",
+ "mlx_kv_quant_requested",
"mlx_kv_quant_eligibility",
"mlx_kv_quant_reason",
"mlx_kv_quant_note",
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 2f37b0369e0..53123838982 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -126,16 +126,29 @@ class LoadRequest(BaseModel):
"(e.g. 'f16', 'bf16', 'q8_0', 'q4_0', 'q4_1', 'q5_0', 'q5_1', 'iq4_nl', 'f32')"
),
)
- mlx_kv_bits: Optional[int] = Field(
+ mlx_kv_quant: Optional[str] = Field(
None,
description = (
- "MLX KV cache quantization bit width (8, 6, 5, 4, 3 or 2). MLX takes a bit "
- "width rather than a llama.cpp dtype name, so this is separate from "
- "cache_type_kv. Omit for an unquantized cache. Ignored by non-MLX "
- "backends; a model whose cache layout cannot be quantized reports "
- "the reason instead of applying it."
+ "MLX KV cache quantization: 'auto' for an unquantized cache, '8'/'6'/'5'/'4'/'3'/'2' "
+ "for an mx.quantize width, or 'tq-4'/'tq-3.5'/'tq-3'/'tq-2' for TurboQuant. MLX names a "
+ "width rather than a llama.cpp dtype, so this is separate from cache_type_kv. Ignored "
+ "by non-MLX backends; a model whose cache layout cannot be quantized reports the reason "
+ "instead of applying it."
),
)
+ mlx_kv_bits: Optional[float] = Field(
+ None, description = "Superseded by mlx_kv_quant; read only when mlx_kv_quant is omitted."
+ )
+
+ @model_validator(mode = "after")
+ def derive_mlx_kv_quant(self):
+ """A bare width only ever meant mx.quantize; null is how a client spells Auto."""
+
+ if "mlx_kv_quant" not in self.model_fields_set and self.mlx_kv_bits is not None:
+ from core.inference.mlx_inference import encode_mlx_kv_quant
+ self.mlx_kv_quant = encode_mlx_kv_quant(self.mlx_kv_bits)
+ return self
+
gpu_ids: Optional[List[int]] = Field(
None,
description = (
@@ -1309,17 +1322,22 @@ class _InferenceRuntimeFields(BaseModel):
False,
description = "Whether the active model runs on the AMD Ryzen AI NPU (FastFlowLM through Lemonade)",
)
- mlx_kv_bits: Optional[int] = Field(
- None, description = "MLX KV quantization bit width actually applied, if any"
+ mlx_kv_quant: Optional[str] = Field(
+ None, description = "MLX KV cache quantization actually applied, in mlx_kv_quant's vocabulary"
)
- mlx_kv_bits_requested: Optional[int] = Field(
+ mlx_kv_quant_requested: Optional[str] = Field(
None,
description = (
- "MLX KV quantization bit width the load asked for. Differs from "
- "mlx_kv_bits when the model could not honor it, which is exactly "
- "when the reason matters."
+ "MLX KV cache quantization the load asked for. Differs from mlx_kv_quant when the model "
+ "could not honor it, which is exactly when the reason matters."
),
)
+ mlx_kv_bits: Optional[float] = Field(
+ None, description = "mlx_kv_quant as a bare width, kept for clients reading the older field"
+ )
+ mlx_kv_bits_requested: Optional[float] = Field(
+ None, description = "mlx_kv_quant_requested as a bare width, kept for the same reason"
+ )
chat_template_override: Optional[str] = Field(
None,
description = "Chat template override this model was loaded with. llama.cpp reports what it applied; MLX reports what was requested, since a refusal applies nothing and the reason accompanies it",
diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py
index 361dd44cb48..9c6f014a1fa 100644
--- a/studio/backend/routes/chat_history.py
+++ b/studio/backend/routes/chat_history.py
@@ -411,6 +411,10 @@ class ChatPresetLoadConfig(BaseModel):
customContextLength: Optional[int] = Field(default = None, gt = 0)
maxSeqLength: Optional[float] = None
kvCacheDtype: Optional[str] = None
+ mlxKvQuant: Optional[
+ Literal["auto", "8", "6", "5", "4", "3", "2", "tq-4", "tq-3.5", "tq-3", "tq-2"]
+ ] = None
+ # Declared only because extra="forbid" would 400 the whole save for a client still sending it.
mlxKvBits: Optional[Literal[8, 6, 5, 4, 3, 2]] = None
speculativeType: Optional[str] = None
specDraftNMax: Optional[int] = Field(default = None, ge = 1, le = 16)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 0514be2f75e..62f1b9ae9a6 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7622,6 +7622,8 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
# Not MLX, so the MLX runtime fields report as absent.
is_mlx = False,
is_npu = False,
+ mlx_kv_quant = None,
+ mlx_kv_quant_requested = None,
mlx_kv_bits = None,
mlx_kv_bits_requested = None,
mlx_kv_quant_eligibility = None,
@@ -15710,11 +15712,12 @@ def _mlx_runtime_settings_match(backend, request) -> bool:
stored None on every request and reload forever.
"""
entry = backend.models.get(backend.active_model_name, {}) or {}
- if "mlx_kv_bits_requested" not in entry:
+ if "mlx_kv_quant_requested" not in entry:
return True
- from core.inference.mlx_inference import _normalize_mlx_kv_bits
+ from core.inference.mlx_inference import encode_mlx_kv_quant, parse_mlx_kv_quant
- return entry["mlx_kv_bits_requested"] == _normalize_mlx_kv_bits(request.mlx_kv_bits) and (
+ requested = encode_mlx_kv_quant(*parse_mlx_kv_quant(getattr(request, "mlx_kv_quant", None)))
+ return entry["mlx_kv_quant_requested"] == requested and (
entry.get("chat_template_override_requested") or None
) == (request.chat_template_override or None)
@@ -16653,6 +16656,8 @@ async def _load_model_impl(
has_audio_input = _model_info.get("has_audio_input", False),
has_video_input = _model_info.get("has_video_input", False),
is_mlx = bool(_model_info.get("is_mlx", False)),
+ mlx_kv_quant = _model_info.get("mlx_kv_quant"),
+ mlx_kv_quant_requested = _model_info.get("mlx_kv_quant_requested"),
**_unsloth_serving_fields(_model_info),
mlx_kv_bits = _model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),
@@ -17334,7 +17339,7 @@ async def _load_model_impl(
else placement.requested_gpu_ids
),
subject = current_subject,
- mlx_kv_bits = request.mlx_kv_bits,
+ mlx_kv_quant = request.mlx_kv_quant,
chat_template_override = request.chat_template_override,
load_cancel_event = load_cancel_event,
on_prior_worker_released = _release_chat_after_teardown,
@@ -17467,6 +17472,8 @@ async def _load_model_impl(
has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),
has_video_input = _model_info.get("has_video_input", False),
is_mlx = bool(_model_info.get("is_mlx", False)),
+ mlx_kv_quant = _model_info.get("mlx_kv_quant"),
+ mlx_kv_quant_requested = _model_info.get("mlx_kv_quant_requested"),
**_unsloth_serving_fields(_model_info),
mlx_kv_bits = _model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),
@@ -19911,6 +19918,8 @@ async def get_status(current_subject: str):
has_audio_input = has_audio_input,
has_video_input = has_video_input,
is_mlx = bool(model_info.get("is_mlx", False)),
+ mlx_kv_quant = model_info.get("mlx_kv_quant"),
+ mlx_kv_quant_requested = model_info.get("mlx_kv_quant_requested"),
**_unsloth_serving_fields(model_info),
mlx_kv_bits = model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = model_info.get("mlx_kv_bits_requested"),
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index e2f2b9c2db1..428b9c201f9 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -25,6 +25,7 @@ from pydantic import (
StrictInt,
ValidationError,
field_validator,
+ model_validator,
)
from auth.authentication import (
@@ -916,7 +917,19 @@ class ModelOverridePayload(BaseModel):
custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576)
kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32)
# A discrete set, enforced by the normalizer; these bounds only block absurd values.
- mlx_kv_bits: Optional[int] = Field(default = None, ge = 2, le = 8)
+ mlx_kv_quant: Optional[str] = Field(default = None, max_length = 16)
+ mlx_kv_bits: Optional[float] = Field(default = None, ge = 2, le = 8)
+
+ @model_validator(mode = "after")
+ def derive_mlx_kv_quant(self):
+ """Fold the pair into the field storage keeps; null is how a client spells Auto."""
+
+ if "mlx_kv_quant" not in self.model_fields_set and self.mlx_kv_bits is not None:
+ from core.inference.mlx_inference import encode_mlx_kv_quant
+ self.mlx_kv_quant = encode_mlx_kv_quant(self.mlx_kv_bits)
+ self.mlx_kv_bits = None
+ return self
+
speculative_type: Optional[str] = Field(default = None, max_length = 32)
spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16)
# Parallel decode slots (llama-server --parallel), GGUF-only; None follows the server default.
@@ -2272,7 +2285,7 @@ def update_openai_auto_switch_override(
max_seq_length = max_seq_length,
custom_context_length = custom_context_length,
kv_cache_dtype = payload.kv_cache_dtype,
- mlx_kv_bits = payload.mlx_kv_bits,
+ mlx_kv_quant = payload.mlx_kv_quant,
speculative_type = payload.speculative_type,
spec_draft_n_max = payload.spec_draft_n_max,
n_parallel = payload.n_parallel,
diff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.py
index 42171f520b5..3c7a66d7fce 100644
--- a/studio/backend/tests/test_chat_history_routes.py
+++ b/studio/backend/tests/test_chat_history_routes.py
@@ -608,12 +608,13 @@ def test_chat_preset_load_config_covers_frontend_persisted_fields():
assert persisted, "no PresetLoadConfig keys parsed"
backend = set(chat_history.ChatPresetLoadConfig.model_fields)
+ backend -= {"mlxKvBits"}
assert (
persisted == backend
), f"schema drift: frontend-only {persisted - backend}, backend-only {backend - persisted}"
-def test_chat_settings_payload_accepts_mlx_kv_bits():
+def test_chat_settings_payload_accepts_mlx_kv_quant():
from pydantic import ValidationError
# extra="forbid" rejects the whole settings write on an undeclared key.
@@ -623,14 +624,15 @@ def test_chat_settings_payload_accepts_mlx_kv_bits():
{
"name": "MLX preset",
"params": {"temperature": 0.7},
- "loadConfig": {"mlxKvBits": 8},
+ "loadConfig": {"mlxKvQuant": "tq-3.5"},
},
],
}
)
dumped = payload.model_dump(exclude_unset = True)
- assert dumped["customPresets"][0]["loadConfig"]["mlxKvBits"] == 8
+ assert dumped["customPresets"][0]["loadConfig"]["mlxKvQuant"] == "tq-3.5"
+ chat_history.ChatPresetLoadConfig.model_validate({"mlxKvQuant": "auto"})
for width in (4, None):
chat_history.ChatPresetLoadConfig.model_validate({"mlxKvBits": width})
# Only the widths MLX supports.
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index a1f9494acbc..553eda6dfca 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1053,6 +1053,8 @@ class TestLoadHubDownloadExclusion:
"parallel_slots",
"is_mlx",
"is_npu",
+ "mlx_kv_quant",
+ "mlx_kv_quant_requested",
"mlx_kv_bits",
"mlx_kv_bits_requested",
"mlx_kv_quant_eligibility",
@@ -1077,7 +1079,7 @@ class TestLoadHubDownloadExclusion:
fields = route._llama_runtime_fields(backend)
assert fields["is_mlx"] is False
- assert fields["mlx_kv_bits_requested"] is None
+ assert fields["mlx_kv_quant_requested"] is None
def test_in_flight_marker_counts_and_normalizes_case(self):
assert not hf_gguf_load_in_flight(REPO)
diff --git a/studio/backend/tests/test_metal_turboquant.py b/studio/backend/tests/test_metal_turboquant.py
new file mode 100644
index 00000000000..981cc137ddc
--- /dev/null
+++ b/studio/backend/tests/test_metal_turboquant.py
@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""What TurboQuant does to a real cache, on the installed mlx-vlm rather than the fake-mlx shim."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+mx = pytest.importorskip("mlx.core", reason = "needs a real MLX runtime")
+pytest.importorskip("mlx_vlm.turboquant", reason = "needs an mlx-vlm with TurboQuant")
+
+import mlx.nn as nn # noqa: E402
+
+from core.inference.mlx_inference import ( # noqa: E402
+ MLX_KV_GROUP_SIZE,
+ _copy_value,
+ _turboquant_status,
+)
+
+
+def _llama(layer_types = None, sliding = None):
+ from mlx_vlm.models.llama.config import ModelConfig
+ from mlx_vlm.models.llama.language import Model
+
+ model = Model(
+ ModelConfig(
+ model_type = "llama",
+ hidden_size = 64,
+ num_hidden_layers = 4,
+ intermediate_size = 128,
+ num_attention_heads = 4,
+ num_key_value_heads = 4,
+ rms_norm_eps = 1e-5,
+ vocab_size = 128,
+ layer_types = layer_types,
+ sliding_window = sliding,
+ )
+ )
+ mx.eval(model.parameters())
+ return model
+
+
[email protected]("bits", [2, 3, 3.5, 4])
+def test_every_offered_width_converts_and_decodes(bits):
+ status = _turboquant_status(_llama(), bits)
+ assert (status["eligibility"], status["kv_bits"], status["note"]) == ("full", bits, "")
+
+
+def test_a_model_holding_layers_native_reports_partial_and_says_so():
+ sliding = _llama(layer_types = ["sliding_attention"] + ["full_attention"] * 3, sliding = 64)
+ status = _turboquant_status(sliding, 4)
+ assert (status["eligibility"], status["kv_bits"]) == ("partial", 4) and status["note"]
+
+
+class FailsOnDecode(nn.Module):
+ def __init__(
+ self,
+ inner,
+ error = "no decode here",
+ ):
+ super().__init__()
+ self.inner = inner
+ self.error = error
+ self.calls = 0
+
+ @property
+ def layers(self):
+ return self.inner.layers
+
+ def __call__(self, *args, **kwargs):
+ self.calls += 1
+ if self.calls > 1:
+ raise RuntimeError(self.error)
+ return self.inner(*args, **kwargs)
+
+
+def test_a_model_that_cannot_decode_is_refused_rather_than_raising():
+ model = FailsOnDecode(_llama())
+ status = _turboquant_status(model, 4)
+ assert model.calls == 2
+ assert status["eligibility"] == "refused" and status["kv_bits"] is None
+ assert "RuntimeError" in status["reason"]
+
+
+def test_a_dead_metal_queue_is_raised_not_reported_as_a_refusal():
+ # The orchestrator retires the worker on this marker; a refusal would keep serving on a dead queue.
+ model = FailsOnDecode(_llama(), "[METAL] Command buffer execution failed: GPU Timeout Error")
+ with pytest.raises(RuntimeError, match = "GPU Timeout"):
+ _turboquant_status(model, 4)
+
+
+def test_a_copied_turboquant_cache_can_still_be_extended():
+ from mlx_vlm.generate.common import maybe_quantize_kv_cache
+ from mlx_vlm.models.cache import make_prompt_cache
+ from mlx_vlm.turboquant import TurboQuantKVCache
+
+ model = _llama()
+ entries = make_prompt_cache(model)
+ maybe_quantize_kv_cache(entries, 0, MLX_KV_GROUP_SIZE, 4, kv_quant_scheme = "turboquant")
+ result = model(mx.array([[1, 2, 3, 4]]), cache = entries)
+ mx.eval(getattr(result, "logits", result), [entry.state for entry in entries])
+
+ at = next(index for index, e in enumerate(entries) if isinstance(e, TurboQuantKVCache))
+ original = entries[at].state
+ copied = _copy_value(entries, mx)
+
+ assert [type(half) for half in copied[at].state] == [type(half) for half in original]
+
+ served = entries[at].offset
+ follow_on = model(mx.array([[5]]), cache = entries)
+ mx.eval(getattr(follow_on, "logits", follow_on), [entry.state for entry in entries])
+ assert copied[at].offset == served < entries[at].offset
+
+ grown = model(mx.array([[6]]), cache = copied)
+ mx.eval(getattr(grown, "logits", grown), [entry.state for entry in copied])
+ assert copied[at].offset == served + 1
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index dfd327f88f9..10eb382295b 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -277,6 +277,8 @@ def test_mlx_vlm_generation_survives_every_fusion_refusing(monkeypatch, mlx_infe
backend = MLXInferenceBackend()
backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})
backend._processor = SimpleNamespace(chat_template = "template")
+ backend._is_vlm = True
+ backend._reads_vision = True
args = (
[{"role": "user", "content": [{"type": "image"}]}],
[object()],
@@ -1123,6 +1125,7 @@ def test_mlx_vlm_reemits_think_prefill_inside_adapter_context(
backend = MLXInferenceBackend()
backend._model = SimpleNamespace(config = {"model_type": "deepseek_vl_v2"})
backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())
+ backend._reads_vision = True
monkeypatch.setattr(
backend, "_release_vlm_snapshots", lambda: order.append("snapshots_released")
)
@@ -1201,6 +1204,7 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):
)
backend = MLXInferenceBackend()
backend._model = SimpleNamespace(config = {"model_type": "deepseek_vl_v2"})
+ backend._reads_vision = True
backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())
args = ([{"role": "user", "content": [{"type": "image"}]}], [object()], 0, 1, 0, 0, 1, 1, None)
tools = [{"function": {"name": "search"}}]
@@ -1232,6 +1236,13 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):
tool_history = args[0] + [{"role": "assistant", "tool_calls": [{"id": "call-1"}]}]
with pytest.raises(RuntimeError, match = "tool-call history"):
list(backend._generate_vlm(*((tool_history,) + args[1:]), tools = tools))
+ state["generic"] = "generic prompt"
+ backend._kv_quant = {"kv_bits": 3.5}
+ backend._turboquant = True
+ list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools))
+ assert calls["stream"][-1][1]["kv_bits"] == 3.5
+ assert calls["stream"][-1][1]["kv_quant_scheme"] == "turboquant"
+
state["generic"] = ValueError("generic rendering failed")
state["model"] = f"User: {args[0][0]['content']}"
with pytest.raises(ValueError, match = "generic rendering failed"):
@@ -1262,6 +1273,7 @@ def test_mlx_vlm_image_injection_reuses_media_aliases(monkeypatch):
backend = MLXInferenceBackend()
backend._model = object()
backend._is_vlm = True
+ backend._reads_vision = True
captured = []
backend._generate_vlm = lambda messages, *_args, **_kwargs: (
captured.append(messages) or iter(())
@@ -1821,6 +1833,7 @@ def test_mlx_vlm_normalizes_native_reasoning_channels(monkeypatch):
apply_chat_template = lambda *_args, **_kwargs: "prompt",
)
backend._is_vlm = True
+ backend._reads_vision = True
assert list(
backend.generate_chat_response(
@@ -1867,6 +1880,7 @@ def test_mlx_vlm_post_tool_prompt_opens_reasoning_channel(monkeypatch):
apply_chat_template = lambda *_args, **_kwargs: post_tool_prompt,
)
backend._is_vlm = True
+ backend._reads_vision = True
snapshots = list(
backend.generate_chat_response(
@@ -2983,6 +2997,25 @@ def test_kv_quant_status_applies_only_when_eligible(monkeypatch):
assert mlx_inference._normalize_mlx_kv_bits(8) == 8
# Every width the runtime accepts is offered; mlx-lm adds no domain of its own.
assert [mlx_inference._normalize_mlx_kv_bits(b) for b in (2, 3, 5, 6)] == [2, 3, 5, 6]
+ assert [mlx_inference.parse_mlx_kv_quant(c) for c in ("auto", "4", "tq-4", "tq-3.5")] == [
+ (None, False),
+ (4, False),
+ (4, True),
+ (3.5, True),
+ ]
+
+ refuse = mlx_inference._turboquant_refusal
+ assert refuse(is_vision = False, is_distributed = False, is_lora = True)
+ assert refuse(is_vision = False, is_distributed = True, is_lora = False)
+ assert not refuse(is_vision = False, is_distributed = False, is_lora = False)
+ assert not refuse(is_vision = True, is_distributed = True, is_lora = True)
+
+ turbo = mlx_inference.MLXInferenceBackend()
+ turbo._kv_quant = {"kv_bits": 3.5}
+ turbo._turboquant = True
+ width = {"kv_bits": 3.5, "kv_quant_scheme": "turboquant", "quantized_kv_start": 0}
+ assert turbo._kv_runtime_quant_kwargs() == width
+ assert turbo._kv_quant_generate_kwargs() == {}
text = mlx_inference._kv_quant_status(8, object(), False)
vlm = mlx_inference._kv_quant_status(8, object(), True)
@@ -3028,6 +3061,57 @@ def _tiny_lm(
return _LM()
+def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypatch):
+ pytest.importorskip("mlx.core")
+ from core.inference import mlx_inference
+
+ attempts = []
+
+ class _Loader:
+ @staticmethod
+ def from_pretrained(name, **kwargs):
+ # The retry must not run under the failed load's traceback, which pins its weights.
+ attempts.append((kwargs["text_only"], sys.exc_info()[0]))
+ if not kwargs["text_only"]:
+ raise ValueError("Expected shape (262144, 640) but received shape (262144, 80)")
+ return SimpleNamespace(config = {}), SimpleNamespace()
+
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader.FastMLXModel = _Loader
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)
+ monkeypatch.setattr(
+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (2048, 2048, 2048)
+ )
+
+ backend = mlx_inference.MLXInferenceBackend()
+ config = SimpleNamespace(identifier = "org/prequantized-text", is_vision = False, is_lora = False)
+ backend.load_model(config, kv_quant = "tq-4")
+
+ assert attempts == [(False, None), (True, None)]
+ assert backend._turboquant_refusal == mlx_inference.MLX_TURBOQUANT_TEXT_LOAD
+ assert backend._is_vlm is False
+ assert backend._kv_quant["kv_bits"] is None
+
+ attempts.clear()
+ vision = SimpleNamespace(identifier = "org/vlm", is_vision = True, is_lora = False)
+ with pytest.raises(ValueError, match = "Expected shape"):
+ mlx_inference.MLXInferenceBackend().load_model(vision, kv_quant = "tq-4")
+ assert attempts == [(False, None)]
+
+ class _DeadQueue(_Loader):
+ @staticmethod
+ def from_pretrained(name, **kwargs):
+ attempts.append((kwargs["text_only"], None))
+ raise RuntimeError("[METAL] Command buffer execution failed: GPU Timeout Error")
+
+ attempts.clear()
+ loader.FastMLXModel = _DeadQueue
+ with pytest.raises(RuntimeError, match = "GPU Timeout"):
+ mlx_inference.MLXInferenceBackend().load_model(config, kv_quant = "tq-4")
+ assert attempts == [(False, None)]
+
+
def test_reload_comparison_and_response_carry_the_resolved_setting():
"""A load-time knob must force a reload and reach the client.
@@ -3041,19 +3125,23 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():
def req(**knobs):
return LoadRequest(model = "m", model_path = "m", **knobs)
- be = SimpleNamespace(active_model_name = "m", models = {"m": {"mlx_kv_bits_requested": 8}})
+ be = SimpleNamespace(active_model_name = "m", models = {"m": {"mlx_kv_quant_requested": "8"}})
+ assert _mlx_runtime_settings_match(be, req(mlx_kv_quant = "8"))
+ assert not _mlx_runtime_settings_match(be, req(mlx_kv_quant = "4"))
assert _mlx_runtime_settings_match(be, req(mlx_kv_bits = 8))
- assert not _mlx_runtime_settings_match(be, req(mlx_kv_bits = 4))
- be.models["m"] = {"mlx_kv_bits_requested": None}
- assert _mlx_runtime_settings_match(be, req(mlx_kv_bits = 7)) # both normalize away
+ assert req(mlx_kv_quant = None, mlx_kv_bits = 8).mlx_kv_quant is None
+ be.models["m"] = {"mlx_kv_quant_requested": "4"}
+ assert not _mlx_runtime_settings_match(be, req(mlx_kv_quant = "tq-4"))
+ be.models["m"] = {"mlx_kv_quant_requested": "auto"}
+ assert _mlx_runtime_settings_match(be, req(mlx_kv_quant = "tq-6")) # both normalize away
assert _mlx_runtime_settings_match(
SimpleNamespace(active_model_name = "m", models = {"m": {}}),
- req(mlx_kv_bits = 8),
+ req(mlx_kv_quant = "8"),
)
# Compared against the REQUESTED value, or a refused template reloads every request.
be.models["m"] = {
- "mlx_kv_bits_requested": None,
+ "mlx_kv_quant_requested": "auto",
"chat_template_override_requested": "{{ custom }}",
}
assert _mlx_runtime_settings_match(be, req(chat_template_override = "{{ custom }}"))
@@ -3061,7 +3149,7 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():
assert not _mlx_runtime_settings_match(be, req())
# A refusal records the request, so the same request still matches.
be.models["m"] = {
- "mlx_kv_bits_requested": None,
+ "mlx_kv_quant_requested": "auto",
"chat_template_override_requested": "{{ refused }}",
"chat_template_override_reason": "it could not render a conversation",
}
@@ -3072,11 +3160,11 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():
model = "m",
display_name = "m",
inference = {},
- mlx_kv_bits = 8,
+ mlx_kv_quant = "8",
mlx_kv_quant_eligibility = "full",
mlx_kv_quant_note = "n",
)
- assert resp.mlx_kv_bits == 8 and resp.mlx_kv_quant_note == "n"
+ assert resp.mlx_kv_quant == "8" and resp.mlx_kv_quant_note == "n"
def test_kv_quant_probe_reports_what_the_runtime_would_really_do(monkeypatch):
@@ -3131,8 +3219,8 @@ def test_parent_mirror_omits_runtime_fields_a_backend_never_reported():
assert _mlx_runtime_mirror_fields({"is_vision": False}) == {}
assert _mlx_runtime_mirror_fields(
- {"mlx_kv_bits": 8, "mlx_kv_bits_requested": 8, "other": 1}
- ) == {"mlx_kv_bits": 8, "mlx_kv_bits_requested": 8}
+ {"mlx_kv_quant": "8", "mlx_kv_quant_requested": "8", "other": 1}
+ ) == {"mlx_kv_quant": "8", "mlx_kv_quant_requested": "8"}
def test_chat_template_override_installs_only_where_one_already_exists():
@@ -3350,6 +3438,8 @@ def _vlm_backend(
chat_template = "t", tokenizer = backend._tokenizer, apply_chat_template = lambda *a, **k: ""
)
backend._model = object()
+ backend._is_vlm = True
+ backend._reads_vision = True
return backend
@@ -4007,6 +4097,7 @@ def test_vlm_seed_rides_on_the_sampler_not_a_seed_kwarg(monkeypatch):
backend = MLXInferenceBackend()
backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})
backend._processor = SimpleNamespace(chat_template = "template")
+ backend._reads_vision = True
args = (
[{"role": "user", "content": [{"type": "image"}]}],
[object()],
@@ -5079,6 +5170,7 @@ def test_the_window_reaches_the_runtime_on_every_generation_route(monkeypatch):
vlm = MLXInferenceBackend()
vlm._kv_cache_window = 4096
vlm._model = SimpleNamespace(config = {"model_type": "m"})
+ vlm._reads_vision = True
vlm._processor = SimpleNamespace(tokenizer = SimpleNamespace())
next(
vlm._generate_vlm(
@@ -6049,6 +6141,7 @@ def _run_spm_vlm_turn(
tokenizer = turn, chat_template = turn.chat_template, detokenizer = turn.detokenizer
)
backend._tokenizer = turn
+ backend._reads_vision = True
return list(
backend._generate_vlm(
[{"role": "user", "content": [{"type": "image"}]}],
@@ -6926,6 +7019,7 @@ def _run_vlm_budget(
backend = _budget_backend(monkeypatch, served = served, marker_tokens = marker_tokens)
backend._is_vlm = True
+ backend._reads_vision = True
backend._model = SimpleNamespace()
backend._processor = SimpleNamespace(tokenizer = backend._tokenizer)
args = (messages, [image] if image is not None else None, 0, 1, 0, 0, max_new_tokens, 1, None)
@@ -7083,6 +7177,7 @@ def _video_vlm_backend(monkeypatch, streams):
backend._model = SimpleNamespace(config = {"model_type": "qwen3_5"})
backend._processor = SimpleNamespace(tokenizer = SimpleNamespace())
backend._is_vlm = True
+ backend._reads_vision = True
return backend
@@ -7367,6 +7462,7 @@ def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_cli
backend = MLXInferenceBackend()
backend._model = object()
backend._is_vlm = True
+ backend._reads_vision = True
captured = []
backend._generate_vlm = lambda messages, *_args, **kwargs: (
captured.append((messages, kwargs)) or iter(())
@@ -7378,11 +7474,12 @@ def test_mlx_generate_chat_response_attaches_the_video_part_and_forwards_the_cli
assert messages[-1]["content"] == [{"type": "video"}, {"type": "text", "text": "what moves"}]
assert kwargs["video"] == _CLIP_B64
- backend._is_vlm = False
+ backend._reads_vision = False
with pytest.raises(RuntimeError, match = "loaded model does not read video"):
list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))
backend._is_vlm = True
+ backend._reads_vision = True
monkeypatch.setattr(mlx_inference, "_mlx_vlm_decodes_video", lambda: False)
with pytest.raises(RuntimeError, match = "installed mlx-vlm does not read video"):
list(backend.generate_chat_response([{"role": "user", "content": "hi"}], video = _CLIP_B64))
@@ -7859,6 +7956,8 @@ def _constrained_vlm_reply(
backend = MLXInferenceBackend()
backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})
backend._processor = SimpleNamespace(chat_template = "template", all_special_tokens = specials)
+ backend._is_vlm = True
+ backend._reads_vision = True
args = ([{"role": "user", "content": "hi"}], [], 0.7, 0.9, 40, 0.01, 64, 1.0, None)
return list(backend._generate_vlm(*args, response_format = {"type": "json_object"}))
@@ -8111,3 +8210,90 @@ def test_the_grammar_ends_on_the_stop_the_decode_loop_names():
_allowed(constraint, len(tokenizer))
constraint.advance(int(token_id))
assert _allowed(constraint, len(tokenizer)) == {runtime_stop}
+
+
+def test_turboquant_leaves_every_cache_for_the_runtime_to_build():
+ lm_cache = pytest.importorskip("mlx_lm.models.cache")
+
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ backend = MLXInferenceBackend()
+ kinds = lambda: [
+ type(entry).__name__
+ for entry in backend._prepare_kv_entries([lm_cache.KVCache(), lm_cache.KVCache()])
+ ]
+
+ backend._kv_quant = {"kv_bits": 3.5}
+ backend._turboquant = True
+ assert kinds() == ["KVCache"] * 2
+ backend._kv_quant = {"kv_bits": 4}
+ assert kinds() == ["KVCache"] * 2
+
+ backend._turboquant = False
+ assert kinds() == ["QuantizedKVCache"] * 2
+
+
+def test_turboquant_sends_its_width_on_both_vlm_cache_branches(monkeypatch):
+ """No pre-converted entry carries the width, so a reused session must still send it."""
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ backend = MLXInferenceBackend()
+ backend._reads_vision = True
+ backend._kv_quant = {"kv_bits": 3.5}
+ backend._turboquant = True
+
+ class _Session(contextlib.AbstractContextManager):
+ cache, media_block, step, produced_tokens, produced_seconds = [object()], None, 8, 0, 0.0
+
+ def __exit__(self, *_exc):
+ return False
+
+ def finish(self):
+ return None
+
+ session = _Session()
+ fresh = dict(_drive_vlm_generation(backend, monkeypatch))
+ reused = dict(_drive_vlm_generation(backend, monkeypatch, session = session))
+ for kwargs in (fresh, reused):
+ assert (kwargs["kv_bits"], kwargs["kv_quant_scheme"], kwargs["quantized_kv_start"]) == (
+ 3.5,
+ "turboquant",
+ 0,
+ )
+ assert "prompt_cache" in reused and "prompt_cache" not in fresh
+
+
+def test_a_turboquant_load_turns_a_pin_into_a_request_budget(monkeypatch):
+ # Same rule as mx.quantize since #11084: an enforceable pin budgets requests, it does not refuse.
+ pytest.importorskip("mlx_lm.models.cache")
+ from core.inference import mlx_inference
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ def status(
+ _model,
+ bits,
+ refusal = "",
+ ):
+ if refusal:
+ return {"kv_bits": None, "eligibility": "refused", "reason": refusal}
+ return {"kv_bits": bits, "eligibility": "full", "reason": ""}
+
+ monkeypatch.setattr(mlx_inference, "_turboquant_status", status)
+ honours = SimpleNamespace(layers = [object(), object()])
+
+ def policy(
+ requested,
+ served,
+ refusal = "",
+ ):
+ backend = MLXInferenceBackend()
+ backend._model = honours
+ backend._is_vlm = True
+ backend._turboquant = True
+ backend._turboquant_refusal = refusal
+ quant, window, enforced, budget = backend._resolve_kv_policy(True, 3.5, requested, served)
+ return quant["kv_bits"], window, enforced, budget
+
+ assert policy(8192, 8192) == (3.5, None, False, 8192)
+ assert policy(0, 262144) == (3.5, None, False, None)
+ assert policy(8192, 8192, "refused") == (None, 8192, True, None)
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 4d7aa9386e6..966f7a59ba1 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -1415,6 +1415,12 @@ def _put(model_id, **fields):
)
+def test_an_older_clients_cache_width_survives_the_override_route(monkeypatch):
+ _mock_override_store(monkeypatch)
+ _put("org/m", mlx_kv_bits = 8)
+ assert settings.get_model_overrides()["org/m"]["mlx_kv_quant"] == "8"
+
+
def test_model_override_roundtrip(monkeypatch):
_mock_override_store(monkeypatch)
@@ -8588,6 +8594,7 @@ def test_fill_absent_fields_carries_the_browser_only_settings_into_a_legacy_entr
max_seq_length = 8192,
)
settings_route.update_openai_auto_switch_override(legacy, "tester")
+ store[settings.MODEL_OVERRIDES_SETTING_KEY]["unsloth/B-GGUF:Q4_K_M"]["mlx_kv_bits"] = 8
backfill = settings_route.ModelOverridePayload(
model_id = "unsloth/B-GGUF:Q4_K_M",
@@ -8595,6 +8602,7 @@ def test_fill_absent_fields_carries_the_browser_only_settings_into_a_legacy_entr
max_seq_length = 2048,
custom_context_length = 32768,
kv_cache_dtype = "q8_0",
+ mlx_kv_quant = "tq-4",
speculative_type = "ngram",
gpu_ids = [0, 1],
fill_absent_fields = True,
@@ -8609,6 +8617,7 @@ def test_fill_absent_fields_carries_the_browser_only_settings_into_a_legacy_entr
assert entry["kv_cache_dtype"] == "q8_0"
assert entry["speculative_type"] == "ngram"
assert entry["gpu_ids"] == [0, 1]
+ assert settings.model_override_load_kwargs(entry, is_gguf = False)["mlx_kv_quant"] == "8"
# One entry, not two: the fill resolves onto the key a load reads.
assert list(store[settings.MODEL_OVERRIDES_SETTING_KEY]) == ["unsloth/B-GGUF:Q4_K_M"]
@@ -9064,22 +9073,44 @@ def test_two_spellings_of_one_cached_quant_do_not_delete_each_others_save(monkey
assert stored[list(stored)[0]]["max_seq_length"] in (4096, 8192)
-def test_mlx_kv_bits_survives_the_whole_override_projection():
+def test_mlx_kv_quant_survives_the_whole_override_projection():
# Dropped here, an API auto-switch would load a remembered MLX model at full
- # precision while the picker honored the width.
- for bits in (8, 6, 5, 4, 3, 2):
- assert settings.normalize_model_override({"mlx_kv_bits": bits}) == {"mlx_kv_bits": bits}
+ # precision while the picker honored the setting.
+ for quant in ("8", "6", "5", "4", "3", "2", "tq-4", "tq-3.5", "tq-3", "tq-2"):
+ assert settings.normalize_model_override({"mlx_kv_quant": quant}) == {"mlx_kv_quant": quant}
+
+ for rejected in ("7", "3.5", "tq-8", "tq-6", "auto", 4, True, None):
+ assert settings.normalize_model_override({"mlx_kv_quant": rejected}) == {}
+ assert settings.normalize_model_override({"mlx_kv_bits": 8}) == {"mlx_kv_quant": "8"}
+ # A hand-edited width drops alone instead of aborting the whole override.
+ for stored in (7, "invalid", "4", True, [4]):
+ assert settings.normalize_model_override({"mlx_kv_bits": stored}) == {}
+ assert settings.model_override_load_kwargs({"mlx_kv_bits": stored}, is_gguf = False) == {}
+ assert settings.model_override_load_kwargs({"mlx_kv_bits": 8}, is_gguf = False) == {
+ "mlx_kv_quant": "8"
+ }
+ assert (
+ settings.model_override_load_kwargs({"mlx_kv_quant": None, "mlx_kv_bits": 8}, is_gguf = False)
+ == {}
+ )
+ both = {"mlx_kv_quant": "auto", "mlx_kv_bits": 4}
+ assert settings.normalize_model_override(both) == {}
+ assert settings.model_override_load_kwargs(both, is_gguf = False) == {}
+ assert LoadRequest(model_path = "unsloth/A", **both).mlx_kv_quant == "auto"
+
+ def _folded(**kw):
+ payload = settings_route.ModelOverridePayload(model_id = "m", **kw)
+ return payload.mlx_kv_quant, payload.mlx_kv_bits
- # A discrete set, so an in-range width can still be one mx.quantize rejects.
- # bool is an int subclass, and a string width would reach LoadRequest untyped.
- for rejected in (7, 1, 0, 9, True, False, "4", 4.5, None):
- assert settings.normalize_model_override({"mlx_kv_bits": rejected}) == {}
+ assert _folded(mlx_kv_bits = 8) == ("8", None)
+ assert _folded(mlx_kv_quant = None, mlx_kv_bits = 8) == (None, None)
+ assert _folded(mlx_kv_quant = "tq-4", mlx_kv_bits = 8) == ("tq-4", None)
# Ungated on is_gguf, matching the picker's own load payload.
for is_gguf in (True, False):
- kwargs = settings.model_override_load_kwargs({"mlx_kv_bits": 4}, is_gguf = is_gguf)
- assert kwargs["mlx_kv_bits"] == 4
- assert LoadRequest(model_path = "unsloth/A", **kwargs).mlx_kv_bits == 4
+ kwargs = settings.model_override_load_kwargs({"mlx_kv_quant": "tq-4"}, is_gguf = is_gguf)
+ assert kwargs["mlx_kv_quant"] == "tq-4"
+ assert LoadRequest(model_path = "unsloth/A", **kwargs).mlx_kv_quant == "tq-4"
def _idle_backend(kw, monkeypatch, *, user_loaded):
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index 7aabfdd22af..576713f8dd7 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -365,8 +365,8 @@ CTX_CHECKPOINTS_MAX = 256
CACHE_RAM_MIN_MIB = -1
CACHE_RAM_MAX_MIB = 1024 * 1024
VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"})
-# Mirrors MLX_KV_BITS_CHOICES in core/inference/mlx_inference.py; a set, not a range.
-VALID_MLX_KV_BITS = frozenset({8, 6, 5, 4, 3, 2})
+# Mirrors MLX_KV_QUANT_CHOICES in core/inference/mlx_inference.py; a set, not a range.
+VALID_MLX_KV_QUANT = frozenset({"8", "6", "5", "4", "3", "2", "tq-4", "tq-3.5", "tq-3", "tq-2"})
# Mirrors PARALLEL_MIN/MAX in llama_server_args.py.
PARALLEL_SLOTS_MIN = 1
@@ -386,6 +386,20 @@ VALID_GPU_INDEX_KINDS = frozenset({"physical", "vulkan"})
LEGACY_GPU_INDEX_KIND = "physical"
+def _mlx_kv_quant_of(entry: dict[str, Any]) -> Optional[str]:
+ """This entry's cache quantization, reading the width it superseded when that is all it
+ holds. Only an entry omitting the field predates the setting."""
+ if "mlx_kv_quant" in entry:
+ return _clean_str(entry["mlx_kv_quant"], VALID_MLX_KV_QUANT)
+ bits = entry.get("mlx_kv_bits")
+ # Only a number: a hand-edited string or a bool must drop the width, not abort the whole override.
+ if isinstance(bits, bool) or not isinstance(bits, (int, float)):
+ return None
+ from core.inference.mlx_inference import encode_mlx_kv_quant
+
+ return _clean_str(encode_mlx_kv_quant(bits), VALID_MLX_KV_QUANT)
+
+
def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]:
if not isinstance(value, str):
return None
@@ -428,10 +442,9 @@ def normalize_model_override(
if kv_cache_dtype:
entry["kv_cache_dtype"] = kv_cache_dtype
- # MLX quantizes by bit width, not by a llama.cpp dtype name, so it is its own field.
- mlx_kv_bits = payload.get("mlx_kv_bits")
- if not isinstance(mlx_kv_bits, bool) and mlx_kv_bits in VALID_MLX_KV_BITS:
- entry["mlx_kv_bits"] = int(mlx_kv_bits)
+ mlx_kv_quant = _mlx_kv_quant_of(payload)
+ if mlx_kv_quant:
+ entry["mlx_kv_quant"] = mlx_kv_quant
speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES)
if speculative_type:
@@ -581,7 +594,6 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
for source, target in (
("llama_extra_args", "llama_extra_args"),
("kv_cache_dtype", "cache_type_kv"),
- ("mlx_kv_bits", "mlx_kv_bits"),
("n_parallel", "n_parallel"),
("speculative_type", "speculative_type"),
("spec_draft_n_max", "spec_draft_n_max"),
@@ -594,6 +606,10 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
if override.get(source) is not None:
kwargs[target] = override[source]
+ mlx_kv_quant = _mlx_kv_quant_of(override)
+ if mlx_kv_quant:
+ kwargs["mlx_kv_quant"] = mlx_kv_quant
+
if is_gguf:
if override.get("n_batch") is not None:
kwargs["n_batch"] = override["n_batch"]
@@ -878,8 +894,11 @@ def set_model_override(
model_id.strip(),
entry or None,
fill_absent_fields = fill_absent_fields,
- # The pin and its index space are one value: filling the qualifier onto ids this browser did not write relabels them.
- coupled_fields = (("gpu_ids", "gpu_index_kind"),),
+ coupled_fields = (
+ # The pin and its index space are one value: filling the qualifier onto ids this browser did not write relabels them.
+ ("gpu_ids", "gpu_index_kind"),
+ ("mlx_kv_quant", "mlx_kv_bits"),
+ ),
)
_invalidate(MODEL_OVERRIDES_SETTING_KEY)
return entry
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 30bcee4f0ff..6a0d5dba518 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2405,8 +2405,8 @@ const VISIBLE_MODEL_RUNTIME_KEYS = [
"chatTemplateOverrideReason",
// Or a background autoload leaves its width and verdict on the restored model.
// The rest of the group mlxRuntimeStateFrom writes.
- "mlxKvBits",
- "loadedMlxKvBitsRequested",
+ "mlxKvQuant",
+ "loadedMlxKvQuantRequested",
"mlxKvQuantReason",
"mlxKvQuantNote",
"loadedContextBudget",
@@ -3451,7 +3451,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
trust_remote_code: trustRemoteCode,
chat_template_override: effectiveChatTemplateOverride,
cache_type_kv: config.kvCacheDtype,
- mlx_kv_bits: config.mlxKvBits ?? null,
+ mlx_kv_quant: config.mlxKvQuant ?? null,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
reasoning_budget:
diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
index ca771dd0a05..3652ad28d40 100644
--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx
+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
@@ -551,7 +551,7 @@ export function ChatSettingsPanel({
);
const customContextLength = useChatRuntimeStore((s) => s.customContextLength);
const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);
- const mlxKvBits = useChatRuntimeStore((s) => s.mlxKvBits);
+ const mlxKvQuant = useChatRuntimeStore((s) => s.mlxKvQuant);
const gpuMemoryMode = useChatRuntimeStore((s) => s.gpuMemoryMode);
const gpuLayers = useChatRuntimeStore((s) => s.gpuLayers);
const nCpuMoe = useChatRuntimeStore((s) => s.nCpuMoe);
@@ -747,7 +747,7 @@ export function ChatSettingsPanel({
customContextLength,
loadedContextLength,
kvCacheDtype,
- mlxKvBits,
+ mlxKvQuant,
gpuMemoryMode,
gpuLayers,
nCpuMoe,
@@ -772,7 +772,7 @@ export function ChatSettingsPanel({
customContextLength,
loadedContextLength,
kvCacheDtype,
- mlxKvBits,
+ mlxKvQuant,
gpuMemoryMode,
gpuLayers,
nCpuMoe,
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 2dfe699584e..030e20868a0 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -1901,10 +1901,9 @@ export function useChatModelRuntime() {
const previousServerTuning: ServerTuningValues =
rollbackConfig ?? useChatRuntimeStore.getState();
// Same reason: the rollback echo would overwrite an edit staged against it.
- const previousMlxKvBits =
- rollbackConfig
- ? (rollbackConfig.mlxKvBits ?? null)
- : useChatRuntimeStore.getState().mlxKvBits;
+ const previousMlxKvQuant = rollbackConfig
+ ? (rollbackConfig.mlxKvQuant ?? null)
+ : useChatRuntimeStore.getState().mlxKvQuant;
if (isGguf && isDiffusion === undefined) {
// Prepare the token exactly as validateModel/loadModel do: the Hub rejects an invalid
// Authorization header with 401 even for a public repo, so sending the raw stored token here would
@@ -2000,8 +1999,10 @@ export function useChatModelRuntime() {
const loadKvCacheDtype =
pendingLoadConfig?.kvCacheDtype ?? stateBeforeUnload.kvCacheDtype;
// Per-model, not a standing preference: eligibility is decided per model.
- let loadMlxKvBits =
- pendingLoadConfig?.mlxKvBits ?? stateBeforeUnload.mlxKvBits;
+ let loadMlxKvQuant =
+ pendingLoadConfig
+ ? pendingLoadConfig.mlxKvQuant ?? null
+ : stateBeforeUnload.mlxKvQuant;
// gpuMemoryMode is a standing preference; the rest are per-model knobs the reset below clears, so
// they are re-baselined there in lock-step with the store. A GGUF native context can exceed
// maxSeqLength, so sizing on raw maxSeqLength could pass, unload, then have /load refuse it. A
@@ -2370,9 +2371,7 @@ export function useChatModelRuntime() {
ctxCheckpoints: pendingLoadConfig?.ctxCheckpoints ?? null,
cacheRam: pendingLoadConfig?.cacheRam ?? null,
};
- // Both payload-only. The store keeps its values: a width is dormant preset state off MLX, and a
- // completed load rewrites both anyway.
- loadMlxKvBits = pendingLoadConfig?.mlxKvBits ?? null;
+ loadMlxKvQuant = pendingLoadConfig?.mlxKvQuant ?? null;
loadChatTemplateOverride =
pendingLoadConfig?.chatTemplateOverride?.trim()
? pendingLoadConfig.chatTemplateOverride
@@ -2469,7 +2468,7 @@ export function useChatModelRuntime() {
approved_remote_code_fingerprint: approvedRemoteCodeFingerprint,
chat_template_override: effectiveChatTemplateOverride,
cache_type_kv: loadKvCacheDtype,
- mlx_kv_bits: loadMlxKvBits ?? null,
+ mlx_kv_quant: loadMlxKvQuant ?? null,
speculative_type: loadSpeculativeType,
spec_draft_n_max: loadSpecDraftNMax,
n_parallel: loadNParallel,
@@ -2842,7 +2841,7 @@ export function useChatModelRuntime() {
chat_template_override:
rollbackState.loadedChatTemplateOverride,
cache_type_kv: rollbackState.loadedKvCacheDtype,
- mlx_kv_bits: rollbackState.loadedMlxKvBitsRequested,
+ mlx_kv_quant: rollbackState.loadedMlxKvQuantRequested,
speculative_type:
rollbackState.loadedSpeculativeType,
spec_draft_n_max:
@@ -2946,7 +2945,7 @@ export function useChatModelRuntime() {
...mlxRuntimeStateFrom(rollbackResponse),
// After the spread, which seeds the control from the echo; the control keeps its intent, like
// nParallel above.
- mlxKvBits: previousMlxKvBits,
+ mlxKvQuant: previousMlxKvQuant,
loadedChatTemplateOverride:
rollbackState.loadedChatTemplateOverride,
...loadedGpuMemoryFields(rollbackResponse),
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index c528c2a6ca9..e4dc6ca9a75 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -10,6 +10,7 @@ import {
savedContextPin,
} from "@/features/model-picker";
// eslint-disable-next-line no-restricted-imports -- Avoid the hub barrel's React and download-manager exports.
+import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";
import { modelDisplayName } from "@/features/hub/lib/model-identity";
import { getInferenceStatus } from "../api/chat-api";
import { isSpeechOnlyStatus } from "./speech-only-status";
@@ -533,11 +534,11 @@ export function applyActiveModelStatusToStore(
// and request move together; a late reply can overwrite a newer one.
...(seedLoadParams &&
hydratingExistingModel &&
- status.mlx_kv_bits !== undefined &&
+ status.mlx_kv_quant_requested !== undefined &&
(status.is_mlx === true
? {
- mlxKvBits: status.mlx_kv_bits_requested ?? null,
- loadedMlxKvBitsRequested: status.mlx_kv_bits_requested ?? null,
+ mlxKvQuant: normalizeMlxKvQuant(status.mlx_kv_quant_requested),
+ loadedMlxKvQuantRequested: normalizeMlxKvQuant(status.mlx_kv_quant_requested),
mlxKvQuantReason: status.mlx_kv_quant_reason ?? null,
chatTemplateOverrideReason:
status.chat_template_override_reason ?? null,
@@ -545,7 +546,7 @@ export function applyActiveModelStatusToStore(
}
: {
// The verdict retires; the editable width is dormant, not wrong.
- loadedMlxKvBitsRequested: null,
+ loadedMlxKvQuantRequested: null,
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
@@ -555,13 +556,13 @@ export function applyActiveModelStatusToStore(
...(seedLoadParams &&
!hydratingExistingModel &&
status.is_mlx === true &&
- status.mlx_kv_bits !== undefined &&
- prevState.mlxKvBits === null &&
- prevState.loadedMlxKvBitsRequested === null &&
+ status.mlx_kv_quant_requested !== undefined &&
+ prevState.mlxKvQuant === null &&
+ prevState.loadedMlxKvQuantRequested === null &&
prevState.mlxKvQuantReason === null &&
prevState.chatTemplateOverrideReason === null && {
- mlxKvBits: status.mlx_kv_bits_requested ?? null,
- loadedMlxKvBitsRequested: status.mlx_kv_bits_requested ?? null,
+ mlxKvQuant: normalizeMlxKvQuant(status.mlx_kv_quant_requested),
+ loadedMlxKvQuantRequested: normalizeMlxKvQuant(status.mlx_kv_quant_requested),
mlxKvQuantReason: status.mlx_kv_quant_reason ?? null,
chatTemplateOverrideReason: status.chat_template_override_reason ?? null,
mlxKvQuantNote: status.mlx_kv_quant_note ?? null,
diff --git a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
index d4706e24051..97ef5d9ca62 100644
--- a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
+++ b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
@@ -1,39 +1,43 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import {
+ normalizeMlxKvQuant,
+ type MlxKvQuant,
+} from "@/features/model-picker/model-config/per-model-config";
import type { LoadModelResponse } from "../types/api";
type MlxRuntimeResponse = Pick<
LoadModelResponse,
| "is_mlx"
- | "mlx_kv_bits_requested"
+ | "mlx_kv_quant_requested"
| "mlx_kv_quant_reason"
| "chat_template_override_reason"
| "mlx_kv_quant_note"
>;
/** MLX KV-quantization state a load response establishes. A non-MLX response retires the verdict but
- * omits mlxKvBits: the width is dormant there, not wrong, and a preset carrying it must survive
+ * omits mlxKvQuant: the setting is dormant there, not wrong, and a preset carrying it must survive
* the round-trip. */
export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
- mlxKvBits?: number | null;
- loadedMlxKvBitsRequested: number | null;
+ mlxKvQuant?: MlxKvQuant | null;
+ loadedMlxKvQuantRequested: MlxKvQuant | null;
mlxKvQuantReason: string | null;
chatTemplateOverrideReason: string | null;
mlxKvQuantNote: string | null;
} {
if (resp.is_mlx !== true) {
return {
- loadedMlxKvBitsRequested: null,
+ loadedMlxKvQuantRequested: null,
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
};
}
+ const requested = normalizeMlxKvQuant(resp.mlx_kv_quant_requested);
return {
- // Requested, not applied: a refusal applies no width but still has a reason.
- mlxKvBits: resp.mlx_kv_bits_requested ?? null,
- loadedMlxKvBitsRequested: resp.mlx_kv_bits_requested ?? null,
+ mlxKvQuant: requested,
+ loadedMlxKvQuantRequested: requested,
mlxKvQuantReason: resp.mlx_kv_quant_reason ?? null,
chatTemplateOverrideReason: resp.chat_template_override_reason ?? null,
mlxKvQuantNote: resp.mlx_kv_quant_note ?? null,
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index eecb3bc01b2..b34e0b869db 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import type { PerModelConfig } from "@/features/model-picker";
+import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";
import {
parseGpuLayersOverride,
@@ -17,7 +18,7 @@ type ResidentRuntime = Pick<
InferenceStatusResponse,
| "requested_context_length"
| "cache_type_kv"
- | "mlx_kv_bits_requested"
+ | "mlx_kv_quant_requested"
| "speculative_type"
| "spec_draft_n_max"
| "requested_parallel_slots"
@@ -260,7 +261,7 @@ const SETTING_CHECKS: SettingCheck[] = [
mlxComparable: true,
pinned: () => true,
agrees: (c, s) =>
- (c.mlxKvBits ?? null) === (s.mlx_kv_bits_requested ?? null),
+ (c.mlxKvQuant ?? null) === normalizeMlxKvQuant(s.mlx_kv_quant_requested),
},
{
// Always pinned: an unset mode resolves to the standing preference and the load sends it. Reading
diff --git a/studio/frontend/src/features/chat/presets/preset-load-config.ts b/studio/frontend/src/features/chat/presets/preset-load-config.ts
index 3191798e18b..007d092fe5a 100644
--- a/studio/frontend/src/features/chat/presets/preset-load-config.ts
+++ b/studio/frontend/src/features/chat/presets/preset-load-config.ts
@@ -12,7 +12,8 @@ import {
MAX_SEQ_LENGTH_MAX,
DEFAULT_MAX_SEQ_LENGTH,
KV_CACHE_DTYPES,
- MLX_KV_BITS,
+ mlxKvQuantLabel,
+ normalizeMlxKvQuant,
N_BATCH_MAX,
N_BATCH_MIN,
N_PARALLEL_MAX,
@@ -45,7 +46,7 @@ export type PresetLoadConfig = Pick<
| "customContextLength"
| "maxSeqLength"
| "kvCacheDtype"
- | "mlxKvBits"
+ | "mlxKvQuant"
| "speculativeType"
| "specDraftNMax"
| "nParallel"
@@ -71,7 +72,7 @@ export const EMPTY_PRESET_LOAD_CONFIG: PresetLoadConfig = {
customContextLength: null,
maxSeqLength: null,
kvCacheDtype: null,
- mlxKvBits: null,
+ mlxKvQuant: null,
speculativeType: null,
specDraftNMax: null,
nParallel: null,
@@ -150,11 +151,7 @@ export function normalizePresetLoadConfig(
const normalized: PresetLoadConfig = {
customContextLength: requestableContextLength(partial.customContextLength),
maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength as number | null),
- mlxKvBits:
- typeof partial.mlxKvBits === "number" &&
- MLX_KV_BITS.includes(partial.mlxKvBits)
- ? partial.mlxKvBits
- : null,
+ mlxKvQuant: normalizeMlxKvQuant(partial.mlxKvQuant, partial.mlxKvBits),
kvCacheDtype:
typeof partial.kvCacheDtype === "string" &&
VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype)
@@ -263,7 +260,7 @@ export function capturePresetLoadConfig(): PresetLoadConfig | undefined {
customContextLength: effectiveContextLength ?? null,
maxSeqLength: isMlx ? null : normalizeMaxSeqLength(snapshot.maxSeqLength),
kvCacheDtype: snapshot.kvCacheDtype ?? null,
- mlxKvBits: snapshot.mlxKvBits ?? null,
+ mlxKvQuant: snapshot.mlxKvQuant ?? null,
speculativeType: normalizeSpeculativeType(snapshot.speculativeType),
specDraftNMax: snapshot.specDraftNMax ?? null,
nParallel: snapshot.nParallel ?? null,
@@ -329,7 +326,7 @@ export function applyPresetLoadConfig(config?: PresetLoadConfig | null): void {
maxSeqLength: normalizeMaxSeqLength(config.maxSeqLength) ?? DEFAULT_MAX_SEQ_LENGTH,
customContextLength: config.customContextLength ?? null,
kvCacheDtype: config.kvCacheDtype ?? null,
- mlxKvBits: config.mlxKvBits ?? null,
+ mlxKvQuant: config.mlxKvQuant ?? null,
speculativeType: config.speculativeType ?? null,
specDraftNMax: config.specDraftNMax ?? null,
nParallel: config.nParallel ?? null,
@@ -367,8 +364,8 @@ export function formatPresetLoadConfigSummary(
if (config.kvCacheDtype) {
parts.push(`KV ${config.kvCacheDtype}`);
}
- if (config.mlxKvBits) {
- parts.push(`MLX KV ${config.mlxKvBits}-bit`);
+ if (config.mlxKvQuant) {
+ parts.push(`MLX KV ${mlxKvQuantLabel(config.mlxKvQuant)}`);
}
if (config.speculativeType && config.speculativeType !== "auto") {
parts.push(`Spec ${config.speculativeType}`);
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index d672d22ee02..068a6e26737 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -1662,7 +1662,7 @@ export function SharedComposer({
approved_remote_code_fingerprint: approvedRemoteCodeFingerprint,
chat_template_override: effectiveChatTemplateOverride,
cache_type_kv: ownConfig.kvCacheDtype ?? null,
- mlx_kv_bits: ownConfig.mlxKvBits ?? null,
+ mlx_kv_quant: ownConfig.mlxKvQuant ?? null,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
reasoning_budget:
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 9dbaaf27440..e280b5122d3 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -11,7 +11,10 @@ import {
pinnedReasoningEffort,
useModelReasoningEffortStore,
} from "@/features/model-picker/components/model-selector/model-reasoning-effort";
-import { loadedContextFields } from "@/features/model-picker/model-config/per-model-config";
+import {
+ loadedContextFields,
+ type MlxKvQuant,
+} from "@/features/model-picker/model-config/per-model-config";
import {
cachedPinnableGpuIndexKind,
reconcileCachedGpuSelection,
@@ -2394,9 +2397,8 @@ type ChatRuntimeStore = {
maxToolCallsPerMessage: number;
toolCallTimeout: number;
kvCacheDtype: string | null;
- mlxKvBits: number | null;
- /** Width the backend was last asked for; the verdict belongs beside it. */
- loadedMlxKvBitsRequested: number | null;
+ mlxKvQuant: MlxKvQuant | null;
+ loadedMlxKvQuantRequested: MlxKvQuant | null;
mlxKvQuantReason: string | null;
chatTemplateOverrideReason: string | null;
mlxKvQuantNote: string | null;
@@ -4178,8 +4180,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
maxToolCallsPerMessage: 25,
toolCallTimeout: 5,
kvCacheDtype: null,
- mlxKvBits: null,
- loadedMlxKvBitsRequested: null,
+ mlxKvQuant: null,
+ loadedMlxKvQuantRequested: null,
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
@@ -5126,8 +5128,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
toolFullOutput: {},
activeDiffusionCanvasByThreadId: {},
kvCacheDtype: null,
- mlxKvBits: null,
- loadedMlxKvBitsRequested: null,
+ mlxKvQuant: null,
+ loadedMlxKvQuantRequested: null,
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 6927e7a5c3f..aac56bc45f8 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -66,7 +66,7 @@ export interface LoadModelRequest {
approved_remote_code_fingerprint?: string | null;
chat_template_override?: string | null;
cache_type_kv?: string | null;
- mlx_kv_bits?: number | null;
+ mlx_kv_quant?: string | null;
/** Speculative decoding mode for GGUF models: "auto" (platform-aware DSpark/DFlash when the model
* ships that sidecar, else MTP on MTP GGUFs, ngram-mod for sub-3B), "mtp", "dspark",
* "dflash", "ngram", "mtp+ngram", "off". The legacy spellings are still accepted. */
@@ -264,8 +264,8 @@ export interface LoadModelResponse {
preserve_thinking_default?: boolean;
supports_tools?: boolean;
cache_type_kv?: string | null;
- mlx_kv_bits?: number | null;
- mlx_kv_bits_requested?: number | null;
+ mlx_kv_quant?: string | null;
+ mlx_kv_quant_requested?: string | null;
mlx_kv_quant_eligibility?: string | null;
mlx_kv_quant_reason?: string | null;
chat_template_override_reason?: string | null;
@@ -377,8 +377,8 @@ export interface InferenceStatusResponse {
context_length_enforced?: boolean | null;
context_unbounded_when_batched?: boolean;
cache_type_kv?: string | null;
- mlx_kv_bits?: number | null;
- mlx_kv_bits_requested?: number | null;
+ mlx_kv_quant?: string | null;
+ mlx_kv_quant_requested?: string | null;
mlx_kv_quant_eligibility?: string | null;
mlx_kv_quant_reason?: string | null;
chat_template_override_reason?: string | null;
diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts
index c240c99d5fe..91b6273c01f 100644
--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts
+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts
@@ -15,6 +15,7 @@ import {
} from "../model-config/model-identity";
import {
DEFAULT_PER_MODEL_CONFIG,
+ normalizeMlxKvQuant,
type PerModelConfig,
deletePerModelConfigsForOverrideKeys,
normalizePerModelConfig,
@@ -33,6 +34,7 @@ export interface ApiModelOverride {
// biome-ignore lint/style/useNamingConvention: API schema
kv_cache_dtype?: string;
// biome-ignore lint/style/useNamingConvention: API schema
+ mlx_kv_quant?: string;
mlx_kv_bits?: number;
// biome-ignore lint/style/useNamingConvention: API schema
speculative_type?: string;
@@ -309,6 +311,8 @@ export function fromApiOverride(
// auto-switch max_seq_length first. So a row stating either field owns both.
const serverStatesPin =
override.custom_context_length != null || override.max_seq_length != null;
+ const serverStatesKvQuant =
+ "mlx_kv_quant" in override || "mlx_kv_bits" in override;
const normalized = normalizePerModelConfig({
...DEFAULT_PER_MODEL_CONFIG,
customContextLength: serverStatesPin
@@ -318,7 +322,9 @@ export function fromApiOverride(
? (override.max_seq_length ?? null)
: local.maxSeqLength,
kvCacheDtype: override.kv_cache_dtype ?? local.kvCacheDtype,
- mlxKvBits: override.mlx_kv_bits ?? local.mlxKvBits,
+ mlxKvQuant: serverStatesKvQuant
+ ? normalizeMlxKvQuant(override.mlx_kv_quant, override.mlx_kv_bits)
+ : (local.mlxKvQuant ?? null),
speculativeType: override.speculative_type ?? local.speculativeType,
specDraftNMax: override.spec_draft_n_max ?? local.specDraftNMax,
specDraftCacheDtype:
@@ -373,8 +379,8 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
payload.kv_cache_dtype = config.kvCacheDtype;
}
// Travels beside kv_cache_dtype, or an API auto-switch loads a remembered MLX model at full precision.
- if (config.mlxKvBits != null) {
- payload.mlx_kv_bits = config.mlxKvBits;
+ if (config.mlxKvQuant) {
+ payload.mlx_kv_quant = config.mlxKvQuant;
}
if (config.speculativeType) {
payload.speculative_type = config.speculativeType;
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index f14861a1b54..3c8b56d3ca1 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -146,7 +146,9 @@ import {
MAX_SEQ_LENGTH_MAX,
MAX_SEQ_LENGTH_MIN,
MAX_SEQ_LENGTH_STEP,
- MLX_KV_BITS,
+ MLX_KV_QUANTS,
+ mlxKvQuantLabel,
+ type MlxKvQuant,
N_BATCH_LLAMA_DEFAULT,
N_BATCH_MAX,
N_BATCH_MIN,
@@ -1047,7 +1049,7 @@ function GpuMemorySettings({
);
}
-const MLX_KV_BITS_AUTO = "auto";
+const MLX_KV_QUANT_AUTO = "auto";
function AdvancedSettingsToggle({
checked,
@@ -1162,12 +1164,15 @@ function MlxAdvancedSettings({
<InfoHint>
Lower KV cache precision to save memory, at some cost to quality.
Auto keeps full precision; 8-bit is the safest step down.
+ TurboQuant holds quality better at the low widths and adds 3.5-bit,
+ which uses 3-bit keys beside 4-bit values; sliding-window and
+ recurrent layers keep their native cache under it.
</InfoHint>
</div>
<Select
- value={config.mlxKvBits ? String(config.mlxKvBits) : MLX_KV_BITS_AUTO}
+ value={config.mlxKvQuant ?? MLX_KV_QUANT_AUTO}
onValueChange={(v) =>
- update({ mlxKvBits: v === MLX_KV_BITS_AUTO ? null : Number(v) })
+ update({ mlxKvQuant: v === MLX_KV_QUANT_AUTO ? null : (v as MlxKvQuant) })
}
>
<SelectTrigger
@@ -1179,10 +1184,10 @@ function MlxAdvancedSettings({
<SelectValue />
</SelectTrigger>
<SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
- <SelectItem value={MLX_KV_BITS_AUTO}>Auto</SelectItem>
- {MLX_KV_BITS.map((bits) => (
- <SelectItem key={bits} value={String(bits)}>
- {bits}-bit
+ <SelectItem value={MLX_KV_QUANT_AUTO}>Auto</SelectItem>
+ {MLX_KV_QUANTS.map((quant) => (
+ <SelectItem key={quant} value={quant}>
+ {mlxKvQuantLabel(quant)}
</SelectItem>
))}
</SelectContent>
@@ -2014,8 +2019,8 @@ export function ModelConfigPage({
useShallow(selectResidentEstimateSettings),
);
const mlxKvQuantNote = useChatRuntimeStore((s) => s.mlxKvQuantNote);
- const loadedMlxKvBitsRequested = useChatRuntimeStore(
- (s) => s.loadedMlxKvBitsRequested,
+ const loadedMlxKvQuantRequested = useChatRuntimeStore(
+ (s) => s.loadedMlxKvQuantRequested,
);
const isActiveModel = loadedConfig != null;
const hfToken = useChatRuntimeStore((s) => s.hfToken);
@@ -2156,7 +2161,7 @@ export function ModelConfigPage({
: null;
const mlxKvQuantOutcome =
isActiveModel &&
- (configState.mlxKvBits ?? null) === (loadedMlxKvBitsRequested ?? null)
+ (configState.mlxKvQuant ?? null) === (loadedMlxKvQuantRequested ?? null)
? // Both, not either: dropping the note promises savings before the offset where quantization actually starts.
[mlxKvQuantReason, mlxKvQuantNote].filter(Boolean).join(". ") || null
: null;
@@ -2178,12 +2183,12 @@ export function ModelConfigPage({
);
// Frozen like the rest of the auto-open decision, so editing the width does not reopen the
// section the user just closed.
- const [initialMlxKvBits] = useState(() => configState.mlxKvBits ?? null);
+ const [initialMlxKvQuant] = useState(() => configState.mlxKvQuant ?? null);
// Applicability stays live, unlike the snapshot above: MLX can become available after mount,
// and a width that starts applying then has to surface.
- const autoOpenForMlxKvBits = servedByMlx && initialMlxKvBits != null;
+ const autoOpenForMlxKvQuant = servedByMlx && initialMlxKvQuant != null;
const showAdvanced =
- advancedPreference ?? (autoOpenAdvanced || autoOpenForMlxKvBits);
+ advancedPreference ?? (autoOpenAdvanced || autoOpenForMlxKvQuant);
const toggleAdvanced = saveAdvancedSettingsOpen;
const contextInputRef = useRef<NumericValueInputHandle>(null);
const maxSeqLengthInputRef = useRef<NumericValueInputHandle>(null);
diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
index 45cd02c9971..bcd9e36259d 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
@@ -27,7 +27,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
);
const customContextLength = useChatRuntimeStore((s) => s.customContextLength);
const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);
- const mlxKvBits = useChatRuntimeStore((s) => s.mlxKvBits);
+ const mlxKvQuant = useChatRuntimeStore((s) => s.mlxKvQuant);
const speculativeType = useChatRuntimeStore((s) => s.speculativeType);
const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax);
const nParallel = useChatRuntimeStore((s) => s.nParallel);
@@ -81,7 +81,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
// Off-backend this stays null, or the model compares unequal to its own defaults
// over a field it cannot show.
- const effectiveMlxKvBits = isMlx ? (mlxKvBits ?? null) : null;
+ const effectiveMlxKvQuant = isMlx ? (mlxKvQuant ?? null) : null;
const config = useMemo<PerModelConfig | null>(() => {
if (!checkpoint || isExternalModelId(checkpoint)) {
@@ -94,7 +94,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
// pin every reload to whatever the first load happened to get.
maxSeqLength: isGguf || isMlx ? null : maxSeqLength,
kvCacheDtype: kvCacheDtype ?? null,
- mlxKvBits: effectiveMlxKvBits,
+ mlxKvQuant: effectiveMlxKvQuant,
speculativeType: speculativeType ?? "auto",
specDraftNMax: specDraftNMax ?? null,
nParallel: nParallel ?? null,
@@ -129,7 +129,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
maxSeqLength,
customContextLength,
kvCacheDtype,
- effectiveMlxKvBits,
+ effectiveMlxKvQuant,
speculativeType,
specDraftNMax,
nParallel,
diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
index e9ec9af341e..897147131bc 100644
--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
@@ -58,7 +58,7 @@ export function applyPerModelConfigToRuntime(
: { ids: null, indexKind: null };
useChatRuntimeStore.setState({
customContextLength: config.customContextLength ?? null,
- mlxKvBits: config.mlxKvBits ?? null,
+ mlxKvQuant: config.mlxKvQuant ?? null,
kvCacheDtype: config.kvCacheDtype ?? null,
speculativeType:
normalizeSpeculativeType(config.speculativeType) ??
@@ -123,7 +123,7 @@ export function currentRuntimePerModelConfig(
? normalizeMaxSeqLength(s.params.maxSeqLength)
: null,
kvCacheDtype: s.kvCacheDtype ?? null,
- mlxKvBits: s.mlxKvBits ?? null,
+ mlxKvQuant: s.mlxKvQuant ?? null,
speculativeType: normalizeSpeculativeType(s.speculativeType),
specDraftNMax: s.specDraftNMax ?? null,
specDraftCacheDtype: s.specDraftCacheDtype ?? null,
@@ -164,7 +164,7 @@ export function perModelConfigsEqual(
normalizeMaxSeqLength(a.maxSeqLength) ===
normalizeMaxSeqLength(b.maxSeqLength) &&
(a.kvCacheDtype ?? null) === (b.kvCacheDtype ?? null) &&
- (a.mlxKvBits ?? null) === (b.mlxKvBits ?? null) &&
+ (a.mlxKvQuant ?? null) === (b.mlxKvQuant ?? null) &&
normalizeSpeculativeType(a.speculativeType) ===
normalizeSpeculativeType(b.speculativeType) &&
(a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) &&
diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
index c9506c44bb7..240f291fec9 100644
--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts
+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
@@ -51,7 +51,7 @@ export function loadedConfigSignature(
config.customContextLength ?? "",
config.maxSeqLength ?? "",
config.kvCacheDtype ?? "",
- config.mlxKvBits ?? "",
+ config.mlxKvQuant ?? "",
config.speculativeType ?? "",
config.specDraftNMax ?? "",
config.specDraftCacheDtype ?? "",
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 8b2bbdbb9af..dc1497f0b7f 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -22,8 +22,7 @@ export interface PerModelConfig {
customContextLength: number | null;
maxSeqLength: number | null;
kvCacheDtype: string | null;
- /** MLX KV cache quantization width. Optional so older blobs still parse. */
- mlxKvBits?: number | null;
+ mlxKvQuant?: MlxKvQuant | null;
speculativeType: string | null;
specDraftNMax: number | null;
/** KV cache dtype for the DRAFT context, sized and quantized independently of kvCacheDtype.
@@ -65,7 +64,7 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {
customContextLength: null,
maxSeqLength: null,
kvCacheDtype: null,
- mlxKvBits: null,
+ mlxKvQuant: null,
speculativeType: null,
specDraftNMax: null,
specDraftCacheDtype: null,
@@ -280,8 +279,40 @@ export const KV_CACHE_DTYPES = [
"f32",
] as const;
-// Every width mx.quantize supports. By bit width, not a dtype name, hence separate from KV_CACHE_DTYPES.
-export const MLX_KV_BITS: readonly number[] = [8, 6, 5, 4, 3, 2];
+export const MLX_KV_QUANTS = [
+ "8",
+ "6",
+ "5",
+ "4",
+ "3",
+ "2",
+ "tq-4",
+ "tq-3.5",
+ "tq-3",
+ "tq-2",
+] as const;
+export type MlxKvQuant = (typeof MLX_KV_QUANTS)[number];
+const VALID_MLX_KV_QUANTS = new Set<string>(MLX_KV_QUANTS);
+
+export function mlxKvQuantLabel(quant: string): string {
+ return quant.startsWith("tq-") ? `TurboQuant ${quant.slice(3)}-bit` : `${quant}-bit`;
+}
+
+/** A bare width only ever meant mx.quantize; null is how a saved Auto spells itself. */
+export function normalizeMlxKvQuant(
+ value: unknown,
+ supersededBits?: unknown,
+): MlxKvQuant | null {
+ if (typeof value === "string") {
+ // Trimmed and lower-cased to match the backend's reader, or one row means two settings.
+ const named = value.trim().toLowerCase();
+ return VALID_MLX_KV_QUANTS.has(named) ? (named as MlxKvQuant) : null;
+ }
+ if (value !== undefined) return null;
+ if (typeof supersededBits !== "number" || !Number.isFinite(supersededBits)) return null;
+ const name = String(supersededBits);
+ return VALID_MLX_KV_QUANTS.has(name) ? (name as MlxKvQuant) : null;
+}
const VALID_KV_CACHE_DTYPES = new Set<string>(KV_CACHE_DTYPES);
// llama-server's --load-mode enum in --help order. "auto" is the default: the UI shows it, storage keeps null and
@@ -322,9 +353,10 @@ const LEGACY_STORAGE_KEY = "unsloth_load_settings";
const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";
// would normalize the unknown field straight back out of the record.
// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group
-// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair; a
-// client from before any of them
-const STORAGE_SCHEMA_VERSION = 6;
+// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,
+// v7 mlxKvQuant
+const STORAGE_SCHEMA_VERSION = 7;
+const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;
const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;
const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;
const PRE_VISION_SCHEMA_VERSION = 3;
@@ -357,14 +389,14 @@ type StoredPerModelConfig = PerModelConfig & {
version: number;
};
type StoredMap = Record<string, PerModelConfig | StoredPerModelConfig>;
-type RawConfig = Partial<PerModelConfig> & { version?: unknown };
+type RawConfig = Partial<PerModelConfig> & { version?: unknown; mlxKvBits?: unknown };
const STORED_CONFIG_FIELDS = new Set([
"version",
"customContextLength",
"maxSeqLength",
"kvCacheDtype",
- "mlxKvBits",
+ "mlxKvQuant",
"speculativeType",
"specDraftNMax",
"specDraftCacheDtype",
@@ -960,11 +992,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
? Math.max(CONTEXT_LENGTH_MIN, Math.floor(partial.customContextLength))
: null,
maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength),
- mlxKvBits:
- typeof partial.mlxKvBits === "number" &&
- MLX_KV_BITS.includes(partial.mlxKvBits)
- ? partial.mlxKvBits
- : null,
+ mlxKvQuant: normalizeMlxKvQuant(partial.mlxKvQuant, partial.mlxKvBits),
kvCacheDtype:
typeof partial.kvCacheDtype === "string" &&
VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype)
@@ -1047,10 +1075,13 @@ function normalize(raw: unknown): PerModelConfig {
* client reconstructs anyway, and stamping every record v4 would put the whole store out of reach.
* The tuning group and the reasoning pair follow the same rule. */
function storedSchemaVersion(normalized: PerModelConfig): number {
+ if (normalized.mlxKvQuant != null) {
+ return STORAGE_SCHEMA_VERSION;
+ }
const hasReasoningBudget =
normalized.reasoningBudget !== -1 || normalized.reasoningBudgetMessage !== "";
if (hasReasoningBudget) {
- return STORAGE_SCHEMA_VERSION;
+ return PRE_MLX_KV_QUANT_SCHEMA_VERSION;
}
const hasServerTuning =
normalized.loadMode != null ||
@@ -1218,7 +1249,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean {
config.customContextLength == null &&
config.maxSeqLength == null &&
(config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype &&
- (config.mlxKvBits ?? null) === DEFAULT_PER_MODEL_CONFIG.mlxKvBits &&
+ (config.mlxKvQuant ?? null) === DEFAULT_PER_MODEL_CONFIG.mlxKvQuant &&
config.speculativeType === DEFAULT_PER_MODEL_CONFIG.speculativeType &&
config.specDraftNMax == null &&
config.nParallel == null &&
diff --git a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
index 7fb5f9bf571..d03a6709487 100644
--- a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
+++ b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
@@ -37,6 +37,18 @@ const {
const MODEL = "unsloth/Model-GGUF";
const VARIANT = "q4_k_m";
+test("a server row holding the superseded cache width hydrates as the single choice", () => {
+ assert.equal(fromApiOverride({ mlx_kv_bits: 8 }).mlxKvQuant, "8");
+ assert.equal(fromApiOverride({ mlx_kv_quant: "tq-4", mlx_kv_bits: 8 }).mlxKvQuant, "tq-4");
+ const local = { ...DEFAULT_PER_MODEL_CONFIG, mlxKvQuant: "tq-3.5" as const };
+ assert.equal(fromApiOverride({ mlx_kv_quant: "auto", mlx_kv_bits: 4 }, local).mlxKvQuant, null);
+ assert.equal(
+ fromApiOverride({ mlx_kv_quant: null } as never, local).mlxKvQuant,
+ null,
+ );
+ assert.equal(fromApiOverride({}, local).mlxKvQuant, "tq-3.5");
+});
+
test("an explicit server clear leaves a config the panel must still persist", () => {
// What this browser remembers: one flag, nothing else.
const stored = { ...DEFAULT_PER_MODEL_CONFIG, llamaExtraArgs: ["--flash-attn"] };
diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
index 1ce40749ce4..96de26b75f3 100644
--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
@@ -204,12 +204,12 @@ const ROWS: Row[] = [
mlxRequest: 32768,
transformersRequest: 32768,
note:
- "The task called v4 the 'future' record. It is not: STORAGE_SCHEMA_VERSION is 6 " +
- "in this tree, not 3, so v4 is a v4-client record and reads normally.",
+ "Not a future record: STORAGE_SCHEMA_VERSION is 7, so v4 is a v4-client record " +
+ "and reads normally.",
},
{
- name: "version 7 (genuinely future)",
- raw: { version: 7, customContextLength: 32768 },
+ name: "version 8 (genuinely future)",
+ raw: { version: 8, customContextLength: 32768 },
normalizedPin: null,
rawPin: 32768,
isDefault: true,
@@ -330,27 +330,34 @@ test("a patched pin round-trips through storage on both backends", () => {
test("both pin shapes are stamped version 1, so neither is distinguishable by version", () => {
assert.equal(stampedVersion({ customContextLength: 32768 }), 1);
assert.equal(stampedVersion({ maxSeqLength: 32768 }), 1);
- // The old client's only forwards guard is `version > 5`, and v1 invites any client
// back to v1 to rewrite the record.
assert.equal(
stage({ version: 1, customContextLength: 32768 }).remembered,
true,
);
assert.equal(
- stage({ version: 5, customContextLength: 32768 }).remembered,
+ stage({ version: 6, customContextLength: 32768 }).remembered,
true,
);
assert.equal(
- stage({ version: 6, customContextLength: 32768 }).remembered,
+ stage({ version: 7, customContextLength: 32768 }).remembered,
true,
);
assert.equal(
- stage({ version: 7, customContextLength: 32768 }).remembered,
+ stage({ version: 8, customContextLength: 32768 }).remembered,
false,
);
});
+test("a stored mlxKvQuant stamps a version older builds refuse", () => {
+ assert.equal(stampedVersion({ mlxKvQuant: "tq-4" }), 7);
+ assert.equal(stampedVersion({ mlxKvQuant: "4" }), 7);
+ assert.equal(stampedVersion({ loadMode: "mmap" }), 5);
+ assert.equal(stampedVersion({ mlxKvQuant: "tq-4", loadMode: "mmap" }), 7);
+});
+
+
/**
* The old model-config page's read rule for a NON-GGUF target, transcribed from
* `studio/frontend/src/features/model-picker/components/model-config-page.tsx` on main:
@@ -472,3 +479,15 @@ test("BACKWARDS COMPAT: an old record still pins under the new code, on either b
// And it is not reported as default, so it survives savePerModelConfig's delete-if-default.
assert.equal(isDefaultConfig(config), false);
});
+
+test("the fractional TurboQuant choice survives the saved model record", () => {
+ store.clear();
+ savePerModelConfig(MODEL, null, { ...DEFAULT_PER_MODEL_CONFIG, mlxKvQuant: "tq-3.5" });
+ assert.equal(resolveInitialConfig(MODEL, null).config.mlxKvQuant, "tq-3.5");
+});
+
+test("a record holding the superseded width reads back as the single choice", () => {
+ assert.equal(stage({ mlxKvBits: 8 }).config.mlxKvQuant, "8");
+ assert.equal(stage({ mlxKvQuant: " TQ-4 ", mlxKvBits: 8 }).config.mlxKvQuant, "tq-4");
+ assert.equal(stage({ mlxKvQuant: null, mlxKvBits: 8 }).config.mlxKvQuant, null);
+});
diff --git a/studio/frontend/tests/model-config-instance-key.test.ts b/studio/frontend/tests/model-config-instance-key.test.ts
index 06428a738dd..44815cc33cf 100644
--- a/studio/frontend/tests/model-config-instance-key.test.ts
+++ b/studio/frontend/tests/model-config-instance-key.test.ts
@@ -107,7 +107,7 @@ test("every mirrored setting moves the instance key", () => {
{ ...LIVE, customContextLength: 8192 },
{ ...LIVE, maxSeqLength: 4096 },
{ ...LIVE, kvCacheDtype: "f16" },
- { ...LIVE, mlxKvBits: 4 },
+ { ...LIVE, mlxKvQuant: "4" },
{ ...LIVE, speculativeType: "off" },
{ ...LIVE, specDraftNMax: 4 },
{ ...LIVE, nParallel: 1 },
diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
index 68ea65f21d7..9844e99ca1c 100644
--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
@@ -63,7 +63,7 @@ const BLANK = {
customContextLength: null,
maxSeqLength: null,
kvCacheDtype: null,
- mlxKvBits: null,
+ mlxKvQuant: null,
speculativeType: null,
specDraftNMax: null,
nParallel: null,
@@ -113,7 +113,7 @@ const ACCELERATORS: Record<string, Record<string, unknown>> = {
// default load leaves the width unrequested; a pinned width is swept below like any other
// field, and is not part of the base for the same reason placement is not.
"apple-mlx": {
- mlx_kv_bits_requested: null,
+ mlx_kv_quant_requested: null,
},
};
@@ -150,10 +150,10 @@ const FIELDS: FieldCase[] = [
different: "f16",
},
{
- key: "mlxKvBits",
- statusKey: "mlx_kv_bits_requested",
- same: 8,
- different: 4,
+ key: "mlxKvQuant",
+ statusKey: "mlx_kv_quant_requested",
+ same: "8",
+ different: "tq-4",
},
{
key: "speculativeType",
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index 93d50341747..b5793a6617b 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -26,7 +26,7 @@ const DEFAULT_ISH = {
customContextLength: null,
maxSeqLength: null,
kvCacheDtype: null,
- mlxKvBits: null,
+ mlxKvQuant: null,
speculativeType: null,
specDraftNMax: null,
nParallel: null,
@@ -43,7 +43,7 @@ const BLANK = {
customContextLength: null,
maxSeqLength: null,
kvCacheDtype: null,
- mlxKvBits: null,
+ mlxKvQuant: null,
speculativeType: null,
specDraftNMax: null,
nParallel: null,
@@ -143,10 +143,11 @@ const FIELDS: {
differs: { cache_type_kv: "f16" },
},
{
- name: "MLX KV bits",
- config: { mlxKvBits: 4 },
- same: { mlx_kv_bits_requested: 4 },
- differs: { mlx_kv_bits_requested: 8 },
+ name: "MLX KV quantization",
+ config: { mlxKvQuant: "4" },
+ same: { mlx_kv_quant_requested: "4" },
+ differs: { mlx_kv_quant_requested: "tq-4" },
+
},
{
name: "speculative mode",
@@ -245,6 +246,11 @@ for (const field of FIELDS) {
});
}
+test("Auto adopts a resident model the backend reported as Auto", () => {
+ assert.equal(matches({ mlx_kv_quant_requested: "auto" }, { ...BLANK, mlxKvQuant: null }), true);
+ assert.equal(matches({ mlx_kv_quant_requested: "8" }, { ...BLANK, mlxKvQuant: null }), false);
+});
+
/** Ordering is the backend's to choose: it narrows and reorders placement at fit time. */
test("GPU placement compares as an order, not as a set", () => {
// The picker hands the list to the backend in order and position decides which
@@ -739,7 +745,7 @@ test("a non-GGUF resident is not judged on a GGUF invocation field", () => {
true,
);
assert.equal(
- matches({ ...DEFAULTS, is_gguf: false, mlx_kv_bits_requested: 4 }, BLANK),
+ matches({ ...DEFAULTS, is_gguf: false, mlx_kv_quant_requested: "4" }, BLANK),
false,
);
assert.equal(
@@ -1444,7 +1450,7 @@ test("unset nullable settings ask for the default, not for the resident value",
...DEFAULTS,
requested_context_length: 8192,
cache_type_kv: "q8_0",
- mlx_kv_bits_requested: 4,
+ mlx_kv_quant_requested: "4",
requested_parallel_slots: 4,
requested_n_batch: 2048,
requested_n_ubatch: 512,
@@ -1455,7 +1461,7 @@ test("unset nullable settings ask for the default, not for the resident value",
for (const [key, value] of Object.entries({
requested_context_length: 8192,
cache_type_kv: "q8_0",
- mlx_kv_bits_requested: 4,
+ mlx_kv_quant_requested: "4",
requested_parallel_slots: 4,
requested_n_batch: 2048,
requested_n_ubatch: 512,
diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.ts
index 127ec617676..eac1cfdc97b 100644
--- a/studio/frontend/tests/server-tuning-settings.test.ts
+++ b/studio/frontend/tests/server-tuning-settings.test.ts
@@ -113,10 +113,15 @@ test("a record only claims the new schema version when it carries one", () => {
// toStoredConfig stamps the OLDEST version that understands every field
// present, so an older client can still rewrite a record it fully knows.
const source = readSrc("features/model-picker/model-config/per-model-config.ts");
- assert.match(source, /const STORAGE_SCHEMA_VERSION = 6;/);
+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 7;/);
+ assert.match(source, /const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;/);
assert.match(source, /const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;/);
assert.match(source, /const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;/);
- assert.match(source, /hasReasoningBudget\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);
+ assert.match(source, /mlxKvQuant != null\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);
+ assert.match(
+ source,
+ /hasReasoningBudget\)\s*\{\s*return PRE_MLX_KV_QUANT_SCHEMA_VERSION;/,
+ );
assert.match(
source,
/hasServerTuning\)\s*\{\s*return PRE_REASONING_BUDGET_SCHEMA_VERSION;/,
diff --git a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts
index e93d86e9d0b..3c451ee82ef 100644
--- a/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts
+++ b/studio/frontend/tests/server-tuning-upgrade-downgrade-compat.test.ts
@@ -356,16 +356,16 @@ test("the backfill offers non-GGUF weights, keyed by repo id, after the v2 pass
const snapshot = "/hf/models--mlx-community--Model-4bit/snapshots/abc";
const template = { chatTemplateOverride: "{{ messages }}" };
assert.ok(savePerModelConfig(folder, null, config(template)));
- assert.ok(savePerModelConfig(snapshot, null, config({ mlxKvBits: 8 })));
+ assert.ok(savePerModelConfig(snapshot, null, config({ mlxKvQuant: "8" })));
// A newer build's record under the repo id makes adoption decline, and the path stays put.
const declined = "/hf/models--org--Newer/snapshots/def";
- assert.ok(savePerModelConfig("org/Newer", null, config({ mlxKvBits: 4 })));
+ assert.ok(savePerModelConfig("org/Newer", null, config({ mlxKvQuant: "4" })));
const map = readMap();
const newer = Object.keys(map).find((key) => key.includes("org/newer"));
assert.ok(newer);
map[newer].version = 99;
writeMap(map);
- assert.ok(savePerModelConfig(declined, null, config({ mlxKvBits: 4 })));
+ assert.ok(savePerModelConfig(declined, null, config({ mlxKvQuant: "4" })));
// Identities the resolver never keys.
const link = "/home/u/.ollama/.studio_links/ab12/model-latest.gguf";
assert.ok(savePerModelConfig(link, null, config(template)));
@@ -387,7 +387,7 @@ test("the backfill offers non-GGUF weights, keyed by repo id, after the v2 pass
new Set(puts),
new Set([folder, "mlx-community/model-4bit"]),
);
- assert.equal(resolveInitialConfig(declined, null).config.mlxKvBits, 4);
+ assert.equal(resolveInitialConfig(declined, null).config.mlxKvQuant, "4");
assert.equal(store.get(BACKFILL_FLAG), "1");
});
diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.py
index fa1d8eae543..8c7ed9c3c8a 100644
--- a/tests/studio/test_chat_autoload_failure_gate.py
+++ b/tests/studio/test_chat_autoload_failure_gate.py
@@ -353,20 +353,20 @@ let STORE: any = makeStore();
// rather than complete-by-luck.
// Imported by chat-adapter.ts from ../lib/mlx-runtime-state. Mirrors the real
// one rather than returning {}: a non-MLX response retires the verdict but
-// leaves mlxKvBits absent, and a scenario that saw an empty object would keep
+// leaves mlxKvQuant absent, and a scenario that saw an empty object would keep
// a stale width alive and pass for the wrong reason.
function mlxRuntimeStateFrom(resp: any) {
if (resp?.is_mlx !== true) {
return {
- loadedMlxKvBitsRequested: null,
+ loadedMlxKvQuantRequested: null,
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
};
}
return {
- mlxKvBits: resp.mlx_kv_bits_requested ?? null,
- loadedMlxKvBitsRequested: resp.mlx_kv_bits_requested ?? null,
+ mlxKvQuant: resp.mlx_kv_quant_requested ?? null,
+ loadedMlxKvQuantRequested: resp.mlx_kv_quant_requested ?? null,
mlxKvQuantReason: resp.mlx_kv_quant_reason ?? null,
chatTemplateOverrideReason: resp.chat_template_override_reason ?? null,
mlxKvQuantNote: resp.mlx_kv_quant_note ?? null,
diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.py
index f4e1b5edb0d..5d8d780d328 100644
--- a/tests/studio/test_cli_attach_load_knobs.py
+++ b/tests/studio/test_cli_attach_load_knobs.py
@@ -926,6 +926,28 @@ def test_the_requested_mlx_kv_width_survives_a_reload(monkeypatch):
assert server.loads[0]["mlx_kv_bits"] == 4
+def test_a_turboquant_resident_reloads_as_turboquant(monkeypatch):
+ """tq-4 and mx.quantize 4 share a width, so only mlx_kv_quant keeps the scheme."""
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "load_in_4bit": True,
+ "mlx_kv_quant": "tq-4",
+ "mlx_kv_quant_requested": "tq-4",
+ "mlx_kv_bits": 4,
+ "mlx_kv_bits_requested": 4,
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert server.loads[0]["mlx_kv_quant"] == "tq-4"
+
+
def test_a_proven_no_op_skips_the_preload_gate(monkeypatch):
"""A proven no-op skips the gate."""
server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index 6eeaf620609..0e4764413a1 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -1789,6 +1789,8 @@ _RESIDENT_RUNTIME_FIELDS = {
"speculative_type": "speculative_type",
"spec_draft_n_max": "spec_draft_n_max",
# The applied value is null when the runtime refused the request.
+ # Scheme and width together; a bare width reads back as mx.quantize, so TurboQuant would reload as it. The width stays for servers without mlx_kv_quant.
+ "mlx_kv_quant_requested": "mlx_kv_quant",
"mlx_kv_bits_requested": "mlx_kv_bits",
# LoadRequest defaults this to True, so omitting it would reload a full-precision model in 4-bit. Null on GGUF, which has no such setting.
"load_in_4bit": "load_in_4bit",