Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: experimental Int8 Prefill run setting for MLX models (#12550)

Long Yixingcc84c7dc7ae8
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index cbfa407f110..180bae6e12f 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -7,6 +7,7 @@ import difflib
import functools
import hashlib
import importlib
+import inspect
import os
import re
import sys
@@ -932,6 +933,76 @@ def _mlx_fused_residual_norm_handoff(model):
return _mlx_optional_fusion("fused_residual_norm_handoff", model)
+def _int8_prefill_zoo():
+ """unsloth_zoo's MLX inference module when it can run int8 prefill, else None."""
+ try:
+ zoo = importlib.import_module("unsloth_zoo.mlx.inference")
+ if "int8_prefill" in inspect.signature(zoo.nax_quantized_linear).parameters:
+ return zoo
+ except Exception:
+ pass
+ return None
+
+
+def _int8_prefill_status(requested, model):
+ """Whether a load runs int8 prefill: asked for, and the loaded model can use it here."""
+ status = {"requested": bool(requested), "active": False, "reason": "", "scope": None}
+ if not requested:
+ return status
+ zoo = _int8_prefill_zoo()
+ available = getattr(zoo, "int8_prefill_available", None)
+ if available is None:
+ status["reason"] = "unsupported_zoo"
+ return status
+ try:
+ verdict = available(model)
+ except Exception as exc:
+ if is_metal_queue_dead(exc):
+ raise
+ logger.warning("MLX int8 prefill check failed: %s", exc)
+ status["reason"] = "probe_failed"
+ return status
+ status["reason"] = verdict.reason
+ if verdict.available:
+ status["active"] = True
+ status["scope"] = zoo.nax_quantized_linear
+ else:
+ logger.info("MLX int8 prefill not applied: %s", verdict.reason)
+ return status
+
+
+# Keyed by checkpoint fingerprint: the panel asks again on every model switch.
+_int8_prefill_checkpoint_cache: dict = {}
+_int8_prefill_checkpoint_lock = threading.Lock()
+
+
+def mlx_int8_prefill_checkpoint_status(model_dir) -> tuple[bool, str]:
+ """(available, reason) of int8 prefill for a downloaded MLX checkpoint, without loading it."""
+ check = getattr(_int8_prefill_zoo(), "int8_prefill_checkpoint_available", None)
+ if check is None:
+ return False, "unsupported_zoo"
+ try:
+ from core.inference.mlx_memory import _checkpoint_fingerprint
+ key = (os.path.realpath(model_dir), _checkpoint_fingerprint(model_dir))
+ except Exception:
+ key = None
+ with _int8_prefill_checkpoint_lock:
+ if key in _int8_prefill_checkpoint_cache:
+ return _int8_prefill_checkpoint_cache[key]
+ try:
+ verdict = check(model_dir)
+ answer = (bool(verdict.available), verdict.reason)
+ except Exception as exc:
+ logger.debug("MLX int8 prefill checkpoint check failed for %s: %s", model_dir, exc)
+ return False, "probe_failed"
+ if key is not None and answer[1] != "probe_failed":
+ with _int8_prefill_checkpoint_lock:
+ _int8_prefill_checkpoint_cache[key] = answer
+ while len(_int8_prefill_checkpoint_cache) > 32:
+ _int8_prefill_checkpoint_cache.pop(next(iter(_int8_prefill_checkpoint_cache)))
+ return answer
+
+
def _vlm_generation_context():
import mlx.core as mx
from mlx_vlm.generate import generation_stream
@@ -3472,6 +3543,7 @@ class _TextBatchSession:
self._held.enter_context(
_temporary_mlx_adapter_state(backend._model, adapter_state)
)
+ self._held.enter_context(backend._int8_prefill_scope())
self.generator = BatchGenerator(
backend._model,
stop_tokens = [
@@ -3754,6 +3826,7 @@ class _VisionBatchSession:
self._held.enter_context(
_temporary_mlx_adapter_state(backend._model, adapter_state)
)
+ self._held.enter_context(backend._int8_prefill_scope(zoo_generation = True))
self.stream = BatchStream(
backend._model,
backend._processor,
@@ -3934,6 +4007,7 @@ class MLXInferenceBackend:
# Load-time runtime knobs; every generation path reads them from here rather than from per-request kwargs.
# Bound now so a load that fails before installing leaves readers a dict rather than raising.
self._kv_quant = _kv_quant_status(None, None, False)
+ self._int8_prefill = _int8_prefill_status(False, None)
self._kv_cache_window = None
self._kv_context_budget = None
self._served_context = None
@@ -4217,6 +4291,15 @@ class MLXInferenceBackend:
max_new_tokens = max(1, min(int(max_new_tokens), int(budget) - len(prompt_tokens)))
return max_new_tokens
+ def _int8_prefill_scope(self, zoo_generation = False):
+ """Outermost scope of a generation, which every Zoo scope nested in it inherits. A path
+ running Zoo's own generation scopes is pinned off when inactive, or they read the env."""
+ scope = (getattr(self, "_int8_prefill", None) or {}).get("scope")
+ if scope is not None:
+ return scope(self._model, True)
+ zoo = _int8_prefill_zoo() if zoo_generation else None
+ return zoo.nax_quantized_linear(self._model, False) if zoo is not None else nullcontext()
+
def _kv_quant_bits(self):
return (getattr(self, "_kv_quant", None) or {}).get("kv_bits")
@@ -4471,6 +4554,7 @@ class MLXInferenceBackend:
distributed_group = None,
kv_quant = None,
chat_template_override = None,
+ int8_prefill = False,
) -> bool:
import mlx.core as mx
@@ -4682,6 +4766,7 @@ class MLXInferenceBackend:
self._kv_quant["kv_bits"],
self._kv_quant["eligibility"],
)
+ self._int8_prefill = _int8_prefill_status(int8_prefill, self._model)
# Captured before installing, so chat_template_info keeps reporting what the model shipped with. From the
# render target, not the nested tokenizer: on a processor owning its own template those differ, and saving the
@@ -4766,6 +4851,9 @@ class MLXInferenceBackend:
"mlx_kv_quant_eligibility": self._kv_quant["eligibility"],
"mlx_kv_quant_reason": self._kv_quant["reason"],
"mlx_kv_quant_note": self._kv_quant["note"],
+ "mlx_int8_prefill": self._int8_prefill["active"],
+ "mlx_int8_prefill_requested": self._int8_prefill["requested"],
+ "mlx_int8_prefill_reason": self._int8_prefill["reason"],
"chat_template_override_requested": self._template_override["requested"],
"chat_template_override_reason": self._template_override["reason"],
}
@@ -5311,6 +5399,7 @@ class MLXInferenceBackend:
with (
self._generation_lock,
_temporary_mlx_adapter_state(self._model, _adapter_state),
+ self._int8_prefill_scope(),
_mlx_fused_moe_gate_up(self._model),
_mlx_fused_decode_conv_silu(self._model),
_mlx_fused_residual_norm(self._model),
@@ -6061,6 +6150,7 @@ class MLXInferenceBackend:
with (
self._generation_lock,
_temporary_mlx_adapter_state(self._model, _adapter_state),
+ self._int8_prefill_scope(),
ExitStack() as generation_scope,
session_scope,
):
@@ -6389,7 +6479,11 @@ class MLXInferenceBackend:
completion_batch_size = len(plans),
)
- with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):
+ with (
+ self._generation_lock,
+ _temporary_mlx_adapter_state(self._model, _adapter_state),
+ self._int8_prefill_scope(zoo_generation = True),
+ ):
if any(plan.images for plan in plans):
self._release_vlm_snapshots()
logger.info(
@@ -6567,6 +6661,7 @@ class MLXInferenceBackend:
with (
self._generation_lock,
_temporary_mlx_adapter_state(self._model, use_adapter),
+ self._int8_prefill_scope(),
ExitStack() as generation_scope,
):
# As on the image path: the tower gets the headroom, under the lock.
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 828eb103a1f..c69152abc47 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -252,6 +252,9 @@ _MLX_RUNTIME_MIRROR_FIELDS = (
"mlx_kv_quant_eligibility",
"mlx_kv_quant_reason",
"mlx_kv_quant_note",
+ "mlx_int8_prefill",
+ "mlx_int8_prefill_requested",
+ "mlx_int8_prefill_reason",
"chat_template_override_requested",
"chat_template_override_reason",
)
@@ -2002,6 +2005,7 @@ class InferenceOrchestrator:
tensor_parallel: bool = False,
mlx_distributed: bool = False,
mlx_kv_quant: Optional[str] = None,
+ mlx_int8_prefill: bool = False,
chat_template_override: Optional[str] = None,
load_cancel_event: Optional[threading.Event] = None,
post_handoff_expected_free_gb: Optional[dict[int, float]] = None,
@@ -2067,6 +2071,7 @@ class InferenceOrchestrator:
if mlx_distributed
else None,
"mlx_kv_quant": mlx_kv_quant,
+ "mlx_int8_prefill": bool(mlx_int8_prefill),
"chat_template_override": chat_template_override,
# Read in the worker, which hides the accelerators before detection.
"audio_device": audio_device,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 8202456dfe7..39fb949b4dc 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -740,6 +740,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
load_kwargs["parallel_mode"] = config.get("mlx_parallel_mode")
load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")
load_kwargs["kv_quant"] = config.get("mlx_kv_quant")
+ load_kwargs["int8_prefill"] = bool(config.get("mlx_int8_prefill"))
load_kwargs["chat_template_override"] = config.get("chat_template_override")
success = backend.load_model(**load_kwargs)
finally:
@@ -823,6 +824,9 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"mlx_kv_quant_eligibility",
"mlx_kv_quant_reason",
"mlx_kv_quant_note",
+ "mlx_int8_prefill",
+ "mlx_int8_prefill_requested",
+ "mlx_int8_prefill_reason",
"chat_template_override_requested",
"chat_template_override_reason",
)
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 317deb21f49..c490f6b8457 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -174,6 +174,16 @@ class LoadRequest(BaseModel):
self.mlx_kv_quant = encode_mlx_kv_quant(self.mlx_kv_bits)
return self
+ mlx_int8_prefill: bool = Field(
+ False,
+ description = (
+ "Experimental, MLX only: run quantized projections with int8 activations on Apple "
+ "neural accelerators for faster prompt processing. Lossy: outputs change and "
+ "accuracy drops. Applied only when the model supports it; the status reports the "
+ "reason otherwise."
+ ),
+ )
+
gpu_ids: Optional[List[int]] = Field(
None,
description = (
@@ -1036,6 +1046,27 @@ class EstimateMemoryRequest(BaseModel):
_resolve_the_handle = field_validator("model_path")(resolve_inventory_handle)
+class Int8PrefillAvailabilityRequest(BaseModel):
+ """A model whose run settings may offer MLX int8 prefill."""
+
+ model_path: str = Field(..., description = "Model identifier or local path")
+ hf_token: Optional[str] = Field(None, description = "Token for gated repositories")
+
+ _resolve_the_handle = field_validator("model_path")(resolve_inventory_handle)
+
+
+class Int8PrefillAvailabilityResponse(BaseModel):
+ """Whether a load of this model could run MLX int8 prefill, judged from its downloaded files."""
+
+ available: bool
+ reason: Optional[str] = Field(
+ None,
+ description = "Cause when available is false: 'not_downloaded', 'unsupported_model', "
+ "'unsupported_zoo', or unsloth_zoo's own reason ('nax_unavailable', "
+ "'no_eligible_projections', 'probe_failed').",
+ )
+
+
class EstimateMemoryResponse(BaseModel):
"""Itemized memory an inference load would occupy, or why it could not be sized."""
@@ -1583,6 +1614,17 @@ class _InferenceRuntimeFields(BaseModel):
mlx_kv_quant_note: Optional[str] = Field(
None, description = "Caveat that applies when KV quantization is active"
)
+ mlx_int8_prefill: Optional[bool] = Field(
+ None, description = "Whether MLX int8 prefill is active for the loaded model"
+ )
+ mlx_int8_prefill_requested: Optional[bool] = Field(
+ None, description = "Whether the load asked for MLX int8 prefill"
+ )
+ mlx_int8_prefill_reason: Optional[str] = Field(
+ None,
+ description = "Why requested int8 prefill is not active, as unsloth_zoo reports it "
+ "(for example nax_unavailable or no_eligible_projections)",
+ )
chat_template: Optional[str] = Field(
None,
description = "Jinja2 chat template string (from GGUF metadata or tokenizer)",
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 370abd48bb9..1c69bff515f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3732,6 +3732,8 @@ from models.inference import (
ValidateModelResponse,
EstimateMemoryRequest,
EstimateMemoryResponse,
+ Int8PrefillAvailabilityRequest,
+ Int8PrefillAvailabilityResponse,
TransformersUpgradeInfo,
TransformersUpgradeCheckRequest,
TransformersUpgradeCheckResponse,
@@ -8250,6 +8252,9 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
mlx_kv_quant_eligibility = None,
mlx_kv_quant_reason = None,
mlx_kv_quant_note = None,
+ mlx_int8_prefill = None,
+ mlx_int8_prefill_requested = None,
+ mlx_int8_prefill_reason = None,
mlx_context_budget = None,
chat_template_override_reason = None,
context_length_enforced = True,
@@ -16909,9 +16914,13 @@ def _mlx_runtime_settings_match(backend, request) -> bool:
from core.inference.mlx_inference import encode_mlx_kv_quant, parse_mlx_kv_quant
requested = encode_mlx_kv_quant(*parse_mlx_kv_quant(getattr(request, "mlx_kv_quant", None)))
- return entry["mlx_kv_quant_requested"] == requested and (
- entry.get("chat_template_override_requested") or None
- ) == (request.chat_template_override or None)
+ return (
+ entry["mlx_kv_quant_requested"] == requested
+ and (entry.get("chat_template_override_requested") or None)
+ == (request.chat_template_override or None)
+ and bool(entry.get("mlx_int8_prefill_requested"))
+ == bool(getattr(request, "mlx_int8_prefill", False))
+ )
def _inherit_resident_load_in_4bit(backend, request, model_identifier: str) -> None:
@@ -18198,6 +18207,9 @@ async def _load_model_impl(
mlx_kv_quant_eligibility = _model_info.get("mlx_kv_quant_eligibility"),
mlx_kv_quant_reason = _model_info.get("mlx_kv_quant_reason"),
mlx_kv_quant_note = _model_info.get("mlx_kv_quant_note"),
+ mlx_int8_prefill = _model_info.get("mlx_int8_prefill"),
+ mlx_int8_prefill_requested = _model_info.get("mlx_int8_prefill_requested"),
+ mlx_int8_prefill_reason = _model_info.get("mlx_int8_prefill_reason"),
# Requested, as /status reports it: a null override would read
# as "using the default".
chat_template_override = _model_info.get("chat_template_override_requested"),
@@ -18918,6 +18930,7 @@ async def _load_model_impl(
),
subject = current_subject,
mlx_kv_quant = request.mlx_kv_quant,
+ mlx_int8_prefill = request.mlx_int8_prefill,
chat_template_override = request.chat_template_override,
load_cancel_event = load_cancel_event,
on_prior_worker_released = _release_chat_after_teardown,
@@ -19079,6 +19092,9 @@ async def _load_model_impl(
mlx_kv_quant_eligibility = _model_info.get("mlx_kv_quant_eligibility"),
mlx_kv_quant_reason = _model_info.get("mlx_kv_quant_reason"),
mlx_kv_quant_note = _model_info.get("mlx_kv_quant_note"),
+ mlx_int8_prefill = _model_info.get("mlx_int8_prefill"),
+ mlx_int8_prefill_requested = _model_info.get("mlx_int8_prefill_requested"),
+ mlx_int8_prefill_reason = _model_info.get("mlx_int8_prefill_reason"),
# Requested, as /status reports it: a null override would read as
# "using the default".
chat_template_override = _model_info.get("chat_template_override_requested"),
@@ -20411,6 +20427,43 @@ def _cached_estimate_config(
return config
[email protected]("/int8-prefill-availability", response_model = Int8PrefillAvailabilityResponse)
+async def int8_prefill_availability(
+ request: Int8PrefillAvailabilityRequest, current_subject: str = Depends(get_current_subject)
+):
+ """Whether MLX int8 prefill would apply to this model, judged from its downloaded checkpoint
+ without loading it. The first answer on a new macOS or MLX version builds the kernels once."""
+ if account_access.managed_account():
+ await asyncio.to_thread(account_access.require_model_access, request.model_path)
+ if is_ollama_manifest_ref(request.model_path):
+ return Int8PrefillAvailabilityResponse(available = False, reason = "unsupported_model")
+
+ def _check() -> Int8PrefillAvailabilityResponse:
+ if not _mlx_estimate_available():
+ return Int8PrefillAvailabilityResponse(available = False, reason = "unsupported_model")
+ config = _cached_estimate_config(request.model_path, None, request.hf_token, False)
+ if config is _ESTIMATE_NOT_ON_DISK:
+ return Int8PrefillAvailabilityResponse(available = False, reason = "not_downloaded")
+ from core.inference.native_audio import is_native_audio_model
+
+ if (
+ config is None
+ or getattr(config, "is_gguf", False)
+ or getattr(config, "is_lora", False)
+ or is_native_audio_model(request.model_path)
+ ):
+ return Int8PrefillAvailabilityResponse(available = False, reason = "unsupported_model")
+ model_dir = _local_mlx_model_dir(config)
+ if not model_dir:
+ return Int8PrefillAvailabilityResponse(available = False, reason = "not_downloaded")
+ from core.inference.mlx_inference import mlx_int8_prefill_checkpoint_status
+
+ available, reason = mlx_int8_prefill_checkpoint_status(model_dir)
+ return Int8PrefillAvailabilityResponse(available = available, reason = reason or None)
+
+ return await asyncio.to_thread(_check)
+
+
@router.post("/estimate-memory", response_model = EstimateMemoryResponse)
async def estimate_memory(
request: EstimateMemoryRequest,
@@ -21784,6 +21837,9 @@ async def _slot_status(current_subject: str):
mlx_kv_quant_eligibility = model_info.get("mlx_kv_quant_eligibility"),
mlx_kv_quant_reason = model_info.get("mlx_kv_quant_reason"),
mlx_kv_quant_note = model_info.get("mlx_kv_quant_note"),
+ mlx_int8_prefill = model_info.get("mlx_int8_prefill"),
+ mlx_int8_prefill_requested = model_info.get("mlx_int8_prefill_requested"),
+ mlx_int8_prefill_reason = model_info.get("mlx_int8_prefill_reason"),
chat_template_override = model_info.get("chat_template_override_requested"),
chat_template_override_reason = model_info.get("chat_template_override_reason"),
loading = _loading_models,
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 8e828c380f5..832b82fa027 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -992,6 +992,7 @@ class ModelOverridePayload(BaseModel):
mirrors_reasoning_budget: bool = False
tensor_parallel: bool = False
disable_vision: bool = False
+ mlx_int8_prefill: bool = False
# Validated in bytes below: pydantic counts characters, so a multi-byte template would pass.
chat_template_override: Optional[str] = None
gpu_memory_mode: Optional[Literal["auto", "manual"]] = None
@@ -2356,10 +2357,11 @@ def update_openai_auto_switch_override(
is_removal = (
not payload.tensor_parallel
and not payload.disable_vision
+ and not payload.mlx_int8_prefill
and not {
key: value
for key, value in saved_fields.items()
- if key not in ("tensor_parallel", "disable_vision")
+ if key not in ("tensor_parallel", "disable_vision", "mlx_int8_prefill")
}
)
if requested_extra_args is None and not is_removal:
@@ -2567,6 +2569,7 @@ def update_openai_auto_switch_override(
cache_ram = _kept_tuning["cache_ram"],
tensor_parallel = payload.tensor_parallel,
disable_vision = payload.disable_vision,
+ mlx_int8_prefill = payload.mlx_int8_prefill,
chat_template_override = payload.chat_template_override,
gpu_memory_mode = payload.gpu_memory_mode,
gpu_layers = payload.gpu_layers,
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 526d1632ce3..820b9b375fb 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1191,6 +1191,9 @@ class TestLoadHubDownloadExclusion:
"mlx_kv_quant_eligibility",
"mlx_kv_quant_reason",
"mlx_kv_quant_note",
+ "mlx_int8_prefill",
+ "mlx_int8_prefill_requested",
+ "mlx_int8_prefill_reason",
"mlx_context_budget",
"chat_template_override_reason",
# Constant True: llama.cpp allocates the window it reports.
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index 01cef6d0d5f..b22cb7bd66c 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -1585,6 +1585,36 @@ class TestEstimateMemoryRoute:
assert resp.n_ctx == 4096
+class TestInt8PrefillAvailabilityRoute:
+ @pytest.mark.parametrize(
+ "downloaded,takes_int8,answer",
+ [
+ (True, True, (True, None)),
+ (False, True, (False, "not_downloaded")),
+ (True, False, (False, "unsupported_zoo")),
+ ],
+ )
+ def test_the_answer(self, monkeypatch, tmp_path, downloaded, takes_int8, answer):
+ from core.inference import mlx_inference
+ from models.inference import Int8PrefillAvailabilityRequest
+
+ asked = []
+ zoo = _types.ModuleType("unsloth_zoo.mlx.inference")
+ zoo.nax_quantized_linear = (lambda m, int8_prefill = None: m) if takes_int8 else len
+ verdict = SimpleNamespace(available = True, reason = "")
+ zoo.int8_prefill_checkpoint_available = lambda path: asked.append(path) or verdict
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", zoo)
+ monkeypatch.setattr(mlx_inference, "_int8_prefill_checkpoint_cache", {})
+ model_dir = str(tmp_path) if downloaded else None
+ TestEstimateMemoryRoute._mlx_target(monkeypatch, model_dir)
+ request = Int8PrefillAvailabilityRequest(model_path = "org/model")
+ resp = asyncio.run(ri.int8_prefill_availability(request, current_subject = "test"))
+ assert (resp.available, resp.reason) == answer
+ if answer[0]:
+ assert mlx_inference.mlx_int8_prefill_checkpoint_status(model_dir) == (True, "")
+ assert asked == ([model_dir] if answer[0] else [])
+
+
class TestParallelSlotResolution:
"""Blank Parallel Slots means the server default, not one.
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 9ea81d9471c..2552b6be31c 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -3390,6 +3390,118 @@ def test_reload_comparison_and_response_carry_the_resolved_setting():
assert resp.mlx_kv_quant == "8" and resp.mlx_kv_quant_note == "n"
+def _int8_zoo(
+ module,
+ available = True,
+ events = None,
+):
+ events = [] if events is None else events
+
+ @contextmanager
+ def nax_quantized_linear(model, int8_prefill = None):
+ events.append(("int8", int8_prefill))
+ yield
+ events.append(("int8_exit", int8_prefill))
+
+ module.nax_quantized_linear = nax_quantized_linear
+ reason = "" if available else "no_eligible_projections"
+ verdict = SimpleNamespace(available = available, reason = reason)
+ module.int8_prefill_available = lambda m: verdict
+ return events
+
+
+def test_int8_prefill_reloads_on_change_and_reports_why_it_is_off(mlx_inference_patches):
+ from core.inference import mlx_inference
+ from models.inference import LoadRequest
+ from routes.inference import _mlx_runtime_settings_match
+
+ req = lambda **knobs: LoadRequest(model = "m", model_path = "m", **knobs)
+ entry = {"mlx_kv_quant_requested": "auto"}
+ be = SimpleNamespace(active_model_name = "m", models = {"m": entry})
+ assert _mlx_runtime_settings_match(be, req())
+ assert not _mlx_runtime_settings_match(be, req(mlx_int8_prefill = True))
+ be.models["m"]["mlx_int8_prefill_requested"] = True
+ assert _mlx_runtime_settings_match(be, req(mlx_int8_prefill = True))
+ assert not _mlx_runtime_settings_match(be, req())
+
+ from core.inference import orchestrator
+
+ assert "mlx_int8_prefill_requested" in orchestrator._MLX_RUNTIME_MIRROR_FIELDS
+ status = mlx_inference._int8_prefill_status
+ _int8_zoo(mlx_inference_patches)
+ mlx_inference_patches.nax_quantized_linear = lambda model: contextlib.nullcontext()
+ assert status(True, object())["reason"] == "unsupported_zoo"
+ _int8_zoo(mlx_inference_patches, available = False)
+ assert status(True, object())["reason"] == "no_eligible_projections"
+ _int8_zoo(mlx_inference_patches)
+ assert status(True, object())["active"] and status(False, object())["scope"] is None
+
+
[email protected]("requested,available", [(True, True), (True, False), (False, True)])
+def test_int8_prefill_is_the_outermost_scope_of_generation_only_when_active(
+ monkeypatch, mlx_inference_patches, requested, available
+):
+ from core.inference import mlx_inference
+
+ events = _int8_zoo(mlx_inference_patches, available = available)
+
+ @contextmanager
+ def fusion(_model):
+ events.append(("fusion", None))
+ yield
+
+ for name in FUSIONS:
+ monkeypatch.setattr(mlx_inference_patches, f"fused_{name}", fusion)
+ mlx_vlm = types.ModuleType("mlx_vlm")
+ mlx_vlm.prompt_utils = SimpleNamespace(
+ MODEL_CONFIG = {}, apply_chat_template = lambda *_a, **_k: "<image> prompt"
+ )
+
+ def stream(*_a, **_k):
+ events.append(("stream", None))
+ yield SimpleNamespace(text = "ok", prompt_tokens = 1, generation_tokens = 1)
+
+ mlx_vlm.stream_generate = stream
+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda *_a, **_k: "<image> prompt",
+ )
+ monkeypatch.setattr(
+ mlx_inference, "_temporary_mlx_adapter_state", lambda *_a, **_k: contextlib.nullcontext()
+ )
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})
+ backend._processor = SimpleNamespace(chat_template = "template")
+ backend._is_vlm = backend._reads_vision = True
+ backend._int8_prefill = mlx_inference._int8_prefill_status(requested, backend._model)
+ args = ([{"role": "user", "content": [{"type": "image"}]}], [object()], 0.7, 0.9, 40, 0.01, 4)
+ assert list(backend._generate_vlm(*args, 1.0, None)) == ["ok"]
+
+ expected = [("int8", True)] + [("fusion", None)] * len(FUSIONS) + [("stream", None)]
+ assert events == (expected + [("int8_exit", True)] if requested and available else expected[1:])
+
+
+def test_a_batch_session_holds_int8_prefill_for_its_whole_life(monkeypatch, mlx_inference_patches):
+ from core.inference import mlx_inference
+
+ events = _int8_zoo(mlx_inference_patches)
+ engine = _batch_engine(monkeypatch)
+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)
+ stream = SimpleNamespace(close = lambda: events.append(("close", None)))
+ built = lambda *a, **k: events.append(("stream", None)) or stream
+ monkeypatch.setattr(engine, "BatchStream", built, raising = False)
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model = backend._processor = object()
+ for requested in (True, False):
+ events.clear()
+ backend._int8_prefill = mlx_inference._int8_prefill_status(requested, backend._model)
+ mlx_inference._VisionBatchSession(backend, width = 2).close()
+ # Off is pinned too, or Zoo's own scopes would read UNSLOTH_MLX_INT8_PREFILL.
+ want = [("int8", requested), ("stream", None), ("close", None), ("int8_exit", requested)]
+ assert events == want
+
+
def test_kv_quant_probe_reports_what_the_runtime_would_really_do(monkeypatch):
"""Attempt the conversion instead of predicting it from config or names.
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index a3a9b487b32..019666615af 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -9316,6 +9316,18 @@ def test_mlx_kv_quant_survives_the_whole_override_projection():
assert LoadRequest(model_path = "unsloth/A", **kwargs).mlx_kv_quant == "tq-4"
+def test_mlx_int8_prefill_is_stored_only_when_on_and_reaches_the_load(monkeypatch):
+ _mock_override_store(monkeypatch)
+ assert settings.normalize_model_override({"mlx_int8_prefill": False}) == {}
+ _put("org/m", mlx_int8_prefill = True)
+ stored = settings.get_model_overrides()["org/m"]
+ assert stored == {"mlx_int8_prefill": True}
+ kwargs = settings.model_override_load_kwargs(stored, is_gguf = False)
+ assert LoadRequest(model_path = "org/m", **kwargs).mlx_int8_prefill is True
+ _put("org/m", mlx_int8_prefill = False)
+ assert "org/m" not in settings.get_model_overrides()
+
+
def _idle_backend(kw, monkeypatch, *, user_loaded):
"""A loaded, long-idle GGUF backend wired into the idle loop."""
_reset_keepwarm()
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index 9ccf4f36485..51a3d0c6997 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -523,6 +523,9 @@ def normalize_model_override(
if _coerce_bool(payload.get("tensor_parallel")):
entry["tensor_parallel"] = True
+ if _coerce_bool(payload.get("mlx_int8_prefill")):
+ entry["mlx_int8_prefill"] = True
+
# Stored only when set. Like tensor_parallel: absent means the default, so an override that never touched the switch does not pin it off for a later load.
if _coerce_bool(payload.get("disable_vision")):
entry["disable_vision"] = True
@@ -628,6 +631,7 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
("tensor_parallel", "tensor_parallel"),
("disable_vision", "disable_vision"),
("chat_template_override", "chat_template_override"),
+ ("mlx_int8_prefill", "mlx_int8_prefill"),
):
if override.get(source) is not None:
kwargs[target] = override[source]
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 93e27116330..da2587e83ac 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2500,6 +2500,8 @@ const VISIBLE_MODEL_RUNTIME_KEYS = [
"loadedMlxKvQuantRequested",
"mlxKvQuantReason",
"mlxKvQuantNote",
+ "mlxInt8Prefill",
+ "loadedMlxInt8PrefillRequested",
"loadedContextBudget",
"loadedIsMultimodal",
"loadedIsDiffusion",
@@ -3546,6 +3548,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
chat_template_override: effectiveChatTemplateOverride,
cache_type_kv: config.kvCacheDtype,
mlx_kv_quant: config.mlxKvQuant ?? null,
+ mlx_int8_prefill: config.mlxInt8Prefill ?? false,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
reasoning_budget:
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index fe16a91c4c8..12cd24c3473 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -2060,6 +2060,9 @@ export function useChatModelRuntime() {
const previousMlxKvQuant = rollbackConfig
? (rollbackConfig.mlxKvQuant ?? null)
: useChatRuntimeStore.getState().mlxKvQuant;
+ const previousMlxInt8Prefill = rollbackConfig
+ ? (rollbackConfig.mlxInt8Prefill ?? false)
+ : useChatRuntimeStore.getState().mlxInt8Prefill;
if (isGguf && isDiffusion === undefined) {
// Prepare the token exactly as validateModel/loadModel do: the Hub rejects an invalid
// Authorization header with 401 even for a public repo, so sending the raw stored token here would
@@ -2159,6 +2162,9 @@ export function useChatModelRuntime() {
pendingLoadConfig
? pendingLoadConfig.mlxKvQuant ?? null
: stateBeforeUnload.mlxKvQuant;
+ let loadMlxInt8Prefill = pendingLoadConfig
+ ? (pendingLoadConfig.mlxInt8Prefill ?? false)
+ : stateBeforeUnload.mlxInt8Prefill;
// gpuMemoryMode is a standing preference; the rest are per-model knobs the reset below clears, so
// they are re-baselined there in lock-step with the store. A GGUF native context can exceed
// maxSeqLength, so sizing on raw maxSeqLength could pass, unload, then have /load refuse it. A
@@ -2536,6 +2542,7 @@ export function useChatModelRuntime() {
cacheRam: pendingLoadConfig?.cacheRam ?? null,
};
loadMlxKvQuant = pendingLoadConfig?.mlxKvQuant ?? null;
+ loadMlxInt8Prefill = pendingLoadConfig?.mlxInt8Prefill ?? false;
loadChatTemplateOverride =
pendingLoadConfig?.chatTemplateOverride?.trim()
? pendingLoadConfig.chatTemplateOverride
@@ -2636,6 +2643,7 @@ export function useChatModelRuntime() {
chat_template_override: effectiveChatTemplateOverride,
cache_type_kv: loadKvCacheDtype,
mlx_kv_quant: loadMlxKvQuant ?? null,
+ mlx_int8_prefill: loadMlxInt8Prefill,
speculative_type: loadSpeculativeType,
spec_draft_n_max: loadSpecDraftNMax,
n_parallel: loadNParallel,
@@ -3028,6 +3036,7 @@ export function useChatModelRuntime() {
rollbackState.loadedChatTemplateOverride,
cache_type_kv: rollbackState.loadedKvCacheDtype,
mlx_kv_quant: rollbackState.loadedMlxKvQuantRequested,
+ mlx_int8_prefill: rollbackState.loadedMlxInt8PrefillRequested,
speculative_type:
rollbackState.loadedSpeculativeType,
spec_draft_n_max:
@@ -3133,6 +3142,7 @@ export function useChatModelRuntime() {
// After the spread, which seeds the control from the echo; the control keeps its intent, like
// nParallel above.
mlxKvQuant: previousMlxKvQuant,
+ mlxInt8Prefill: previousMlxInt8Prefill,
loadedChatTemplateOverride:
rollbackState.loadedChatTemplateOverride,
...loadedGpuMemoryFields(rollbackResponse),
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index 9943ac8e94c..03dd97b25b3 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -552,6 +552,9 @@ export function applyActiveModelStatusToStore(
chatTemplateOverrideReason:
status.chat_template_override_reason ?? null,
mlxKvQuantNote: status.mlx_kv_quant_note ?? null,
+ mlxInt8Prefill: status.mlx_int8_prefill_requested === true,
+ loadedMlxInt8PrefillRequested:
+ status.mlx_int8_prefill_requested === true,
}
: {
// The verdict retires; the editable width is dormant, not wrong.
@@ -559,6 +562,7 @@ export function applyActiveModelStatusToStore(
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
+ loadedMlxInt8PrefillRequested: false,
})),
// Recovery for a hydration this tab never saw, and only when nothing is staged: re-seeding
// over an earlier edit would discard it.
@@ -569,12 +573,16 @@ export function applyActiveModelStatusToStore(
prevState.mlxKvQuant === null &&
prevState.loadedMlxKvQuantRequested === null &&
prevState.mlxKvQuantReason === null &&
- prevState.chatTemplateOverrideReason === null && {
+ prevState.chatTemplateOverrideReason === null &&
+ !prevState.mlxInt8Prefill && {
mlxKvQuant: normalizeMlxKvQuant(status.mlx_kv_quant_requested),
loadedMlxKvQuantRequested: normalizeMlxKvQuant(status.mlx_kv_quant_requested),
mlxKvQuantReason: status.mlx_kv_quant_reason ?? null,
chatTemplateOverrideReason: status.chat_template_override_reason ?? null,
mlxKvQuantNote: status.mlx_kv_quant_note ?? null,
+ mlxInt8Prefill: status.mlx_int8_prefill_requested === true,
+ loadedMlxInt8PrefillRequested:
+ status.mlx_int8_prefill_requested === true,
}),
// Baseline only, never the control: the echo is the RESOLVED count and would pin a blank
...(seedLoadParams &&
diff --git a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
index 97ef5d9ca62..5dee433b4fb 100644
--- a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
+++ b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
@@ -14,17 +14,20 @@ type MlxRuntimeResponse = Pick<
| "mlx_kv_quant_reason"
| "chat_template_override_reason"
| "mlx_kv_quant_note"
+ | "mlx_int8_prefill_requested"
>;
-/** MLX KV-quantization state a load response establishes. A non-MLX response retires the verdict but
- * omits mlxKvQuant: the setting is dormant there, not wrong, and a preset carrying it must survive
- * the round-trip. */
+/** MLX runtime state a load response establishes (KV quantization, template verdict, int8 prefill). A
+ * non-MLX response retires the verdicts but omits mlxKvQuant and mlxInt8Prefill: the settings are
+ * dormant there, not wrong, and a preset carrying the width must survive the round-trip. */
export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
mlxKvQuant?: MlxKvQuant | null;
loadedMlxKvQuantRequested: MlxKvQuant | null;
mlxKvQuantReason: string | null;
chatTemplateOverrideReason: string | null;
mlxKvQuantNote: string | null;
+ mlxInt8Prefill?: boolean;
+ loadedMlxInt8PrefillRequested: boolean;
} {
if (resp.is_mlx !== true) {
return {
@@ -32,6 +35,7 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
+ loadedMlxInt8PrefillRequested: false,
};
}
const requested = normalizeMlxKvQuant(resp.mlx_kv_quant_requested);
@@ -41,5 +45,7 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
mlxKvQuantReason: resp.mlx_kv_quant_reason ?? null,
chatTemplateOverrideReason: resp.chat_template_override_reason ?? null,
mlxKvQuantNote: resp.mlx_kv_quant_note ?? null,
+ mlxInt8Prefill: resp.mlx_int8_prefill_requested === true,
+ loadedMlxInt8PrefillRequested: resp.mlx_int8_prefill_requested === true,
};
}
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index a84662b5280..52bf70527f9 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -23,6 +23,7 @@ type ResidentRuntime = Pick<
| "requested_context_length"
| "cache_type_kv"
| "mlx_kv_quant_requested"
+ | "mlx_int8_prefill_requested"
| "speculative_type"
| "spec_draft_n_max"
| "requested_parallel_slots"
@@ -269,6 +270,12 @@ const SETTING_CHECKS: SettingCheck[] = [
agrees: (c, s) =>
(c.mlxKvQuant ?? null) === normalizeMlxKvQuant(s.mlx_kv_quant_requested),
},
+ {
+ mlxComparable: true,
+ pinned: () => true,
+ agrees: (c, s) =>
+ Boolean(c.mlxInt8Prefill) === (s.mlx_int8_prefill_requested === true),
+ },
{
// Always pinned: an unset mode resolves to the standing preference and the load sends it. Reading
// it as silence let a pick asking for "off" adopt a resident MTP runtime.
diff --git a/studio/frontend/src/features/chat/presets/preset-load-config.ts b/studio/frontend/src/features/chat/presets/preset-load-config.ts
index 007d092fe5a..9d87b950499 100644
--- a/studio/frontend/src/features/chat/presets/preset-load-config.ts
+++ b/studio/frontend/src/features/chat/presets/preset-load-config.ts
@@ -341,6 +341,8 @@ export function applyPresetLoadConfig(config?: PresetLoadConfig | null): void {
tensorParallel: config.tensorParallel ?? false,
disableVision: config.disableVision ?? false,
chatTemplateOverride: null,
+ // Not carried by presets: enabling it always goes through its confirmation.
+ mlxInt8Prefill: store.mlxInt8Prefill,
gpuMemoryMode: config.gpuMemoryMode,
gpuLayers: config.gpuLayers,
nCpuMoe: config.nCpuMoe,
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 85109d4ee53..4c3de6e5223 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -1824,6 +1824,7 @@ export function SharedComposer({
chat_template_override: effectiveChatTemplateOverride,
cache_type_kv: ownConfig.kvCacheDtype ?? null,
mlx_kv_quant: ownConfig.mlxKvQuant ?? null,
+ mlx_int8_prefill: ownConfig.mlxInt8Prefill ?? false,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
reasoning_budget:
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index f7e8d50f6b7..6ce0b1d536c 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -2425,6 +2425,8 @@ type ChatRuntimeStore = {
mlxKvQuantReason: string | null;
chatTemplateOverrideReason: string | null;
mlxKvQuantNote: string | null;
+ mlxInt8Prefill: boolean;
+ loadedMlxInt8PrefillRequested: boolean;
loadedKvCacheDtype: string | null;
speculativeType: string | null;
loadedSpeculativeType: string | null;
@@ -4215,6 +4217,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
+ mlxInt8Prefill: false,
+ loadedMlxInt8PrefillRequested: false,
loadedKvCacheDtype: null,
speculativeType: readPersistedSpeculativeType(),
loadedSpeculativeType: null,
@@ -5163,6 +5167,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
mlxKvQuantReason: null,
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
+ mlxInt8Prefill: false,
+ loadedMlxInt8PrefillRequested: false,
loadedKvCacheDtype: null,
speculativeType: readPersistedSpeculativeType(),
loadedSpeculativeType: null,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index f99664f2ca2..40905fcc9f7 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -72,6 +72,7 @@ export interface LoadModelRequest {
chat_template_override?: string | null;
cache_type_kv?: string | null;
mlx_kv_quant?: string | null;
+ mlx_int8_prefill?: boolean;
/** Speculative decoding mode for GGUF models: "auto" (platform-aware DSpark/DFlash when the model
* ships that sidecar, else MTP on MTP GGUFs, ngram-mod for sub-3B), "mtp", "dspark",
* "dflash", "ngram", "mtp+ngram", "off". The legacy spellings are still accepted. */
@@ -280,6 +281,9 @@ export interface LoadModelResponse {
mlx_kv_quant_reason?: string | null;
chat_template_override_reason?: string | null;
mlx_kv_quant_note?: string | null;
+ mlx_int8_prefill?: boolean | null;
+ mlx_int8_prefill_requested?: boolean | null;
+ mlx_int8_prefill_reason?: string | null;
chat_template?: string | null;
/** Canonical UI-facing mode the load request resolved to. See LoadModelRequest. */
speculative_type?: string | null;
@@ -422,6 +426,9 @@ export interface InferenceStatusResponse {
mlx_kv_quant_reason?: string | null;
chat_template_override_reason?: string | null;
mlx_kv_quant_note?: string | null;
+ mlx_int8_prefill?: boolean | null;
+ mlx_int8_prefill_requested?: boolean | null;
+ mlx_int8_prefill_reason?: string | null;
chat_template_override?: string | null;
/** Canonical UI-facing mode currently active. See LoadModelRequest. */
speculative_type?: string | null;
diff --git a/studio/frontend/src/features/model-picker/api/int8-prefill.ts b/studio/frontend/src/features/model-picker/api/int8-prefill.ts
new file mode 100644
index 00000000000..52ad76014bd
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/api/int8-prefill.ts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { authFetch } from "@/features/auth";
+
+/** Whether a load of this MLX model could run int8 prefill. Any failure reads as unavailable, which
+ * only hides the setting: a stored choice is kept, and the load-time check guards it. */
+export async function fetchInt8PrefillAvailable(
+ modelPath: string,
+ hfToken: string | null,
+ signal?: AbortSignal,
+): Promise<boolean> {
+ const response = await authFetch("/api/inference/int8-prefill-availability", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ signal,
+ body: JSON.stringify({ model_path: modelPath, hf_token: hfToken }),
+ });
+ if (!response.ok) {
+ return false;
+ }
+ const body: unknown = await response.json().catch(() => null);
+ return (
+ typeof body === "object" &&
+ body !== null &&
+ (body as { available?: unknown }).available === true
+ );
+}
diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts
index 5dd90daefef..7a45374e452 100644
--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts
+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts
@@ -40,6 +40,8 @@ export interface ApiModelOverride {
mlx_kv_quant?: string;
mlx_kv_bits?: number;
// biome-ignore lint/style/useNamingConvention: API schema
+ mlx_int8_prefill?: boolean;
+ // biome-ignore lint/style/useNamingConvention: API schema
speculative_type?: string;
// biome-ignore lint/style/useNamingConvention: API schema
spec_draft_n_max?: number;
@@ -348,6 +350,7 @@ export function fromApiOverride(
// Both are stored only when true, so an absent one is a gap like any other.
tensorParallel: override.tensor_parallel ?? local.tensorParallel,
disableVision: override.disable_vision ?? local.disableVision,
+ mlxInt8Prefill: override.mlx_int8_prefill ?? local.mlxInt8Prefill,
chatTemplateOverride:
override.chat_template_override ?? local.chatTemplateOverride,
llamaExtraArgs: extraArgs,
@@ -439,6 +442,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
if (config.disableVision) {
payload.disable_vision = true;
}
+ if (config.mlxInt8Prefill) {
+ payload.mlx_int8_prefill = true;
+ }
if (config.chatTemplateOverride?.trim()) {
payload.chat_template_override = config.chatTemplateOverride;
}
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 22c7b996030..e4e0a0c984e 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -2,6 +2,16 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { InferenceEnginePicker } from "./inference-engines";
+import {
+ AlertDialog,
+ AlertDialogAction,
+ AlertDialogCancel,
+ AlertDialogContent,
+ AlertDialogDescription,
+ AlertDialogFooter,
+ AlertDialogHeader,
+ AlertDialogTitle,
+} from "@/components/ui/alert-dialog";
import { useLlamaCppBackend } from "@/hooks/use-llama-backend";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
@@ -96,6 +106,7 @@ import {
selectResidentEstimateSettings,
} from "../model-config/resident-memory-request";
import { useMemoryEstimate } from "../hooks/use-memory-estimate";
+import { useInt8PrefillAvailable } from "../hooks/use-int8-prefill-available";
import {
fetchLoadModelOverride,
fromApiOverride,
@@ -1166,6 +1177,8 @@ function MlxAdvancedSettings({
update,
outcome,
servedByMlx,
+ int8PrefillAvailable,
+ onInt8PrefillChange,
onEditTemplate,
templateOutcome,
}: {
@@ -1175,6 +1188,8 @@ function MlxAdvancedSettings({
outcome: string | null;
/** KV quantization is MLX-only; a CUDA safetensors model has no such control. */
servedByMlx: boolean;
+ int8PrefillAvailable: boolean;
+ onInt8PrefillChange: (checked: boolean) => void;
onEditTemplate: () => void;
/** Why the loaded model could not take the override it was given. */
templateOutcome: string | null;
@@ -1224,6 +1239,26 @@ function MlxAdvancedSettings({
) : null}
</div>
)}
+ {/* An enabled choice stays visible so it can be turned off when availability is unknown. */}
+ {servedByMlx && (int8PrefillAvailable || config.mlxInt8Prefill) && (
+ <div className={ROW_CLASS}>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <span className={LABEL_CLASS}>Int8 Prefill</span>
+ <span className="shrink-0 rounded-md bg-[rgb(0_0_0_/_calc(0.04*var(--contrast-wash-gain,1)))] px-1.5 py-0.5 text-ui-10 font-medium uppercase tracking-wide text-muted-foreground dark:bg-muted">
+ Exp
+ </span>
+ <InfoHint>
+ Reads long prompts faster by doing part of the math at lower
+ precision. Answers can change and may be less accurate.
+ </InfoHint>
+ </div>
+ <Switch
+ className="panel-switch shrink-0"
+ checked={config.mlxInt8Prefill ?? false}
+ onCheckedChange={onInt8PrefillChange}
+ />
+ </div>
+ )}
{servedByMlx && (
<ParallelSlotsRow config={config} update={update} hint={MLX_PARALLEL_HINT} />
)}
@@ -2203,6 +2238,14 @@ export function ModelConfigPage({
platformDeviceType,
platformChatOnlyReason,
);
+ const int8PrefillAvailable = useInt8PrefillAvailable(
+ servedByMlx &&
+ !(target.meta.nativePathToken ?? (isActiveModel ? activeNativePathToken : null))
+ ? target.id
+ : null,
+ hfToken || null,
+ );
+ const [int8PrefillConfirmOpen, setInt8PrefillConfirmOpen] = useState(false);
// Read live, not snapshotted at mount: the sidebar copy stays mounted while collapsed.
const advancedPreference = useSyncExternalStore(
subscribeAdvancedSettingsOpen,
@@ -2226,11 +2269,16 @@ export function ModelConfigPage({
// Frozen like the rest of the auto-open decision, so editing the width does not reopen the
// section the user just closed.
const [initialMlxKvQuant] = useState(() => configState.mlxKvQuant ?? null);
+ const [initialMlxInt8Prefill] = useState(
+ () => configState.mlxInt8Prefill ?? false,
+ );
// Applicability stays live, unlike the snapshot above: MLX can become available after mount,
// and a width that starts applying then has to surface.
const autoOpenForMlxKvQuant = servedByMlx && initialMlxKvQuant != null;
+ const autoOpenForMlxInt8Prefill = servedByMlx && initialMlxInt8Prefill;
const showAdvanced =
- advancedPreference ?? (autoOpenAdvanced || autoOpenForMlxKvQuant);
+ advancedPreference ??
+ (autoOpenAdvanced || autoOpenForMlxKvQuant || autoOpenForMlxInt8Prefill);
const toggleAdvanced = saveAdvancedSettingsOpen;
const contextInputRef = useRef<NumericValueInputHandle>(null);
const maxSeqLengthInputRef = useRef<NumericValueInputHandle>(null);
@@ -3547,6 +3595,12 @@ export function ModelConfigPage({
update={update}
outcome={mlxKvQuantOutcome}
servedByMlx={servedByMlx}
+ int8PrefillAvailable={int8PrefillAvailable}
+ onInt8PrefillChange={(checked) =>
+ checked
+ ? setInt8PrefillConfirmOpen(true)
+ : update({ mlxInt8Prefill: false })
+ }
onEditTemplate={() => setTemplateOpen(true)}
templateOutcome={chatTemplateOutcome}
/>
@@ -3675,6 +3729,34 @@ export function ModelConfigPage({
readOnly={!target.isGguf && !servedByMlx}
onSave={(override) => update({ chatTemplateOverride: override })}
/>
+ <AlertDialog
+ open={int8PrefillConfirmOpen}
+ onOpenChange={setInt8PrefillConfirmOpen}
+ >
+ <AlertDialogContent size="sm">
+ <AlertDialogHeader>
+ <AlertDialogTitle>Turn on Int8 Prefill?</AlertDialogTitle>
+ <AlertDialogDescription>
+ Long prompts are read faster, but the model&apos;s answers will
+ change and may be less accurate. Some models are affected more
+ than others. You can turn it off at any time.
+ </AlertDialogDescription>
+ </AlertDialogHeader>
+ <AlertDialogFooter>
+ <AlertDialogCancel>Cancel</AlertDialogCancel>
+ <AlertDialogAction
+ variant="destructive"
+ className="!bg-destructive !text-destructive-foreground hover:!bg-destructive/90"
+ onClick={() => {
+ update({ mlxInt8Prefill: true });
+ setInt8PrefillConfirmOpen(false);
+ }}
+ >
+ Turn on
+ </AlertDialogAction>
+ </AlertDialogFooter>
+ </AlertDialogContent>
+ </AlertDialog>
</div>
);
}
diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
index ac6c1092819..e13c2cc731f 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
@@ -32,6 +32,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
const customContextLength = useChatRuntimeStore((s) => s.customContextLength);
const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);
const mlxKvQuant = useChatRuntimeStore((s) => s.mlxKvQuant);
+ const mlxInt8Prefill = useChatRuntimeStore((s) => s.mlxInt8Prefill);
const speculativeType = useChatRuntimeStore((s) => s.speculativeType);
const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax);
const nParallel = useChatRuntimeStore((s) => s.nParallel);
@@ -89,6 +90,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
// Off-backend this stays null, or the model compares unequal to its own defaults
// over a field it cannot show.
const effectiveMlxKvQuant = isMlx ? (mlxKvQuant ?? null) : null;
+ const effectiveMlxInt8Prefill = isMlx && mlxInt8Prefill;
const config = useMemo<PerModelConfig | null>(() => {
if (!checkpoint || isExternalModelId(checkpoint)) {
@@ -105,6 +107,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
maxSeqLength: isGguf || isMlx || isNpuModelId(checkpoint) ? null : maxSeqLength,
kvCacheDtype: kvCacheDtype ?? null,
mlxKvQuant: effectiveMlxKvQuant,
+ mlxInt8Prefill: effectiveMlxInt8Prefill,
speculativeType: speculativeType ?? "auto",
specDraftNMax: specDraftNMax ?? null,
nParallel: nParallel ?? null,
@@ -149,6 +152,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
customContextLength,
kvCacheDtype,
effectiveMlxKvQuant,
+ effectiveMlxInt8Prefill,
speculativeType,
specDraftNMax,
nParallel,
diff --git a/studio/frontend/src/features/model-picker/hooks/use-int8-prefill-available.ts b/studio/frontend/src/features/model-picker/hooks/use-int8-prefill-available.ts
new file mode 100644
index 00000000000..c75f71448f3
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/hooks/use-int8-prefill-available.ts
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useEffect, useState } from "react";
+import { fetchInt8PrefillAvailable } from "../api/int8-prefill";
+
+/** Whether the model can run MLX int8 prefill: false until the backend says so, and for a null model. */
+export function useInt8PrefillAvailable(
+ modelPath: string | null,
+ hfToken: string | null,
+): boolean {
+ const key = modelPath == null ? null : `${modelPath}\n${hfToken ?? ""}`;
+ const [answer, setAnswer] = useState<{
+ key: string;
+ available: boolean;
+ } | null>(null);
+ useEffect(() => {
+ if (key == null || modelPath == null) {
+ return;
+ }
+ const controller = new AbortController();
+ fetchInt8PrefillAvailable(modelPath, hfToken, controller.signal)
+ .then((available) => {
+ if (!controller.signal.aborted) {
+ setAnswer({ key, available });
+ }
+ })
+ .catch(() => {
+ if (!controller.signal.aborted) {
+ setAnswer({ key, available: false });
+ }
+ });
+ return () => controller.abort();
+ }, [key, modelPath, hfToken]);
+ return answer != null && answer.key === key && answer.available;
+}
diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
index 63053c0c20b..b9a367368c9 100644
--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
@@ -67,6 +67,7 @@ export function applyPerModelConfigToRuntime(
useChatRuntimeStore.setState({
customContextLength: config.customContextLength ?? null,
mlxKvQuant: config.mlxKvQuant ?? null,
+ mlxInt8Prefill: config.mlxInt8Prefill ?? false,
kvCacheDtype: config.kvCacheDtype ?? null,
speculativeType:
normalizeSpeculativeType(config.speculativeType) ??
@@ -135,6 +136,7 @@ export function currentRuntimePerModelConfig(
: null,
kvCacheDtype: s.kvCacheDtype ?? null,
mlxKvQuant: s.mlxKvQuant ?? null,
+ mlxInt8Prefill: s.mlxInt8Prefill ?? false,
speculativeType: normalizeSpeculativeType(s.speculativeType),
specDraftNMax: s.specDraftNMax ?? null,
specDraftCacheDtype: s.specDraftCacheDtype ?? null,
@@ -185,6 +187,7 @@ export function perModelConfigsEqual(
normalizeMaxSeqLength(b.maxSeqLength) &&
(a.kvCacheDtype ?? null) === (b.kvCacheDtype ?? null) &&
(a.mlxKvQuant ?? null) === (b.mlxKvQuant ?? null) &&
+ Boolean(a.mlxInt8Prefill) === Boolean(b.mlxInt8Prefill) &&
speculative(a.speculativeType) === speculative(b.speculativeType) &&
(a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) &&
(a.specDraftCacheDtype ?? null) === (b.specDraftCacheDtype ?? null) &&
diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
index 644d8af2510..c9f86156b76 100644
--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts
+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
@@ -55,6 +55,7 @@ export function loadedConfigSignature(
config.maxSeqLength ?? "",
config.kvCacheDtype ?? "",
config.mlxKvQuant ?? "",
+ config.mlxInt8Prefill ? "1" : "0",
config.speculativeType ?? "",
config.specDraftNMax ?? "",
config.specDraftCacheDtype ?? "",
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index a24a4d37801..a9b8ee4972c 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -26,6 +26,8 @@ export interface PerModelConfig {
maxSeqLength: number | null;
kvCacheDtype: string | null;
mlxKvQuant?: MlxKvQuant | null;
+ /** MLX only: run quantized projections with int8 activations, where the model supports it. */
+ mlxInt8Prefill?: boolean;
speculativeType: string | null;
specDraftNMax: number | null;
/** KV cache dtype for the DRAFT context, sized and quantized independently of kvCacheDtype.
@@ -71,6 +73,7 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {
maxSeqLength: null,
kvCacheDtype: null,
mlxKvQuant: null,
+ mlxInt8Prefill: false,
speculativeType: null,
specDraftNMax: null,
specDraftCacheDtype: null,
@@ -386,8 +389,10 @@ const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";
// would normalize the unknown field straight back out of the record.
// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group
// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,
-// v7 mlxKvQuant
-const STORAGE_SCHEMA_VERSION = 7;
+// v7 mlxKvQuant, v9 mlxInt8Prefill. v8 is skipped: nightly builds stamped it for the reverted custom
+// llama.cpp config (#12725), so a v8 client must not claim to understand an int8 prefill record.
+const STORAGE_SCHEMA_VERSION = 9;
+const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;
const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;
const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;
const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;
@@ -429,6 +434,7 @@ const STORED_CONFIG_FIELDS = new Set([
"maxSeqLength",
"kvCacheDtype",
"mlxKvQuant",
+ "mlxInt8Prefill",
"speculativeType",
"specDraftNMax",
"specDraftCacheDtype",
@@ -1039,6 +1045,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
: null,
maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength),
mlxKvQuant: normalizeMlxKvQuant(partial.mlxKvQuant, partial.mlxKvBits),
+ mlxInt8Prefill: partial.mlxInt8Prefill === true,
kvCacheDtype:
typeof partial.kvCacheDtype === "string" &&
VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype)
@@ -1121,9 +1128,12 @@ function normalize(raw: unknown): PerModelConfig {
* client reconstructs anyway, and stamping every record v4 would put the whole store out of reach.
* The tuning group and the reasoning pair follow the same rule. */
function storedSchemaVersion(normalized: PerModelConfig): number {
- if (normalized.mlxKvQuant != null) {
+ if (normalized.mlxInt8Prefill) {
return STORAGE_SCHEMA_VERSION;
}
+ if (normalized.mlxKvQuant != null) {
+ return PRE_MLX_INT8_PREFILL_SCHEMA_VERSION;
+ }
const hasReasoningBudget =
normalized.reasoningBudget !== -1 || normalized.reasoningBudgetMessage !== "";
if (hasReasoningBudget) {
@@ -1299,6 +1309,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean {
config.maxSeqLength == null &&
(config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype &&
(config.mlxKvQuant ?? null) === DEFAULT_PER_MODEL_CONFIG.mlxKvQuant &&
+ !config.mlxInt8Prefill &&
config.speculativeType === DEFAULT_PER_MODEL_CONFIG.speculativeType &&
config.specDraftNMax == null &&
config.nParallel == null &&
diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.ts
index 7adc2c43560..b8c14f74d31 100644
--- a/studio/frontend/src/features/model-picker/sharing/fields.ts
+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts
@@ -46,6 +46,7 @@ export type SharedConfigKey = Exclude<
| "tensorSplit"
| "maxSeqLength"
| "mlxKvQuant"
+ | "mlxInt8Prefill"
| "engine"
| "enginePrecision"
| "engineParallelism"
diff --git a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
index d03a6709487..707efd9c722 100644
--- a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
+++ b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
@@ -24,7 +24,7 @@ import {
registerStoreStubResolver();
const { storage } = installLocalStorageFake();
-const { fromApiOverride } = await import(
+const { fromApiOverride, toApiOverride } = await import(
"../src/features/model-picker/api/model-overrides.ts"
);
const {
@@ -37,6 +37,14 @@ const {
const MODEL = "unsloth/Model-GGUF";
const VARIANT = "q4_k_m";
+test("int8 prefill reaches the server row only when on", () => {
+ const on = { ...DEFAULT_PER_MODEL_CONFIG, mlxInt8Prefill: true };
+ assert.equal(toApiOverride(on).mlx_int8_prefill, true);
+ assert.equal("mlx_int8_prefill" in toApiOverride(DEFAULT_PER_MODEL_CONFIG), false);
+ assert.equal(fromApiOverride({ mlx_int8_prefill: true }).mlxInt8Prefill, true);
+ assert.equal(fromApiOverride({}, on).mlxInt8Prefill, true);
+});
+
test("a server row holding the superseded cache width hydrates as the single choice", () => {
assert.equal(fromApiOverride({ mlx_kv_bits: 8 }).mlxKvQuant, "8");
assert.equal(fromApiOverride({ mlx_kv_quant: "tq-4", mlx_kv_bits: 8 }).mlxKvQuant, "tq-4");
diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
index 96de26b75f3..ac02ba4eea3 100644
--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
@@ -204,12 +204,12 @@ const ROWS: Row[] = [
mlxRequest: 32768,
transformersRequest: 32768,
note:
- "Not a future record: STORAGE_SCHEMA_VERSION is 7, so v4 is a v4-client record " +
+ "Not a future record: STORAGE_SCHEMA_VERSION is 9, so v4 is a v4-client record " +
"and reads normally.",
},
{
- name: "version 8 (genuinely future)",
- raw: { version: 8, customContextLength: 32768 },
+ name: "version 10 (genuinely future)",
+ raw: { version: 10, customContextLength: 32768 },
normalizedPin: null,
rawPin: 32768,
isDefault: true,
@@ -330,6 +330,7 @@ test("a patched pin round-trips through storage on both backends", () => {
test("both pin shapes are stamped version 1, so neither is distinguishable by version", () => {
assert.equal(stampedVersion({ customContextLength: 32768 }), 1);
assert.equal(stampedVersion({ maxSeqLength: 32768 }), 1);
+ // The current client's forwards guard is `version > 9`, and v1 invites any client
// back to v1 to rewrite the record.
assert.equal(
stage({ version: 1, customContextLength: 32768 }).remembered,
@@ -345,6 +346,14 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by ve
);
assert.equal(
stage({ version: 8, customContextLength: 32768 }).remembered,
+ true,
+ );
+ assert.equal(
+ stage({ version: 9, customContextLength: 32768 }).remembered,
+ true,
+ );
+ assert.equal(
+ stage({ version: 10, customContextLength: 32768 }).remembered,
false,
);
});
@@ -357,6 +366,15 @@ test("a stored mlxKvQuant stamps a version older builds refuse", () => {
assert.equal(stampedVersion({ mlxKvQuant: "tq-4", loadMode: "mmap" }), 7);
});
+test("int8 prefill defaults off, and only an enabled one stamps v9", () => {
+ assert.equal(normalizePerModelConfig({ version: 8 }).mlxInt8Prefill, false);
+ assert.equal(normalizePerModelConfig({ mlxInt8Prefill: "true" }).mlxInt8Prefill, false);
+ assert.equal(isDefaultConfig({ ...DEFAULT_PER_MODEL_CONFIG, mlxInt8Prefill: true }), false);
+ assert.equal(stampedVersion({ mlxInt8Prefill: false, mlxKvQuant: "4" }), 7);
+ assert.equal(stampedVersion({ mlxInt8Prefill: true }), 9);
+ assert.equal(stage({ mlxInt8Prefill: true }).config.mlxInt8Prefill, true);
+});
+
/**
* The old model-config page's read rule for a NON-GGUF target, transcribed from
diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
index b192e25240e..5d20fb28c19 100644
--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
@@ -280,6 +280,12 @@ const FIELDS: FieldCase[] = [
same: true,
different: false,
},
+ {
+ key: "mlxInt8Prefill",
+ statusKey: "mlx_int8_prefill_requested",
+ same: true,
+ different: false,
+ },
];
for (const [accelerator, base] of Object.entries(ACCELERATORS)) {
diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.ts
index 0565ac1ceac..d7b6b79d66c 100644
--- a/studio/frontend/tests/server-tuning-settings.test.ts
+++ b/studio/frontend/tests/server-tuning-settings.test.ts
@@ -113,11 +113,19 @@ test("a record only claims the new schema version when it carries one", () => {
// toStoredConfig stamps the OLDEST version that understands every field
// present, so an older client can still rewrite a record it fully knows.
const source = readSrc("features/model-picker/model-config/per-model-config.ts");
- assert.match(source, /const STORAGE_SCHEMA_VERSION = 7;/);
+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 9;/);
+ assert.match(source, /const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;/);
assert.match(source, /const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;/);
assert.match(source, /const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;/);
assert.match(source, /const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;/);
- assert.match(source, /mlxKvQuant != null\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);
+ assert.match(
+ source,
+ /normalized\.mlxInt8Prefill\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/,
+ );
+ assert.match(
+ source,
+ /mlxKvQuant != null\)\s*\{\s*return PRE_MLX_INT8_PREFILL_SCHEMA_VERSION;/,
+ );
assert.match(
source,
/hasReasoningBudget\)\s*\{\s*return PRE_MLX_KV_QUANT_SCHEMA_VERSION;/,