Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): price DeepSeek-V4.1 oQ quants as unsloth-zoo loads them (#15508)
Long Yixinge798a579bdad
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex ba21d690782..6661d033e42 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -455,6 +455,8 @@ def _whole_model(config: dict, dtype):raise ValueError("this checkpoint carries its own model module")if _builds_on_mlx_vlm(config):from mlx_vlm.utils import get_model_and_args++ _install_oq_renames(config)arch = get_model_and_args(config)[0]model = arch.Model(_loader_config(arch, config))else:@@ -1149,11 +1151,43 @@ def _lazy_checkpoint(model_dir: str, config: dict):with weightless_forward():if _builds_on_mlx_vlm(config):from mlx_vlm.utils import load_model- return load_model(Path(model_dir), lazy = True, strict = False)++ _install_oq_renames(config)+ view, owner = _loader_view(model_dir, config)+ if owner is None:+ return load_model(Path(view), lazy = True, strict = False)+ model = owner.call(load_model, Path(view), lazy = True, strict = False)+ owner.transfer_to(model)+ return modelfrom mlx_lm.utils import load_modelreturn load_model(Path(model_dir), lazy = True, strict = False)[0]+def _loader_view(model_dir: str, config: dict) -> tuple:+ """``(path, owner)``: unsloth_zoo's config view of ``model_dir`` (owner None without unsloth_zoo)."""+ import io++ try:+ from unsloth_zoo.mlx.loader import (+ _MLXVLMConfigViewOwner,+ _materialize_mlx_vlm_config_override,+ )+ except Exception:+ return model_dir, None+ with contextlib.redirect_stdout(io.StringIO()):+ view, _ = _materialize_mlx_vlm_config_override(model_dir, config)+ return view, _MLXVLMConfigViewOwner(view, model_dir)+++def _install_oq_renames(config: dict) -> None:+ """unsloth_zoo's sanitize renames for DeepSeek-V4.1 oQ quants, as its load installs them."""+ try:+ from unsloth_zoo.mlx.loader import _ensure_deepseek_v41_omlx_layout+ except Exception:+ return+ _ensure_deepseek_v41_omlx_layout(config.get("model_type"))++@functools.lru_cache(maxsize = 4)def _offloadable_cached(model_dir: str, fingerprint: tuple, cast_dtype, on_mlx_vlm: bool):from unsloth_zoo.mlx.offload import offloadablediff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex 34cea441659..d54443552f0 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -3826,6 +3826,138 @@ def test_the_offload_probe_never_evaluates_a_converted_checkpoint(tmp_path, monkmm._lazy_checkpoint(str(tmp_path), config)+def test_a_deepseek_v41_oq_quant_is_priced_as_unsloth_zoo_loads_it(tmp_path, monkeypatch):+ if not _HAVE_MLX:+ pytest.skip("needs MLX to build and load a checkpoint")+ v41 = pytest.importorskip("mlx_vlm.models.deepseek_v41.deepseek_v41")+ loader = pytest.importorskip("unsloth_zoo.mlx.loader")+ if not hasattr(loader, "_ensure_deepseek_v41_omlx_layout"):+ pytest.skip("this unsloth_zoo loads no oQ quant")+ import gc+ import re++ import mlx.core as mx+ import mlx.nn as nn+ from mlx.utils import tree_flatten+ from mlx_vlm.models.deepseek_v41.config import ModelConfig++ text = dict(+ vocab_size = 512,+ hidden_size = 256,+ moe_intermediate_size = 128,+ num_hidden_layers = 4,+ num_attention_heads = 4,+ head_dim = 64,+ qk_rope_head_dim = 32,+ q_lora_rank = 128,+ o_lora_rank = 64,+ o_groups = 2,+ n_routed_experts = 16,+ num_experts_per_tok = 2,+ sliding_window = 16,+ compress_ratios = [0, 2, 1, 1],+ kv_source_layer_ids = [1, 2],+ index_source_layer_ids = [1, 2, 3],+ index_n_heads = 2,+ index_head_dim = 64,+ candidate_source_layer_id = 2,+ engram_layer_ids = [1],+ engram_num_embeddings = [1024],+ engram_n_heads = 2,+ engram_head_dim = 64,+ )+ config = {+ "model_type": "deepseek_v41",+ "text_config": text,+ "vision_config": {"num_hidden_layers": 1, "hidden_size": 64},+ }+ model = v41.Model(ModelConfig.from_dict(json.loads(json.dumps(config))))++ def oq2e(path, module):+ if (+ not hasattr(module, "to_quantized")+ or path.startswith(("vision", "aligner"))+ or path.endswith(("head", "embed_tokens", "ffn.gate", "wo_a"))+ ):+ return False+ if path.endswith("engram.wkv"):+ return {"group_size": 32, "bits": 8, "mode": "mxfp8"}+ if path.endswith("engram.embed"):+ return {"group_size": 32, "bits": 4, "mode": "affine"}+ return {"group_size": 64, "bits": 2 if ".switch_mlp." in path else 8, "mode": "affine"}++ specs = {}+ nn.quantize(+ model, class_predicate = lambda path, module: specs.setdefault(path, oq2e(path, module))+ )+ names = {"gate_proj": "w1", "down_proj": "w2", "up_proj": "w3"}++ def omlx(key):+ key = key.replace("language_model.embed_tokens.", "language_model.embed.").replace(+ ".switch_mlp.", ".experts."+ )+ key = re.sub(r"(experts)\.(\w+_proj)\.", lambda m: f"{m[1]}.{names[m[2]]}.", key)+ return re.sub(r"^(vision\.blocks\.\d+)\.mlp\.", r"\1.ffn.", key)++ # bf16 as stored, beside the float32 the model is built in: a name sanitize misses keeps the latter.+ params = {+ k: v.astype(mx.bfloat16)+ if v.dtype == mx.float32 and "hc_" not in k and "gate.bias" not in k+ else v+ for k, v in tree_flatten(model.parameters())+ }+ stored = {+ omlx(k): v.reshape(-1, v.shape[-1]) if k.endswith("wo_a.weight") else v+ for k, v in params.items()+ }+ stored["language_model.mtp.0.ffn.experts.w1.weight"] = mx.zeros(+ (16, 128, 32), mx.uint32+ ) # a DSpark stage+ mx.save_safetensors(str(tmp_path / "model.safetensors"), stored)+ # No "quantization": only unsloth_zoo's reading of the config supplies it.+ quantized = {+ omlx(f"{p}.weight").removesuffix(".weight"): s+ if s["mode"] == "affine"+ else {"bits": s["bits"], "mode": s["mode"]}+ for p, s in specs.items()+ if s and not p.endswith("engram.embed")+ }+ quantized["language_model.mtp.0.ffn.experts.w1"] = {"bits": 4, "mode": "mxfp4"}+ table = "language_model.layers.1.engram.embed"+ keys = {+ "weight_key": f"{table}.weight",+ "scale_key": f"{table}.scales",+ "bias_key": f"{table}.biases",+ }+ config["omlx_deepseek_v41"] = {+ "version": 1,+ "quantized_modules": quantized,+ "engram_tables": {table: {**keys, "bits": 4, "group_size": 32, "mode": "affine"}},+ }+ (tmp_path / "config.json").write_text(json.dumps(config))++ model_dir = str(tmp_path)+ priced = mm._snapshot_config(model_dir)+ # Installed process-wide: reset between checks so each must install it itself.+ sanitize = v41.Model.sanitize+ if getattr(sanitize, "_unsloth_omlx_layout", False):+ pytest.skip("an earlier load in this process installed the oQ layout")+ monkeypatch.setattr(v41.Model, "sanitize", sanitize)+ assert mm._load_is_refused(model_dir, priced, load_in_4bit = True) is None+ v41.Model.sanitize = sanitize+ experts = sum(v.nbytes for k, v in params.items() if ".switch_mlp." in k)+ found = mm._offloadable(model_dir, priced)+ assert (found["expert_bytes"], found["experts"]) == (experts, None)+ assert mm.mlx_offloadable_parts(model_dir) == ("experts",)+ # The view carrying that config lives exactly as long as the model mapping its shards.+ lazy = mm._lazy_checkpoint(model_dir, priced)+ (view,) = lazy._unsloth_mlx_config_view_paths+ assert os.path.isfile(os.path.join(view, "model.safetensors"))+ del lazy+ gc.collect()+ assert not os.path.exists(view)++try:import importlib