Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
studio: keep MTP on for GLM-5.3-Flash (#10119)
Daniel Han651ec448392c
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex c5760e5005e..ae3be1fecb3 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -3376,6 +3376,23 @@ def _auto_mode_drops_mtp(return req_mode == "auto" and size_b is not None and size_b < _MTP_MIN_SIZE_B+# MLA archs whose MTP context covers only the NextN block instead of duplicating the+# trunk KV: glm5next holds 4+3 MiB over one layer where its trunk holds 48+36 over+# twelve. That one fact is why Auto keeps MTP (gate below) and why the fit must not+# reserve the copy (_estimate_mtp_overhead_bytes). Not "glm5-next": no NextN graph.+_MLA_MTP_FAST_ARCHS = frozenset({"glm5next"})+++def _arch_has_fast_mla_mtp(architecture: Optional[str]) -> bool:+ """Whether this MLA architecture's embedded MTP head is worth promoting in Auto."""+ return bool(architecture) and str(architecture).strip().lower() in _MLA_MTP_FAST_ARCHS+++def _arch_mtp_skips_target_kv_copy(architecture: Optional[str]) -> bool:+ """Whether this MLA architecture's MTP context skips the duplicated target KV."""+ return _arch_has_fast_mla_mtp(architecture)++def _mla_mtp_auto_enabled() -> bool:"""Whether Auto may pick embedded MTP for an MLA model (GLM-5.2/DeepSeek/Kimi).@@ -4068,6 +4085,9 @@ _TARGET_KV_EXCLUDES_NEXTN_ARCHS = frozenset(# MLA/DSA trunk with a dense MTP head, llama-model.cpp:2129"glm-dsa","deepseek32",+ # Hybrid KDA + DSA trunk; both GLM-5.3-Flash ports filter blk.45 out+ "glm5next",+ "glm5-next",# Plain attention trunk with an explicit nextn filter, llama-model.cpp:2356"step35","hy_v3",@@ -11335,6 +11355,17 @@ class LlamaCppBackend:n_embd_s = head_dim * head_dim * n_headreturn int(n_recurrent * (n_embd_r + n_embd_s) * 4 * max(1, n_parallel))+ def _rollback_state_bytes(self, n_parallel: int = 1) -> int:+ """One target-context rollback snapshot, whichever recurrent family this is.++ Both callers price the same thing (llama.cpp's `1 seqs N rs_seq`) and both+ used to reach for the Mamba helper alone, which answers 0 for a KDA hybrid+ and silently dropped the reserve. Route every caller through here.+ """+ return self._mamba_recurrent_state_bytes(n_parallel) or self._recurrent_state_bytes(+ n_parallel+ )+def _target_kv_excludes_nextn(self) -> bool:"""Whether this model's TARGET KV cache skips the embedded MTP blocks.@@ -11837,8 +11868,14 @@ class LlamaCppBackend:# separate-drafter spec modes (draft-simple/draft-eagle3) load a small# distinct drafter with its own KV -- already counted in draft_kv/weights --# rather than duplicating the target, so they must not be charged for it.+ # Third gate: a NextN-only MTP context allocates no copy, and charging one+ # trips drafter_no_vram, losing the MTP being reserved for.target_ctx_copy = 0- if mtp_keeps_target_ctx and self._kv_lora_rank is not None:+ if (+ mtp_keeps_target_ctx+ and self._kv_lora_rank is not None+ and not _arch_mtp_skips_target_kv_copy(getattr(self, "_architecture", None))+ ):target_ctx_copy = self._estimate_kv_cache_bytes(n_ctx,"f16",@@ -11856,8 +11893,7 @@ class LlamaCppBackend:# dominant hidden cost on Qwen3.5/3.8 at multiple parallel slots.target_recurrent_copies = 0if target_rollback and spec_draft_n_max > 0:- base_recurrent = self._mamba_recurrent_state_bytes(n_parallel)- target_recurrent_copies = base_recurrent * spec_draft_n_max+ target_recurrent_copies = self._rollback_state_bytes(n_parallel) * spec_draft_n_maxif draft_kv is None:# KV unsized (exotic/remote drafter): still reserve known weights + any# MLA target copy so a large config can't launch over budget (the small@@ -19048,9 +19084,9 @@ class LlamaCppBackend:# displaced does not run. The flat fraction below is gated on# this; the byte-accurate callback was not, so the fit went on# charging VRAM no drafter allocates.- # One term survives: a Hybrid Mamba target's recurrent rollback- # snapshots sit in the TARGET context, so pinning the drafter to- # CPU does not move them. Charge those alone. Flat in ctx (the+ # One term survives: a recurrent target's rollback snapshots sit+ # in the TARGET context, so pinning the drafter to CPU does not+ # move them. Charge those alone. Flat in ctx (the# state is per-slot), hence the same _np/_n_ubatch keywords the# replaced callback takes, so _mtp_bytes can still re-price slots.def _cpu_draft_target_state(@@ -19062,7 +19098,7 @@ class LlamaCppBackend:) -> int:if not _rollback or _n <= 0:return 0- return self._mamba_recurrent_state_bytes(_np) * _n+ return self._rollback_state_bytes(_np) * _nmtp_overhead_fn = (_cpu_draft_target_state@@ -24149,6 +24185,7 @@ class LlamaCppBackend:and self._kv_lora_rank is not Noneand not bool(mtp_draft_path)and not _mla_mtp_auto_enabled()+ and not _arch_has_fast_mla_mtp(getattr(self, "_architecture", None)))if user_owns_spec_type:diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.pyindex 6eb4242a067..d4fef26ebe7 100644--- a/studio/backend/tests/test_llama_cpp_mtp_detection.py+++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py@@ -75,6 +75,8 @@ from core.inference.llama_cpp import (_extra_args_set_spec_type,_is_mtp_model_name,_kv_unified_from_args,+ _TARGET_KV_EXCLUDES_NEXTN_ARCHS,+ _arch_has_fast_mla_mtp,_mla_mtp_auto_enabled,_swa_full_from_args_or_env,)@@ -2278,6 +2280,60 @@ def test_reload_forced_mtp_bounces_auto_mla():)+# glm5next matches the MLA gate on metadata, but its MTP is 1.31x faster, not slower.+_GLM5NEXT_MODEL = "unsloth/GLM-5.3-Flash-GGUF"+++def test_auto_glm5next_keeps_draft_mtp(monkeypatch):+ backend = _mla_resolver_backend(monkeypatch)+ backend._architecture = "glm5next"+ flags = backend._build_speculative_flags(+ speculative_type = "auto",+ spec_draft_n_max = None,+ extra_args = None,+ model_identifier = _GLM5NEXT_MODEL,+ model_path = None,+ gpus = True,+ binary = "/fake/llama-server",+ )+ parsed = _flags_dict(flags)+ assert parsed.get("--spec-type") == "draft-mtp"+ assert backend.speculative_type == "draft-mtp"+ assert backend.spec_fallback_reason != "mla_mtp_disabled"+++def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):+ # The "glm5-next" port never builds the NextN graph: no spillover.+ backend = _mla_resolver_backend(monkeypatch)+ backend._architecture = "glm5-next"+ flags = backend._build_speculative_flags(+ speculative_type = "auto",+ spec_draft_n_max = None,+ extra_args = None,+ model_identifier = _GLM5NEXT_MODEL,+ model_path = None,+ gpus = True,+ binary = "/fake/llama-server",+ )+ assert _flags_dict(flags).get("--spec-type") != "draft-mtp"+ assert backend.spec_fallback_reason == "mla_mtp_disabled"+++def test_arch_has_fast_mla_mtp_is_case_and_space_tolerant():+ assert _arch_has_fast_mla_mtp("glm5next")+ assert _arch_has_fast_mla_mtp(" GLM5Next ")+ assert not _arch_has_fast_mla_mtp("glm5-next")+ assert not _arch_has_fast_mla_mtp("glm-dsa")+ assert not _arch_has_fast_mla_mtp(None)+ assert not _arch_has_fast_mla_mtp("")+++def test_glm5next_target_kv_excludes_nextn():+ # Both ports filter il < n_layer() && !is_recr(il), so blk.45 gets no target KV.+ assert "glm5next" in _TARGET_KV_EXCLUDES_NEXTN_ARCHS+ assert "glm5-next" in _TARGET_KV_EXCLUDES_NEXTN_ARCHS++# ── Full named-repo resolver matrix (the shipping Unsloth families) ─────## Locks auto / off / forced-mtp routing for every Qwen3.5 (MTP + plain) anddiff --git a/studio/backend/tests/test_mtp_mla_target_ctx.py b/studio/backend/tests/test_mtp_mla_target_ctx.pyindex c38e4c7b62c..5c7e60e41f3 100644--- a/studio/backend/tests/test_mtp_mla_target_ctx.py+++ b/studio/backend/tests/test_mtp_mla_target_ctx.py@@ -188,6 +188,95 @@ class TestMlaTargetCtxReserve:assert mtp == separate + b._estimate_kv_cache_bytes(ctx, "f16")assert mtp > separate+ def test_one_layer_mtp_arch_drops_target_copy(self):+ # Charging the absent copy trips drafter_no_vram, dropping the MTP itself.+ b = _make_mla_backend()+ b._architecture = "glm5next"+ other = _make_mla_backend()+ other._architecture = "glm-dsa" # same dims, still pays the copy+ ctx = 262144+ assert b._estimate_mtp_overhead_bytes(ctx) == b._mtp_draft_kv_bytes(ctx)+ assert other._estimate_mtp_overhead_bytes(ctx) == (+ b._mtp_draft_kv_bytes(ctx) + b._estimate_kv_cache_bytes(ctx, "f16")+ )+ # The "glm5-next" port builds no NextN graph, so it keeps the safe default.+ hyphenated = _make_mla_backend()+ hyphenated._architecture = "glm5-next"+ assert hyphenated._estimate_mtp_overhead_bytes(ctx) == other._estimate_mtp_overhead_bytes(+ ctx+ )+++class TestKdaRollbackReserve:+ """A KDA hybrid pays draft rollback copies the Mamba helper cannot see.++ Dims are GLM-5.3-Flash UD-IQ1_S as shipped (34 recurrent layers of 46,+ kda.head_dim 128, head_count 64, ssm.conv_kernel 4). llama.cpp allocates+ 582.25 MiB for `1 seqs 3 rs_seq`, i.e. 4 x 145.5625 MiB, so the MTP share+ is the 3 extra copies and the reserve must carry them.+ """++ MIB = 1024**2+ PER_SEQ = 145.5625 # MiB, and the size llama.cpp logs per context checkpoint++ def _kda(self):+ b = _make_mla_backend()+ b._architecture = "glm5next"+ # Every 4th block is DSA, and so is blk.45 (NextN): 34 recurrent of 46.+ b._n_kv_heads_by_layer = [1 if ((i + 1) % 4 == 0 or i == 45) else 0 for i in range(46)]+ b._n_layers = 46+ b._n_heads = 64+ b._kda_head_dim = 128+ b._ssm_conv_kernel = 4+ return b++ def test_base_state_matches_llama_cpp(self):+ b = self._kda()+ assert b._mamba_recurrent_state_bytes(1) == 0 # no SSM fields: the gap+ assert b._recurrent_state_bytes(1) / self.MIB == pytest.approx(self.PER_SEQ)++ def test_rollback_copies_are_reserved(self):+ b = self._kda()+ base = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 0)+ with_draft = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 3)+ assert (with_draft - base) / self.MIB == pytest.approx(3 * self.PER_SEQ)++ def test_rollback_scales_with_slots(self):+ b = self._kda()+ one = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 3, n_parallel = 1)+ four = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 3, n_parallel = 4)+ assert four > one++ def test_cpu_pinned_drafter_keeps_target_rollback(self):+ # Pinning the drafter to CPU does not move the target's snapshots, and the+ # loader drops its whole rollback-only callback when this reads 0.+ b = self._kda()+ assert b._rollback_state_bytes(1) / self.MIB == pytest.approx(self.PER_SEQ)+ assert b._rollback_state_bytes(4) == 4 * b._rollback_state_bytes(1)++ def test_rollback_helper_prefers_mamba(self):+ b = _make_mla_backend()+ b._ssm_inner_size = 6144+ b._ssm_state_size = 128+ b._ssm_group_count = 1+ b._ssm_conv_kernel = 4+ b._full_attention_interval = 4+ assert b._rollback_state_bytes(1) == b._mamba_recurrent_state_bytes(1)++ def test_mamba_path_unchanged(self):+ # The KDA fallback must not shadow or double-count the Mamba helper.+ b = _make_mla_backend()+ b._ssm_inner_size = 6144+ b._ssm_state_size = 128+ b._ssm_group_count = 1+ b._ssm_conv_kernel = 4+ b._full_attention_interval = 4+ assert b._mamba_recurrent_state_bytes(1) > 0+ delta = b._estimate_mtp_overhead_bytes(+ 65536, spec_draft_n_max = 2+ ) - b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 0)+ assert delta == 2 * b._mamba_recurrent_state_bytes(1)+class TestMlaFitPreventsOom:"""The corrected reserve must actually lower the auto-fit context so the