Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

studio: keep MTP on for GLM-5.3-Flash (#10119)

Daniel Han651ec448392c
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index c5760e5005e..ae3be1fecb3 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -3376,6 +3376,23 @@ def _auto_mode_drops_mtp(
return req_mode == "auto" and size_b is not None and size_b < _MTP_MIN_SIZE_B
+# MLA archs whose MTP context covers only the NextN block instead of duplicating the
+# trunk KV: glm5next holds 4+3 MiB over one layer where its trunk holds 48+36 over
+# twelve. That one fact is why Auto keeps MTP (gate below) and why the fit must not
+# reserve the copy (_estimate_mtp_overhead_bytes). Not "glm5-next": no NextN graph.
+_MLA_MTP_FAST_ARCHS = frozenset({"glm5next"})
+
+
+def _arch_has_fast_mla_mtp(architecture: Optional[str]) -> bool:
+ """Whether this MLA architecture's embedded MTP head is worth promoting in Auto."""
+ return bool(architecture) and str(architecture).strip().lower() in _MLA_MTP_FAST_ARCHS
+
+
+def _arch_mtp_skips_target_kv_copy(architecture: Optional[str]) -> bool:
+ """Whether this MLA architecture's MTP context skips the duplicated target KV."""
+ return _arch_has_fast_mla_mtp(architecture)
+
+
def _mla_mtp_auto_enabled() -> bool:
"""Whether Auto may pick embedded MTP for an MLA model (GLM-5.2/DeepSeek/Kimi).
@@ -4068,6 +4085,9 @@ _TARGET_KV_EXCLUDES_NEXTN_ARCHS = frozenset(
# MLA/DSA trunk with a dense MTP head, llama-model.cpp:2129
"glm-dsa",
"deepseek32",
+ # Hybrid KDA + DSA trunk; both GLM-5.3-Flash ports filter blk.45 out
+ "glm5next",
+ "glm5-next",
# Plain attention trunk with an explicit nextn filter, llama-model.cpp:2356
"step35",
"hy_v3",
@@ -11335,6 +11355,17 @@ class LlamaCppBackend:
n_embd_s = head_dim * head_dim * n_head
return int(n_recurrent * (n_embd_r + n_embd_s) * 4 * max(1, n_parallel))
+ def _rollback_state_bytes(self, n_parallel: int = 1) -> int:
+ """One target-context rollback snapshot, whichever recurrent family this is.
+
+ Both callers price the same thing (llama.cpp's `1 seqs N rs_seq`) and both
+ used to reach for the Mamba helper alone, which answers 0 for a KDA hybrid
+ and silently dropped the reserve. Route every caller through here.
+ """
+ return self._mamba_recurrent_state_bytes(n_parallel) or self._recurrent_state_bytes(
+ n_parallel
+ )
+
def _target_kv_excludes_nextn(self) -> bool:
"""Whether this model's TARGET KV cache skips the embedded MTP blocks.
@@ -11837,8 +11868,14 @@ class LlamaCppBackend:
# separate-drafter spec modes (draft-simple/draft-eagle3) load a small
# distinct drafter with its own KV -- already counted in draft_kv/weights --
# rather than duplicating the target, so they must not be charged for it.
+ # Third gate: a NextN-only MTP context allocates no copy, and charging one
+ # trips drafter_no_vram, losing the MTP being reserved for.
target_ctx_copy = 0
- if mtp_keeps_target_ctx and self._kv_lora_rank is not None:
+ if (
+ mtp_keeps_target_ctx
+ and self._kv_lora_rank is not None
+ and not _arch_mtp_skips_target_kv_copy(getattr(self, "_architecture", None))
+ ):
target_ctx_copy = self._estimate_kv_cache_bytes(
n_ctx,
"f16",
@@ -11856,8 +11893,7 @@ class LlamaCppBackend:
# dominant hidden cost on Qwen3.5/3.8 at multiple parallel slots.
target_recurrent_copies = 0
if target_rollback and spec_draft_n_max > 0:
- base_recurrent = self._mamba_recurrent_state_bytes(n_parallel)
- target_recurrent_copies = base_recurrent * spec_draft_n_max
+ target_recurrent_copies = self._rollback_state_bytes(n_parallel) * spec_draft_n_max
if draft_kv is None:
# KV unsized (exotic/remote drafter): still reserve known weights + any
# MLA target copy so a large config can't launch over budget (the small
@@ -19048,9 +19084,9 @@ class LlamaCppBackend:
# displaced does not run. The flat fraction below is gated on
# this; the byte-accurate callback was not, so the fit went on
# charging VRAM no drafter allocates.
- # One term survives: a Hybrid Mamba target's recurrent rollback
- # snapshots sit in the TARGET context, so pinning the drafter to
- # CPU does not move them. Charge those alone. Flat in ctx (the
+ # One term survives: a recurrent target's rollback snapshots sit
+ # in the TARGET context, so pinning the drafter to CPU does not
+ # move them. Charge those alone. Flat in ctx (the
# state is per-slot), hence the same _np/_n_ubatch keywords the
# replaced callback takes, so _mtp_bytes can still re-price slots.
def _cpu_draft_target_state(
@@ -19062,7 +19098,7 @@ class LlamaCppBackend:
) -> int:
if not _rollback or _n <= 0:
return 0
- return self._mamba_recurrent_state_bytes(_np) * _n
+ return self._rollback_state_bytes(_np) * _n
mtp_overhead_fn = (
_cpu_draft_target_state
@@ -24149,6 +24185,7 @@ class LlamaCppBackend:
and self._kv_lora_rank is not None
and not bool(mtp_draft_path)
and not _mla_mtp_auto_enabled()
+ and not _arch_has_fast_mla_mtp(getattr(self, "_architecture", None))
)
if user_owns_spec_type:
diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.py
index 6eb4242a067..d4fef26ebe7 100644
--- a/studio/backend/tests/test_llama_cpp_mtp_detection.py
+++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py
@@ -75,6 +75,8 @@ from core.inference.llama_cpp import (
_extra_args_set_spec_type,
_is_mtp_model_name,
_kv_unified_from_args,
+ _TARGET_KV_EXCLUDES_NEXTN_ARCHS,
+ _arch_has_fast_mla_mtp,
_mla_mtp_auto_enabled,
_swa_full_from_args_or_env,
)
@@ -2278,6 +2280,60 @@ def test_reload_forced_mtp_bounces_auto_mla():
)
+# glm5next matches the MLA gate on metadata, but its MTP is 1.31x faster, not slower.
+_GLM5NEXT_MODEL = "unsloth/GLM-5.3-Flash-GGUF"
+
+
+def test_auto_glm5next_keeps_draft_mtp(monkeypatch):
+ backend = _mla_resolver_backend(monkeypatch)
+ backend._architecture = "glm5next"
+ flags = backend._build_speculative_flags(
+ speculative_type = "auto",
+ spec_draft_n_max = None,
+ extra_args = None,
+ model_identifier = _GLM5NEXT_MODEL,
+ model_path = None,
+ gpus = True,
+ binary = "/fake/llama-server",
+ )
+ parsed = _flags_dict(flags)
+ assert parsed.get("--spec-type") == "draft-mtp"
+ assert backend.speculative_type == "draft-mtp"
+ assert backend.spec_fallback_reason != "mla_mtp_disabled"
+
+
+def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):
+ # The "glm5-next" port never builds the NextN graph: no spillover.
+ backend = _mla_resolver_backend(monkeypatch)
+ backend._architecture = "glm5-next"
+ flags = backend._build_speculative_flags(
+ speculative_type = "auto",
+ spec_draft_n_max = None,
+ extra_args = None,
+ model_identifier = _GLM5NEXT_MODEL,
+ model_path = None,
+ gpus = True,
+ binary = "/fake/llama-server",
+ )
+ assert _flags_dict(flags).get("--spec-type") != "draft-mtp"
+ assert backend.spec_fallback_reason == "mla_mtp_disabled"
+
+
+def test_arch_has_fast_mla_mtp_is_case_and_space_tolerant():
+ assert _arch_has_fast_mla_mtp("glm5next")
+ assert _arch_has_fast_mla_mtp(" GLM5Next ")
+ assert not _arch_has_fast_mla_mtp("glm5-next")
+ assert not _arch_has_fast_mla_mtp("glm-dsa")
+ assert not _arch_has_fast_mla_mtp(None)
+ assert not _arch_has_fast_mla_mtp("")
+
+
+def test_glm5next_target_kv_excludes_nextn():
+ # Both ports filter il < n_layer() && !is_recr(il), so blk.45 gets no target KV.
+ assert "glm5next" in _TARGET_KV_EXCLUDES_NEXTN_ARCHS
+ assert "glm5-next" in _TARGET_KV_EXCLUDES_NEXTN_ARCHS
+
+
# ── Full named-repo resolver matrix (the shipping Unsloth families) ─────
#
# Locks auto / off / forced-mtp routing for every Qwen3.5 (MTP + plain) and
diff --git a/studio/backend/tests/test_mtp_mla_target_ctx.py b/studio/backend/tests/test_mtp_mla_target_ctx.py
index c38e4c7b62c..5c7e60e41f3 100644
--- a/studio/backend/tests/test_mtp_mla_target_ctx.py
+++ b/studio/backend/tests/test_mtp_mla_target_ctx.py
@@ -188,6 +188,95 @@ class TestMlaTargetCtxReserve:
assert mtp == separate + b._estimate_kv_cache_bytes(ctx, "f16")
assert mtp > separate
+ def test_one_layer_mtp_arch_drops_target_copy(self):
+ # Charging the absent copy trips drafter_no_vram, dropping the MTP itself.
+ b = _make_mla_backend()
+ b._architecture = "glm5next"
+ other = _make_mla_backend()
+ other._architecture = "glm-dsa" # same dims, still pays the copy
+ ctx = 262144
+ assert b._estimate_mtp_overhead_bytes(ctx) == b._mtp_draft_kv_bytes(ctx)
+ assert other._estimate_mtp_overhead_bytes(ctx) == (
+ b._mtp_draft_kv_bytes(ctx) + b._estimate_kv_cache_bytes(ctx, "f16")
+ )
+ # The "glm5-next" port builds no NextN graph, so it keeps the safe default.
+ hyphenated = _make_mla_backend()
+ hyphenated._architecture = "glm5-next"
+ assert hyphenated._estimate_mtp_overhead_bytes(ctx) == other._estimate_mtp_overhead_bytes(
+ ctx
+ )
+
+
+class TestKdaRollbackReserve:
+ """A KDA hybrid pays draft rollback copies the Mamba helper cannot see.
+
+ Dims are GLM-5.3-Flash UD-IQ1_S as shipped (34 recurrent layers of 46,
+ kda.head_dim 128, head_count 64, ssm.conv_kernel 4). llama.cpp allocates
+ 582.25 MiB for `1 seqs 3 rs_seq`, i.e. 4 x 145.5625 MiB, so the MTP share
+ is the 3 extra copies and the reserve must carry them.
+ """
+
+ MIB = 1024**2
+ PER_SEQ = 145.5625 # MiB, and the size llama.cpp logs per context checkpoint
+
+ def _kda(self):
+ b = _make_mla_backend()
+ b._architecture = "glm5next"
+ # Every 4th block is DSA, and so is blk.45 (NextN): 34 recurrent of 46.
+ b._n_kv_heads_by_layer = [1 if ((i + 1) % 4 == 0 or i == 45) else 0 for i in range(46)]
+ b._n_layers = 46
+ b._n_heads = 64
+ b._kda_head_dim = 128
+ b._ssm_conv_kernel = 4
+ return b
+
+ def test_base_state_matches_llama_cpp(self):
+ b = self._kda()
+ assert b._mamba_recurrent_state_bytes(1) == 0 # no SSM fields: the gap
+ assert b._recurrent_state_bytes(1) / self.MIB == pytest.approx(self.PER_SEQ)
+
+ def test_rollback_copies_are_reserved(self):
+ b = self._kda()
+ base = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 0)
+ with_draft = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 3)
+ assert (with_draft - base) / self.MIB == pytest.approx(3 * self.PER_SEQ)
+
+ def test_rollback_scales_with_slots(self):
+ b = self._kda()
+ one = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 3, n_parallel = 1)
+ four = b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 3, n_parallel = 4)
+ assert four > one
+
+ def test_cpu_pinned_drafter_keeps_target_rollback(self):
+ # Pinning the drafter to CPU does not move the target's snapshots, and the
+ # loader drops its whole rollback-only callback when this reads 0.
+ b = self._kda()
+ assert b._rollback_state_bytes(1) / self.MIB == pytest.approx(self.PER_SEQ)
+ assert b._rollback_state_bytes(4) == 4 * b._rollback_state_bytes(1)
+
+ def test_rollback_helper_prefers_mamba(self):
+ b = _make_mla_backend()
+ b._ssm_inner_size = 6144
+ b._ssm_state_size = 128
+ b._ssm_group_count = 1
+ b._ssm_conv_kernel = 4
+ b._full_attention_interval = 4
+ assert b._rollback_state_bytes(1) == b._mamba_recurrent_state_bytes(1)
+
+ def test_mamba_path_unchanged(self):
+ # The KDA fallback must not shadow or double-count the Mamba helper.
+ b = _make_mla_backend()
+ b._ssm_inner_size = 6144
+ b._ssm_state_size = 128
+ b._ssm_group_count = 1
+ b._ssm_conv_kernel = 4
+ b._full_attention_interval = 4
+ assert b._mamba_recurrent_state_bytes(1) > 0
+ delta = b._estimate_mtp_overhead_bytes(
+ 65536, spec_draft_n_max = 2
+ ) - b._estimate_mtp_overhead_bytes(65536, spec_draft_n_max = 0)
+ assert delta == 2 * b._mamba_recurrent_state_bytes(1)
+
class TestMlaFitPreventsOom:
"""The corrected reserve must actually lower the auto-fit context so the