Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep GLM-5.3-Flash's MTP head in Auto (#12833)

Daniel Han662e65e60a65
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 53dabe57bbc..fc84f610040 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4480,8 +4480,11 @@ def _auto_mode_drops_mtp(
# MLA archs whose MTP context covers only the NextN block instead of duplicating the
# trunk KV: glm5next holds 4+3 MiB over one layer where its trunk holds 48+36 over
# twelve. That one fact is why Auto keeps MTP (gate below) and why the fit must not
-# reserve the copy (_estimate_mtp_overhead_bytes). Not "glm5-next": no NextN graph.
-_MLA_MTP_FAST_ARCHS = frozenset({"glm5next"})
+# reserve the copy (_estimate_mtp_overhead_bytes). "glm5-next" = upstream name
+# (ggml-org/llama.cpp#27773); builds lacking its NextN graph retry without speculation.
+_MLA_MTP_FAST_ARCHS = frozenset({"glm5next", "glm5-next"})
+# Smallest context Auto shrinks to so a fast MLA head still launches.
+_FAST_MTP_MIN_CTX = 65536
def _arch_has_fast_mla_mtp(architecture: Optional[str]) -> bool:
@@ -25951,6 +25954,22 @@ class LlamaCppBackend:
/ (1024 * 1024)
<= _budget_w
):
+ # A NextN-only head is the main decode speedup: pay context.
+ if (
+ _engaged_is_mtp
+ and _arch_has_fast_mla_mtp(
+ getattr(self, "_architecture", None)
+ )
+ and _ctx_w >= min(_ctx_wo, _FAST_MTP_MIN_CTX)
+ ):
+ _both_fit_somewhere = True
+ logger.info(
+ "Auto: keeping the embedded MTP head; context %d -> %d "
+ "so it fits (%.1f GB budget).",
+ _ctx_wo,
+ _ctx_w,
+ _budget_w / 1024,
+ )
break
if _target_fits_somewhere and not _both_fit_somewhere:
_spec_dropped_no_vram = True
@@ -31273,6 +31292,7 @@ class LlamaCppBackend:
_arch_unsupported
or "failed to measure draft model memory" in _lo
or "failed to measure mtp context memory" in _lo
+ or "failed to create mtp context" in _lo
or "failed to create llama_context" in _lo
):
_retry_reason = (
diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.py
index 01b69cc2bd2..cdd5fc15523 100644
--- a/studio/backend/tests/test_llama_cpp_mtp_detection.py
+++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py
@@ -2650,8 +2650,8 @@ def test_auto_glm5next_keeps_draft_mtp(monkeypatch):
assert backend.spec_fallback_reason != "mla_mtp_disabled"
-def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):
- # The "glm5-next" port never builds the NextN graph: no spillover.
+def test_auto_glm5next_hyphenated_arch_keeps_draft_mtp(monkeypatch):
+ # The upstream "glm5-next" name, as the rewritten GLM-5.3-Flash shards carry it.
backend = _mla_resolver_backend(monkeypatch)
backend._architecture = "glm5-next"
flags = backend._build_speculative_flags(
@@ -2663,6 +2663,24 @@ def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):
gpus = True,
binary = "/fake/llama-server",
)
+ assert _flags_dict(flags).get("--spec-type") == "draft-mtp"
+ assert backend.speculative_type == "draft-mtp"
+ assert backend.spec_fallback_reason != "mla_mtp_disabled"
+
+
+def test_auto_glm_dsa_mtp_still_gated(monkeypatch):
+ # Only the NextN-only archs are exempt: glm-dsa keeps the duplicated target KV.
+ backend = _mla_resolver_backend(monkeypatch)
+ backend._architecture = "glm-dsa"
+ flags = backend._build_speculative_flags(
+ speculative_type = "auto",
+ spec_draft_n_max = None,
+ extra_args = None,
+ model_identifier = _GLM_MLA_MODEL,
+ model_path = None,
+ gpus = True,
+ binary = "/fake/llama-server",
+ )
assert _flags_dict(flags).get("--spec-type") != "draft-mtp"
assert backend.spec_fallback_reason == "mla_mtp_disabled"
@@ -2670,7 +2688,8 @@ def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):
def test_arch_has_fast_mla_mtp_is_case_and_space_tolerant():
assert _arch_has_fast_mla_mtp("glm5next")
assert _arch_has_fast_mla_mtp(" GLM5Next ")
- assert not _arch_has_fast_mla_mtp("glm5-next")
+ assert _arch_has_fast_mla_mtp("glm5-next")
+ assert _arch_has_fast_mla_mtp(" GLM5-Next ")
assert not _arch_has_fast_mla_mtp("glm-dsa")
assert not _arch_has_fast_mla_mtp(None)
assert not _arch_has_fast_mla_mtp("")
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index 017943c9a92..266d02ccf51 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -69,6 +69,7 @@ if "httpx" not in sys.modules:
sys.modules["httpx"] = module
from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend, _loader_path_var
+import core.inference.llama_cpp as llama_cpp_module
_REAL_POPEN = subprocess.Popen
@@ -1007,6 +1008,86 @@ def _tight_vram_backend(tmp_path: Path, *, drafter_gb: float):
return backend, gguf, sidecar
+def _tight_embedded_mtp_backend(tmp_path: Path, monkeypatch, *, architecture: str, floor: int):
+ """Embedded MLA head (8 GB at 8192) that misses the 24 GB card at 8192 and fits at half."""
+ gb = 1024**3
+ backend, gguf, _sidecar = _tight_vram_backend(tmp_path, drafter_gb = 0.0)
+
+ def read_metadata(_path):
+ backend._nextn_predict_layers = 1
+ backend._kv_lora_rank = 512
+ backend._architecture = architecture
+ backend._context_length = 8192
+
+ backend._read_gguf_metadata = read_metadata
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: int(8 * gb * n_ctx / 8192)
+ backend._fit_context_to_vram = lambda requested, *args, mtp_engaged = False, **kwargs: (
+ requested // 2 if mtp_engaged else requested
+ )
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "mtp_token": "draft-mtp",
+ "supports_ngram_mod": True,
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+ }
+ monkeypatch.setattr(llama_cpp_module, "_FAST_MTP_MIN_CTX", floor)
+ return backend, gguf
+
+
+def test_auto_shrinks_context_to_keep_a_fast_mla_mtp_head(tmp_path, monkeypatch):
+ """Auto pays context for a NextN-only head instead of dropping it."""
+ backend, gguf = _tight_embedded_mtp_backend(
+ tmp_path, monkeypatch, architecture = "glm5-next", floor = 4096
+ )
+
+ result = _launch_auto_8k(backend, gguf, n_ctx = 0)
+
+ cmd = result["cmd"]
+ assert cmd[cmd.index("--spec-type") + 1] == "draft-mtp"
+ assert cmd[cmd.index("-c") + 1] == "4096"
+ assert backend.spec_fallback_reason != "drafter_no_vram"
+
+
+ "architecture, floor",
+ [
+ ("glm5-next", 8192), # the context that keeps the head is below the floor
+ ("deepseek2", 4096), # not a NextN-only head: the usual drop
+ ],
+)
+def test_auto_still_drops_mla_mtp_past_the_floor_or_off_the_list(
+ tmp_path, monkeypatch, architecture, floor
+):
+ backend, gguf = _tight_embedded_mtp_backend(
+ tmp_path, monkeypatch, architecture = architecture, floor = floor
+ )
+ monkeypatch.setenv("UNSLOTH_MLA_MTP_ENABLED", "1")
+
+ result = _launch_auto_8k(backend, gguf, n_ctx = 0)
+
+ cmd = result["cmd"]
+ assert "draft-mtp" not in cmd
+ assert cmd[cmd.index("-c") + 1] == "8192"
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
+def test_auto_still_drops_a_sidecar_drafter_on_a_fast_mla_target(tmp_path, monkeypatch):
+ """The context exception is for the embedded head, not a DSpark sidecar Auto picked."""
+ backend, gguf = _tight_embedded_mtp_backend(
+ tmp_path, monkeypatch, architecture = "glm5-next", floor = 4096
+ )
+ sidecar = tmp_path / "dspark-model-Q8_0.gguf"
+ sidecar.write_bytes(b"draft")
+ caps = backend.probe_server_capabilities()
+ backend.probe_server_capabilities = lambda _binary = None: {**caps, "supports_dspark": True}
+
+ result = _launch_auto_8k(backend, gguf, n_ctx = 0, dspark_draft_path = str(sidecar))
+
+ cmd = result["cmd"]
+ assert "draft-dspark" not in cmd and "draft-mtp" not in cmd
+ assert cmd[cmd.index("-c") + 1] == "8192"
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+
+
def test_auto_drops_the_drafter_when_only_the_target_fits(tmp_path):
"""Model fits, drafter does not: Auto keeps the context and runs without it.
diff --git a/studio/backend/tests/test_mtp_mla_target_ctx.py b/studio/backend/tests/test_mtp_mla_target_ctx.py
index 70b17065f7f..def8bbeca78 100644
--- a/studio/backend/tests/test_mtp_mla_target_ctx.py
+++ b/studio/backend/tests/test_mtp_mla_target_ctx.py
@@ -202,12 +202,10 @@ class TestMlaTargetCtxReserve:
assert other._estimate_mtp_overhead_bytes(ctx) == (
b._mtp_draft_kv_bytes(ctx) + b._estimate_kv_cache_bytes(ctx, "f16")
)
- # The "glm5-next" port builds no NextN graph, so it keeps the safe default.
+ # The upstream "glm5-next" name: its NextN graph allocates no copy either.
hyphenated = _make_mla_backend()
hyphenated._architecture = "glm5-next"
- assert hyphenated._estimate_mtp_overhead_bytes(ctx) == other._estimate_mtp_overhead_bytes(
- ctx
- )
+ assert hyphenated._estimate_mtp_overhead_bytes(ctx) == b._estimate_mtp_overhead_bytes(ctx)
class TestKdaRollbackReserve: