Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep GLM-5.3-Flash's MTP head in Auto (#12833)
Daniel Han662e65e60a65
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 53dabe57bbc..fc84f610040 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -4480,8 +4480,11 @@ def _auto_mode_drops_mtp(# MLA archs whose MTP context covers only the NextN block instead of duplicating the# trunk KV: glm5next holds 4+3 MiB over one layer where its trunk holds 48+36 over# twelve. That one fact is why Auto keeps MTP (gate below) and why the fit must not-# reserve the copy (_estimate_mtp_overhead_bytes). Not "glm5-next": no NextN graph.-_MLA_MTP_FAST_ARCHS = frozenset({"glm5next"})+# reserve the copy (_estimate_mtp_overhead_bytes). "glm5-next" = upstream name+# (ggml-org/llama.cpp#27773); builds lacking its NextN graph retry without speculation.+_MLA_MTP_FAST_ARCHS = frozenset({"glm5next", "glm5-next"})+# Smallest context Auto shrinks to so a fast MLA head still launches.+_FAST_MTP_MIN_CTX = 65536def _arch_has_fast_mla_mtp(architecture: Optional[str]) -> bool:@@ -25951,6 +25954,22 @@ class LlamaCppBackend:/ (1024 * 1024)<= _budget_w):+ # A NextN-only head is the main decode speedup: pay context.+ if (+ _engaged_is_mtp+ and _arch_has_fast_mla_mtp(+ getattr(self, "_architecture", None)+ )+ and _ctx_w >= min(_ctx_wo, _FAST_MTP_MIN_CTX)+ ):+ _both_fit_somewhere = True+ logger.info(+ "Auto: keeping the embedded MTP head; context %d -> %d "+ "so it fits (%.1f GB budget).",+ _ctx_wo,+ _ctx_w,+ _budget_w / 1024,+ )breakif _target_fits_somewhere and not _both_fit_somewhere:_spec_dropped_no_vram = True@@ -31273,6 +31292,7 @@ class LlamaCppBackend:_arch_unsupportedor "failed to measure draft model memory" in _loor "failed to measure mtp context memory" in _lo+ or "failed to create mtp context" in _loor "failed to create llama_context" in _lo):_retry_reason = (diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.pyindex 01b69cc2bd2..cdd5fc15523 100644--- a/studio/backend/tests/test_llama_cpp_mtp_detection.py+++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py@@ -2650,8 +2650,8 @@ def test_auto_glm5next_keeps_draft_mtp(monkeypatch):assert backend.spec_fallback_reason != "mla_mtp_disabled"-def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):- # The "glm5-next" port never builds the NextN graph: no spillover.+def test_auto_glm5next_hyphenated_arch_keeps_draft_mtp(monkeypatch):+ # The upstream "glm5-next" name, as the rewritten GLM-5.3-Flash shards carry it.backend = _mla_resolver_backend(monkeypatch)backend._architecture = "glm5-next"flags = backend._build_speculative_flags(@@ -2663,6 +2663,24 @@ def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):gpus = True,binary = "/fake/llama-server",)+ assert _flags_dict(flags).get("--spec-type") == "draft-mtp"+ assert backend.speculative_type == "draft-mtp"+ assert backend.spec_fallback_reason != "mla_mtp_disabled"+++def test_auto_glm_dsa_mtp_still_gated(monkeypatch):+ # Only the NextN-only archs are exempt: glm-dsa keeps the duplicated target KV.+ backend = _mla_resolver_backend(monkeypatch)+ backend._architecture = "glm-dsa"+ flags = backend._build_speculative_flags(+ speculative_type = "auto",+ spec_draft_n_max = None,+ extra_args = None,+ model_identifier = _GLM_MLA_MODEL,+ model_path = None,+ gpus = True,+ binary = "/fake/llama-server",+ )assert _flags_dict(flags).get("--spec-type") != "draft-mtp"assert backend.spec_fallback_reason == "mla_mtp_disabled"@@ -2670,7 +2688,8 @@ def test_auto_glm5next_hyphenated_arch_still_gated(monkeypatch):def test_arch_has_fast_mla_mtp_is_case_and_space_tolerant():assert _arch_has_fast_mla_mtp("glm5next")assert _arch_has_fast_mla_mtp(" GLM5Next ")- assert not _arch_has_fast_mla_mtp("glm5-next")+ assert _arch_has_fast_mla_mtp("glm5-next")+ assert _arch_has_fast_mla_mtp(" GLM5-Next ")assert not _arch_has_fast_mla_mtp("glm-dsa")assert not _arch_has_fast_mla_mtp(None)assert not _arch_has_fast_mla_mtp("")diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex 017943c9a92..266d02ccf51 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -69,6 +69,7 @@ if "httpx" not in sys.modules:sys.modules["httpx"] = modulefrom core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend, _loader_path_var+import core.inference.llama_cpp as llama_cpp_module_REAL_POPEN = subprocess.Popen@@ -1007,6 +1008,86 @@ def _tight_vram_backend(tmp_path: Path, *, drafter_gb: float):return backend, gguf, sidecar+def _tight_embedded_mtp_backend(tmp_path: Path, monkeypatch, *, architecture: str, floor: int):+ """Embedded MLA head (8 GB at 8192) that misses the 24 GB card at 8192 and fits at half."""+ gb = 1024**3+ backend, gguf, _sidecar = _tight_vram_backend(tmp_path, drafter_gb = 0.0)++ def read_metadata(_path):+ backend._nextn_predict_layers = 1+ backend._kv_lora_rank = 512+ backend._architecture = architecture+ backend._context_length = 8192++ backend._read_gguf_metadata = read_metadata+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: int(8 * gb * n_ctx / 8192)+ backend._fit_context_to_vram = lambda requested, *args, mtp_engaged = False, **kwargs: (+ requested // 2 if mtp_engaged else requested+ )+ backend.probe_server_capabilities = lambda _binary = None: {+ "mtp_token": "draft-mtp",+ "supports_ngram_mod": True,+ "spec_draft_n_max_flag": "--spec-draft-n-max",+ }+ monkeypatch.setattr(llama_cpp_module, "_FAST_MTP_MIN_CTX", floor)+ return backend, gguf+++def test_auto_shrinks_context_to_keep_a_fast_mla_mtp_head(tmp_path, monkeypatch):+ """Auto pays context for a NextN-only head instead of dropping it."""+ backend, gguf = _tight_embedded_mtp_backend(+ tmp_path, monkeypatch, architecture = "glm5-next", floor = 4096+ )++ result = _launch_auto_8k(backend, gguf, n_ctx = 0)++ cmd = result["cmd"]+ assert cmd[cmd.index("--spec-type") + 1] == "draft-mtp"+ assert cmd[cmd.index("-c") + 1] == "4096"+ assert backend.spec_fallback_reason != "drafter_no_vram"+++ "architecture, floor",+ [+ ("glm5-next", 8192), # the context that keeps the head is below the floor+ ("deepseek2", 4096), # not a NextN-only head: the usual drop+ ],+)+def test_auto_still_drops_mla_mtp_past_the_floor_or_off_the_list(+ tmp_path, monkeypatch, architecture, floor+):+ backend, gguf = _tight_embedded_mtp_backend(+ tmp_path, monkeypatch, architecture = architecture, floor = floor+ )+ monkeypatch.setenv("UNSLOTH_MLA_MTP_ENABLED", "1")++ result = _launch_auto_8k(backend, gguf, n_ctx = 0)++ cmd = result["cmd"]+ assert "draft-mtp" not in cmd+ assert cmd[cmd.index("-c") + 1] == "8192"+ assert backend.spec_fallback_reason == "drafter_no_vram"+++def test_auto_still_drops_a_sidecar_drafter_on_a_fast_mla_target(tmp_path, monkeypatch):+ """The context exception is for the embedded head, not a DSpark sidecar Auto picked."""+ backend, gguf = _tight_embedded_mtp_backend(+ tmp_path, monkeypatch, architecture = "glm5-next", floor = 4096+ )+ sidecar = tmp_path / "dspark-model-Q8_0.gguf"+ sidecar.write_bytes(b"draft")+ caps = backend.probe_server_capabilities()+ backend.probe_server_capabilities = lambda _binary = None: {**caps, "supports_dspark": True}++ result = _launch_auto_8k(backend, gguf, n_ctx = 0, dspark_draft_path = str(sidecar))++ cmd = result["cmd"]+ assert "draft-dspark" not in cmd and "draft-mtp" not in cmd+ assert cmd[cmd.index("-c") + 1] == "8192"+ assert backend.spec_fallback_reason == "drafter_no_vram"++def test_auto_drops_the_drafter_when_only_the_target_fits(tmp_path):"""Model fits, drafter does not: Auto keeps the context and runs without it.diff --git a/studio/backend/tests/test_mtp_mla_target_ctx.py b/studio/backend/tests/test_mtp_mla_target_ctx.pyindex 70b17065f7f..def8bbeca78 100644--- a/studio/backend/tests/test_mtp_mla_target_ctx.py+++ b/studio/backend/tests/test_mtp_mla_target_ctx.py@@ -202,12 +202,10 @@ class TestMlaTargetCtxReserve:assert other._estimate_mtp_overhead_bytes(ctx) == (b._mtp_draft_kv_bytes(ctx) + b._estimate_kv_cache_bytes(ctx, "f16"))- # The "glm5-next" port builds no NextN graph, so it keeps the safe default.+ # The upstream "glm5-next" name: its NextN graph allocates no copy either.hyphenated = _make_mla_backend()hyphenated._architecture = "glm5-next"- assert hyphenated._estimate_mtp_overhead_bytes(ctx) == other._estimate_mtp_overhead_bytes(- ctx- )+ assert hyphenated._estimate_mtp_overhead_bytes(ctx) == b._estimate_mtp_overhead_bytes(ctx)class TestKdaRollbackReserve: