Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Drop an MTP sidecar llama-server cannot load as --model-draft (#10197)

Long Yixing645a350b939a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index efdfbc35151..ac183f12f2a 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -3378,6 +3378,16 @@ def _pick_mtp_root_only(candidates: list[str]) -> Optional[str]:
return _pick_mtp(candidates, allow_nested = False)
+def _mtp_drafter_loads_standalone(path: str) -> bool:
+ """Does *path* have the embeddings a ``--model-draft`` file needs?
+
+ The rule lives in ``gguf_metadata`` so the memory-estimate route, which asks on
+ every settings change, shares its ``(path, mtime, size)`` cache.
+ """
+ from utils.models.gguf_metadata import mtp_drafter_loads_standalone
+ return mtp_drafter_loads_standalone(path)
+
+
def _pick_mmproj(candidates: list[str]) -> Optional[str]:
from hub.utils.gguf import drop_shadowed_appledouble_names
@@ -6445,11 +6455,13 @@ class LlamaCppBackend:
# Separate MTP drafter launched with the current model; reload-dedup
# key so a drafter that appears next to the weights forces a reload.
self._mtp_draft_path: Optional[str] = None
- # Drafter this load resolved then deliberately suppressed (virtualised Metal
- # with no draft-layer flag). The dedup keys on the drafter the caller asked
- # for, still on disk, so without this the detected path never matches the
- # launched None and every repeat Apply tears down a healthy server.
+ # Drafter this load resolved then suppressed: unloadable sidecar, or virtualised
+ # Metal with no draft-layer flag. The caller keeps detecting it on disk, so
+ # without this record every repeat Apply tears down a healthy server.
self._mtp_draft_suppressed_path: Optional[str] = None
+ # WHICH drop it was, "unloadable" or "paravirtual": only the unloadable one is a
+ # verdict on the CONTENTS, so only it reloads on repair and renames the reason.
+ self._mtp_draft_suppressed_reason: Optional[str] = None
# Why MTP was disabled on the last load that asked for it (auto on an
# MTP model, or forced mtp / mtp+ngram), else None. Drives the "update
# llama.cpp" hint in the UI. "binary_no_mtp" / "binary_outdated" ->
@@ -7585,6 +7597,8 @@ class LlamaCppBackend:
if (
intent.gguf_path is None
and self._spec_fallback_reason == "drafter_not_found"
+ # The fetch did find it and the load dropped it; refetching finds it again.
+ and self._mtp_draft_suppressed_path is None
and speculative_type in ("auto", "mtp", "mtp+ngram", "dspark", "dflash")
and not (self._spec_drafter_kind == "dspark" and self._dspark_sidecar_absent)
# DFlash asks through _dflash_retry_needed below, which is set only for
@@ -7690,6 +7704,16 @@ class LlamaCppBackend:
return False
if requested_draft != loaded_draft:
return False
+ # Repair a rejected sidecar in place and a path-only comparison would keep the
+ # drafter-free server. Only the unloadable drop is re-asked: the paravirtual
+ # one is a property of the build, so the same bytes drop again. Cached on
+ # (path, mtime, size), so only a repaired file is reparsed.
+ if (
+ self._mtp_draft_suppressed_reason == "unloadable"
+ and self._mtp_draft_suppressed_path
+ and _mtp_drafter_loads_standalone(self._mtp_draft_suppressed_path)
+ ):
+ return False
return True
@property
@@ -20989,6 +21013,7 @@ class LlamaCppBackend:
# The canonical mode drives which drafter is downloaded, sized and
# launched, so resolve it once before either branch can use it.
_spec_canon = _canonicalize_spec_mode(speculative_type) or "auto"
+ _unloadable_mtp_draft_path: Optional[str] = None
# Scope HF_HUB_OFFLINE to the download block only when DNS is
# dead; cleanup runs even on exception so a transient hiccup
# can't quarantine future loads.
@@ -21174,6 +21199,31 @@ class LlamaCppBackend:
logger.info("Main GGUF contains an embedded MTP head; ignoring separate drafter.")
mtp_draft_path = None
+ # Before the fit prices it, or a drafter that never launches pushes layers
+ # off the GPU. DSpark/DFlash borrow token_embd from the target by design.
+ #
+ # Not when the extras name their own drafter: draft flags are last-wins, so
+ # that file is the one llama-server opens and the sibling is never touched.
+ # Dropping it anyway reads as drafterless, and the fallback emits ngram-mod
+ # or --spec-default BEFORE the extras are appended, so the override stops
+ # running as MTP. Same helper and child-env rule as the paravirtual drop
+ # below: inherited LLAMA_ARG_SPEC_DRAFT_* reaches the child only when the
+ # extras own --spec-type.
+ if (
+ mtp_draft_path
+ and _spec_canon not in ("dspark", "dflash")
+ and not _extra_args_mtp_draft_path(extra_args, env = _child_spec_env(extra_args))
+ and not _mtp_drafter_loads_standalone(mtp_draft_path)
+ ):
+ logger.warning(
+ "Dropping MTP drafter %s: it carries neither token_embd.weight nor "
+ "nextn_shared_target_tensors, so llama-server cannot load it as "
+ "--model-draft; loading without it.",
+ mtp_draft_path,
+ )
+ _unloadable_mtp_draft_path = mtp_draft_path
+ mtp_draft_path = None
+
if _load_cancelled():
logger.info("Load cancelled after download phase")
return False
@@ -21201,6 +21251,7 @@ class LlamaCppBackend:
# reloads a healthy diffusion server on every Apply.
self._mtp_draft_path = None
self._mtp_draft_suppressed_path = None
+ self._mtp_draft_suppressed_reason = None
# And the verdict on why the last load had no drafter, for the same
# reason once more: _build_speculative_flags clears it at the top of
# every load that reaches it, and this path never does. Both retry
@@ -24564,7 +24615,10 @@ class LlamaCppBackend:
mtp_draft_path = mtp_draft_path,
drafter_label = _DRAFTER_DISPLAY_LABELS.get(_spec_canon, "MTP"),
)
- _pv_suppressed_draft_path: Optional[str] = None
+ _suppressed_draft_path: Optional[str] = _unloadable_mtp_draft_path
+ _suppressed_draft_reason: Optional[str] = (
+ "unloadable" if _unloadable_mtp_draft_path else None
+ )
_pv_suppressed_spec_extra_args: Optional[List[str]] = None
# Same shape as the projector drop above: the CPU pin below needs a
# draft-layer flag from the probe, and without one the drafter keeps its
@@ -24613,7 +24667,12 @@ class LlamaCppBackend:
# Remember what was suppressed: the drafter stays on disk, so caller
# and route keep detecting it, and comparing that against the
# launched None would reload a healthy server on every repeat Apply.
- _pv_suppressed_draft_path = launch_mtp_draft_path
+ # Only what this drop removed: extras trigger it with none of ours
+ # launched, and None would clear a record an earlier drop made.
+ if launch_mtp_draft_path:
+ _suppressed_draft_path = launch_mtp_draft_path
+ # A property of the build, not the file, so a repair must not reload.
+ _suppressed_draft_reason = "paravirtual"
launch_mtp_draft_path = None
if extra_args:
# Same reason: record the extras as REQUESTED next to the
@@ -24665,6 +24724,9 @@ class LlamaCppBackend:
dspark_fit_sized = not use_fit,
dflash_draft_path = (launch_mtp_draft_path if _spec_canon == "dflash" else None),
dflash_fit_sized = not use_fit,
+ # So the fallback says unopenable, not missing. DSpark/DFlash never
+ # reach the drop.
+ mtp_drafter_unloadable = bool(_unloadable_mtp_draft_path),
drafter_no_vram = _spec_dropped_no_vram,
embedded_mtp_partial_offload = bool(
_spec_canon == "auto"
@@ -26795,6 +26857,16 @@ class LlamaCppBackend:
avail_mib = _preflight_avail_mib,
pageable_note = _cpu_pageable_note,
)
+ # This return skips the commit block below, so write the drafter
+ # records here too: left describing the PREVIOUS load, the comparator
+ # judges this child against another model's drafter and a carried-over
+ # suppressed path stands the refetch down for a load that dropped
+ # nothing. Inert while only virtualised Metal could suppress, where
+ # auto-Vulkan fallback cannot happen; an unloadable sidecar can be
+ # suppressed anywhere.
+ self._mtp_draft_path = launch_mtp_draft_path
+ self._mtp_draft_suppressed_path = _suppressed_draft_path
+ self._mtp_draft_suppressed_reason = _suppressed_draft_reason
logger.warning(
"llama-server loaded successfully on CPU after the "
"auto-selected Vulkan backend crashed. GPU acceleration "
@@ -26811,7 +26883,8 @@ class LlamaCppBackend:
self._gguf_load_identity = self._gguf_load_source_identity(model_path, mmproj_path)
self._hf_repo = hf_repo
self._mtp_draft_path = launch_mtp_draft_path
- self._mtp_draft_suppressed_path = _pv_suppressed_draft_path
+ self._mtp_draft_suppressed_path = _suppressed_draft_path
+ self._mtp_draft_suppressed_reason = _suppressed_draft_reason
# For local GGUF files, extract variant from filename if absent
if hf_variant:
self._hf_variant = hf_variant
@@ -28154,6 +28227,7 @@ class LlamaCppBackend:
drafter_no_vram: bool = False,
embedded_mtp_partial_offload: bool = False,
draft_device: Optional[str] = None,
+ mtp_drafter_unloadable: bool = False,
) -> List[str]:
"""Return the llama-server flag list for the requested spec mode.
@@ -28223,6 +28297,11 @@ class LlamaCppBackend:
bool(self._nextn_predict_layers)
or _is_mtp_model_name(model_identifier, model_path)
or bool(mtp_draft_path)
+ # A dropped sidecar was this quant's ONLY MTP signal: the motivating
+ # RVN-Q6_K.gguf has no nextn_predict_layers and no -mtp in its name, so
+ # clearing mtp_draft_path read as a plain model, skipping every arm below and
+ # leaving the drop unexplained. Kept as a signal, it reaches the fallback.
+ or mtp_drafter_unloadable
)
_mtp_size_b = _extract_model_size_b(model_identifier)
# The sub-3B regression is an embedded-head cost; a separate drafter
@@ -28230,12 +28309,14 @@ class LlamaCppBackend:
_mtp_too_small = (
_mtp_size_b is not None and _mtp_size_b < _MTP_MIN_SIZE_B and not bool(mtp_draft_path)
)
- # Drafterless Gemma (name-only MTP, no embedded head): emitting MTP
- # would abort llama-server, so every mode below falls back instead.
+ # Drafterless Gemma (name-only MTP, no embedded head), or any quant whose only
+ # drafter was dropped as unopenable: emitting MTP would abort llama-server, so
+ # every mode below falls back. Without the second case a dropped sidecar on any
+ # non-Gemma repo disables MTP in silence.
_mtp_drafter_missing = (
- _is_gemma_mtp_name(model_identifier, model_path)
- and not mtp_draft_path
+ not mtp_draft_path
and not self._nextn_predict_layers
+ and (_is_gemma_mtp_name(model_identifier, model_path) or mtp_drafter_unloadable)
)
# Embedded MTP head on an MLA model (GLM-5.2/DeepSeek/Kimi, detected by
# kv_lora_rank): llama.cpp's MLA/DSA MTP path is ~2x slower than no spec,
@@ -28504,17 +28585,28 @@ class LlamaCppBackend:
def _fallback_drafter_not_found() -> None:
"""Drafterless Gemma: use ngram-mod (or spec-default) and record why."""
- logger.warning(
- "Model %s is MTP-capable but no drafter or head was found; "
- "falling back. Check network or run `unsloth studio update`.",
- model_identifier,
- )
+ if mtp_drafter_unloadable:
+ logger.warning(
+ "Model %s had an MTP drafter that llama-server cannot open as "
+ "--model-draft; falling back without it.",
+ model_identifier,
+ )
+ else:
+ logger.warning(
+ "Model %s is MTP-capable but no drafter or head was found; "
+ "falling back. Check network or run `unsloth studio update`.",
+ model_identifier,
+ )
if caps.get("supports_ngram_mod"):
_emit_ngram_mod()
else:
flags.append("--spec-default")
self._speculative_type = "default"
- self._spec_fallback_reason = "drafter_not_found"
+ # Its own reason: the drafter_not_found copy tells a local load to place a file
+ # already on disk, and offers a refetch this load deliberately stands down.
+ self._spec_fallback_reason = (
+ "drafter_unloadable" if mtp_drafter_unloadable else "drafter_not_found"
+ )
if effective_mode == "ngram":
_emit_ngram_mod()
@@ -28905,6 +28997,7 @@ class LlamaCppBackend:
self._hf_repo = None
self._mtp_draft_path = None
self._mtp_draft_suppressed_path = None
+ self._mtp_draft_suppressed_reason = None
self._spec_fallback_reason = None
self._mmproj_fallback_reason = None
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 515ebb43aca..03dea9f0f3c 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -10733,6 +10733,7 @@ def _estimate_gguf_required_gb(
_extra_args_requests_dspark,
_extra_args_set_spec_type,
_mmproj_env_is_audio_only,
+ _mtp_drafter_loads_standalone,
extra_args_disable_mmproj,
)
@@ -10916,7 +10917,13 @@ def _estimate_gguf_required_gb(
):
_sized_attrs.append("gguf_dflash_file")
else:
- _sized_attrs.append("gguf_mtp_file")
+ # The launch drops a drafter it cannot open, so charging one would refuse
+ # a load that fits.
+ _mtp = getattr(config, "gguf_mtp_file", None)
+ if not (
+ _mtp and Path(_mtp).is_file() and not _mtp_drafter_loads_standalone(str(_mtp))
+ ):
+ _sized_attrs.append("gguf_mtp_file")
for attr in _sized_attrs:
f = getattr(config, attr, None)
diff --git a/studio/backend/tests/test_llama_cpp_mtp_detection.py b/studio/backend/tests/test_llama_cpp_mtp_detection.py
index 00232d0bc5a..90f36a65b81 100644
--- a/studio/backend/tests/test_llama_cpp_mtp_detection.py
+++ b/studio/backend/tests/test_llama_cpp_mtp_detection.py
@@ -3550,3 +3550,97 @@ def test_the_dspark_gate_uses_the_probe_it_is_given():
# No sidecar on disk and an incapable binary: the fetch is skipped, which is exactly
# the degraded launch the accumulator has to remember.
assert result is None
+
+
+def _write_head_only_drafter(path, *, with_token_embd: bool):
+ """A real GGUF the predicate will judge, so these exercise the actual verdict."""
+ import numpy as np
+ from gguf import GGUFWriter
+
+ writer = GGUFWriter(str(path), "qwen35")
+ names = ["output.weight", "blk.64.nextn.eh_proj.weight"]
+ if with_token_embd:
+ names.insert(0, "token_embd.weight")
+ for name in names:
+ writer.add_tensor(name, np.zeros((2, 2), dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ return path
+
+
+def test_repairing_an_unloadable_drafter_in_place_reloads(tmp_path):
+ """Apply after a repair must reload, or speculative decoding stays off for good.
+
+ The dedupe counts a deliberately dropped drafter as launched, comparing resolved
+ PATHS. Replace the rejected sidecar with a self-contained head at the same path and
+ a path-only comparison still says "already loaded", so the drafter-free server is
+ kept and nothing re-runs the header check.
+ """
+ sidecar = _write_head_only_drafter(tmp_path / "mtp-model.gguf", with_token_embd = False)
+ backend = _mtp_backend(
+ _speculative_type = "ngram-mod",
+ _requested_spec_mode = "auto",
+ _spec_fallback_reason = "drafter_unloadable",
+ _spec_drafter_kind = "mtp",
+ _mtp_draft_path = None,
+ _mtp_draft_suppressed_path = str(sidecar),
+ _mtp_draft_suppressed_reason = "unloadable",
+ )
+ # Still head-only: re-asking would drop it again, so a reload would buy nothing.
+ assert _matches_mtp(backend, mtp_draft_path = str(sidecar), compare_mtp_draft = True) is True
+
+ _write_head_only_drafter(tmp_path / "mtp-model.gguf", with_token_embd = True)
+ assert _matches_mtp(backend, mtp_draft_path = str(sidecar), compare_mtp_draft = True) is False
+
+
+def test_repairing_a_paravirtually_suppressed_drafter_does_not_reload(tmp_path):
+ """The negative: that drop is a property of the BUILD, not of the file's contents.
+
+ A drafter suppressed because the probe offered no draft-layer flag would be
+ suppressed again byte-for-byte, so re-asking the header question there would tear
+ down a healthy server for nothing.
+ """
+ sidecar = _write_head_only_drafter(tmp_path / "mtp-model.gguf", with_token_embd = True)
+ backend = _mtp_backend(
+ _speculative_type = "ngram-mod",
+ _requested_spec_mode = "auto",
+ _spec_fallback_reason = "drafter_not_found",
+ _spec_drafter_kind = "mtp",
+ _mtp_draft_path = None,
+ _mtp_draft_suppressed_path = str(sidecar),
+ _mtp_draft_suppressed_reason = "paravirtual",
+ )
+ assert _matches_mtp(backend, mtp_draft_path = str(sidecar), compare_mtp_draft = True) is True
+
+
+def test_a_drafter_dropped_as_unloadable_does_not_reload_to_refetch_it():
+ """Refetching returns the same file, so the retry rule must stand down."""
+ sidecar = "/cache/snapshots/abc/mtp-gemma-4-12b-it.gguf"
+ backend = _mtp_backend(
+ _model_identifier = "unsloth/gemma-4-12b-it-GGUF",
+ _speculative_type = "ngram-mod",
+ _requested_spec_mode = "auto",
+ _spec_fallback_reason = "drafter_not_found",
+ _spec_drafter_kind = "mtp",
+ _mtp_draft_path = None,
+ _mtp_draft_suppressed_path = sidecar,
+ )
+ assert (
+ _matches(
+ backend,
+ gguf_path = None,
+ model_identifier = "unsloth/gemma-4-12b-it-GGUF",
+ hf_variant = "Q4_K_M",
+ n_ctx = 8192,
+ cache_type_kv = None,
+ speculative_type = "auto",
+ chat_template_override = None,
+ extra_args = None,
+ is_vision = False,
+ mtp_draft_path = sidecar,
+ compare_mtp_draft = True,
+ )
+ is True
+ )
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index 5986f0303e2..05371d102b7 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -3507,3 +3507,84 @@ def test_a_restored_cpu_fallback_the_host_can_hold_says_nothing(tmp_path, monkey
_launch_with_vulkan_cpu_replay(backend, gguf, crash = False, cpu_fallback = True)
assert backend.last_load_warning is None
+
+
+def _write_mtp_drafter(path: Path, *, with_token_embd: bool) -> Path:
+ import numpy as np
+ from gguf import GGUFWriter
+
+ writer = GGUFWriter(str(path), "qwen35")
+ names = ["output.weight", "blk.64.nextn.eh_proj.weight"]
+ if with_token_embd:
+ names += ["token_embd.weight", "output_norm.weight"]
+ for name in names:
+ writer.add_tensor(name, np.zeros((2, 2), dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ return path
+
+
+def _headless_mtp_backend(tmp_path):
+ backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 24_000, 24_000)])
+ backend._select_gpus = lambda *args, **kwargs: ([0], False)
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "mtp_token": "draft-mtp",
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+ }
+ return backend, gguf
+
+
+def test_an_mtp_drafter_llama_server_cannot_load_is_dropped(tmp_path):
+ """Dropped before the launch it would abort, and recorded so Apply does not reload."""
+ backend, gguf = _headless_mtp_backend(tmp_path)
+ drafter = _write_mtp_drafter(tmp_path / "mtp-model.gguf", with_token_embd = False)
+
+ cmd = _launch(backend, gguf, mtp_draft_path = str(drafter), speculative_type = "mtp")["cmd"]
+
+ assert "--model-draft" not in cmd
+ assert str(drafter) not in cmd
+ assert backend.mtp_draft_path is None
+ assert backend.mtp_draft_suppressed_path == str(drafter)
+
+
+def test_an_advanced_argument_drafter_survives_the_unloadable_drop(tmp_path):
+ """A user-named --model-draft wins last, so the head-only sibling never opens.
+
+ Dropping it anyway made the load read as drafterless, and the fallback emits
+ ngram-mod or --spec-default BEFORE the extras are appended, so the override stopped
+ running as MTP at all. Nothing here needs protecting: the file llama-server opens is
+ the user's, and the sibling is not passed.
+ """
+ backend, gguf = _headless_mtp_backend(tmp_path)
+ bad = _write_mtp_drafter(tmp_path / "mtp-model.gguf", with_token_embd = False)
+ good = _write_mtp_drafter(tmp_path / "user-draft.gguf", with_token_embd = True)
+
+ cmd = _launch(
+ backend,
+ gguf,
+ mtp_draft_path = str(bad),
+ speculative_type = "mtp",
+ extra_args = ["--model-draft", str(good)],
+ )["cmd"]
+
+ # The user's file is what runs, as MTP, and the rejected sibling is nowhere.
+ assert cmd[-2:] == ["--model-draft", str(good)]
+ assert "draft-mtp" in cmd
+ assert "--spec-default" not in cmd
+ assert "ngram-mod" not in cmd
+ assert backend.mtp_draft_suppressed_path is None
+ assert backend.spec_fallback_reason is None
+
+
+def test_a_drafter_carrying_its_own_embeddings_still_reaches_the_command(tmp_path):
+ """The control for the drop above: same load, one tensor different."""
+ backend, gguf = _headless_mtp_backend(tmp_path)
+ drafter = _write_mtp_drafter(tmp_path / "mtp-model.gguf", with_token_embd = True)
+
+ cmd = _launch(backend, gguf, mtp_draft_path = str(drafter), speculative_type = "mtp")["cmd"]
+
+ assert cmd[cmd.index("--model-draft") + 1] == str(drafter)
+ assert backend.mtp_draft_path == str(drafter)
+ assert backend.mtp_draft_suppressed_path is None
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index 1dc5090b618..09b5926feea 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -158,6 +158,32 @@ def _write_gguf(
return str(path)
+def _write_gguf_with_embeddings(
+ tmp_path: Path,
+ arch: str,
+ fields: dict,
+ name: str = "model.gguf",
+) -> str:
+ """A weight file carrying ``token_embd.weight``, as every real one does.
+
+ ``_make_gguf_bytes`` writes metadata and no tensors, which is a file the launch
+ would refuse to pass as ``--model-draft`` and the estimate therefore never charges.
+ """
+ import numpy as np
+ from gguf import GGUFWriter
+
+ path = tmp_path / name
+ writer = GGUFWriter(str(path), arch)
+ for key, value in fields.items():
+ writer.add_uint32(f"{arch}.{key}", int(value))
+ writer.add_tensor("token_embd.weight", np.zeros((64, 64), dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ return str(path)
+
+
@pytest.fixture
def gqa_gguf(tmp_path) -> str:
return _write_gguf(tmp_path, "qwen3", _GQA_FIELDS)
@@ -2222,7 +2248,7 @@ class TestSpeculationOffChargesNoDrafter:
@pytest.fixture
def config_with_a_sidecar(self, tmp_path):
- target = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS, name = "target.gguf")
+ target = _write_gguf_with_embeddings(tmp_path, "qwen3", _GQA_FIELDS, name = "target.gguf")
sidecar = tmp_path / "mtp.gguf"
sidecar.write_bytes(Path(target).read_bytes())
return SimpleNamespace(
@@ -2253,6 +2279,62 @@ class TestSpeculationOffChargesNoDrafter:
assert with_mtp == auto
+class TestAnUnloadableSidecarIsNotCharged:
+ """A drafter the launch drops must not be priced, or the guard refuses a load that fits.
+
+ ``load_model`` drops a ``mtp-*.gguf`` carrying neither ``token_embd.weight`` nor
+ ``nextn_shared_target_tensors``, since llama-server opens a draft model as a
+ complete model. Charging it here is VRAM the launch never asks for, and the
+ chat-load admission guard turns that into a 409 against a running training job.
+ """
+
+ def _config(self, tmp_path, sidecar):
+ # A target per case: the files cache keys on the main weight's path, not the
+ # sidecar's, so one directory would serve the first answer twice.
+ home = tmp_path / sidecar.stem
+ home.mkdir()
+ return SimpleNamespace(
+ identifier = "local/target",
+ gguf_file = _write_gguf_with_embeddings(home, "qwen3", _GQA_FIELDS, name = "target.gguf"),
+ is_gguf = True,
+ gguf_variant = None,
+ gguf_mmproj_file = None,
+ gguf_mtp_file = str(sidecar),
+ gguf_dspark_file = None,
+ gguf_dflash_file = None,
+ )
+
+ def test_a_head_only_sidecar_is_uncharged_and_a_complete_one_is_not(self, tmp_path):
+ import numpy as np
+ from gguf import GGUFWriter
+
+ def drafter(name, tensors):
+ path = tmp_path / name
+ writer = GGUFWriter(str(path), "qwen3")
+ for tensor in tensors:
+ writer.add_tensor(tensor, np.zeros((512, 512), dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ return path
+
+ heads = ["output.weight", "blk.64.nextn.eh_proj.weight"]
+ head_only = drafter("mtp-head.gguf", heads)
+ complete = drafter("mtp-full.gguf", ["token_embd.weight", *heads])
+ assert head_only.stat().st_size > 1024 * 1024
+
+ without = ri._gguf_resident_file_gb(
+ self._config(tmp_path, head_only), speculative_type = "mtp"
+ )
+ with_it = ri._gguf_resident_file_gb(
+ self._config(tmp_path, complete), speculative_type = "mtp"
+ )
+ assert without is not None and with_it is not None
+ assert with_it > without
+ assert without == pytest.approx(with_it - complete.stat().st_size / (1024**3), rel = 0.02)
+
+
class TestAProjectorOverrideIsTheOneCharged:
"""``--mmproj`` in Advanced Arguments last-wins, so it is the file that opens.
diff --git a/studio/backend/tests/test_metal_paravirtual_guard.py b/studio/backend/tests/test_metal_paravirtual_guard.py
index 4b7e13089cd..74eccb7b2b6 100644
--- a/studio/backend/tests/test_metal_paravirtual_guard.py
+++ b/studio/backend/tests/test_metal_paravirtual_guard.py
@@ -669,6 +669,21 @@ def _drafter_gate(
):
"""Run load_model's real unpinnable-drafter statements and report what the launch
would see: (resolved drafter, extra args, warnings)."""
+ scope, warnings = _drafter_gate_scope(
+ paravirtual = paravirtual, caps = caps, drafter = drafter, extra_args = extra_args
+ )
+ return scope["launch_mtp_draft_path"], scope["extra_args"], warnings
+
+
+def _drafter_gate_scope(
+ *,
+ paravirtual: bool,
+ caps: dict,
+ drafter = "/models/mtp-gemma.gguf",
+ extra_args = None,
+ suppressed = None,
+):
+ """The same statements, returning the scope, for callers reading more than the drafter."""
body = None
for node in ast.walk(_load_model_tree()):
stmts = getattr(node, "body", None)
@@ -708,9 +723,10 @@ def _drafter_gate(
"n_parallel": 1,
"cmd": ["--parallel", "1"],
"logger": log,
+ "_suppressed_draft_path": suppressed,
}
exec(ast.unparse(ast.Module(body = body, type_ignores = [])), scope)
- return scope["launch_mtp_draft_path"], scope["extra_args"], log.warnings
+ return scope, log.warnings
def _paravirtual_gate(
@@ -1538,10 +1554,33 @@ def test_a_launched_drafter_records_no_suppression(monkeypatch, tmp_path):
assert backend.mtp_draft_suppressed_path is None
src = _load_model_source()
# Only the unpinnable branch records it, and it records what it is about to clear.
- assert src.index("_pv_suppressed_draft_path = launch_mtp_draft_path") < src.index(
+ assert src.index("_suppressed_draft_path = launch_mtp_draft_path") < src.index(
" launch_mtp_draft_path = None"
)
- assert "self._mtp_draft_suppressed_path = _pv_suppressed_draft_path" in src
+ assert "self._mtp_draft_suppressed_path = _suppressed_draft_path" in src
+
+
+def test_every_successful_return_records_the_drafter_it_launched():
+ """The CPU-fallback return commits the records too, or they describe the last load.
+
+ ``load_model`` has two returns that leave a server running: the ordinary commit
+ block, and the auto-Vulkan-crash replay that comes up on CPU and returns early.
+ Only the first wrote ``_mtp_draft_path`` / ``_mtp_draft_suppressed_path``, so the
+ replay kept the PREVIOUS load's pair. That was inert while a drafter could only be
+ suppressed on virtualised Metal, where an auto-Vulkan fallback cannot happen; an
+ unloadable sidecar can be suppressed on any platform, and a carried-over suppressed
+ path stands the drafter_not_found refetch down for a load that dropped nothing.
+ """
+ src = _load_model_source()
+ assert (
+ src.count("self._mtp_draft_suppressed_path = _suppressed_draft_path") == 2
+ ), "both successful returns must record the drafter they launched"
+ # The replay writes its record before returning, not after.
+ replay = src.index("loaded successfully on CPU after the")
+ assert src.rindex("self._mtp_draft_path = launch_mtp_draft_path", 0, replay) < replay
+ assert (
+ src.rindex("self._mtp_draft_suppressed_path = _suppressed_draft_path", 0, replay) < replay
+ )
# ── an inherited projector must not slip past the projector guard ────
@@ -1956,7 +1995,7 @@ def test_the_requested_extras_default_to_the_launched_ones():
def test_the_drop_records_the_requested_extras_before_rewriting_them():
- """The recording contract, mirroring _pv_suppressed_draft_path: capture, then strip."""
+ """The recording contract, mirroring _suppressed_draft_path: capture, then strip."""
src = _load_model_source()
assert src.index("_pv_suppressed_spec_extra_args = list(extra_args)") < src.index(
" strip_spec = True,"
@@ -2021,3 +2060,18 @@ def test_the_route_really_can_deliver_a_manual_cpu_request_carrying_an_override(
)
== extras
)
+
+
+def test_the_drop_records_only_the_drafter_it_removed():
+ """Extras trigger this drop with none of ours launched, where recording the launched
+ None would erase a record an earlier drop made."""
+ earlier = "/cache/snapshots/abc/mtp-model.gguf"
+ scope, _warnings = _drafter_gate_scope(
+ paravirtual = True,
+ caps = {},
+ drafter = None,
+ extra_args = ["--model-draft", "/models/user-drafter.gguf"],
+ suppressed = earlier,
+ )
+ assert scope["launch_mtp_draft_path"] is None
+ assert scope["_suppressed_draft_path"] == earlier
diff --git a/studio/backend/tests/test_mmproj_placement_policy.py b/studio/backend/tests/test_mmproj_placement_policy.py
index dcb7bba2c48..8f19047bf43 100644
--- a/studio/backend/tests/test_mmproj_placement_policy.py
+++ b/studio/backend/tests/test_mmproj_placement_policy.py
@@ -58,6 +58,23 @@ def _write_gguf(path: Path) -> Path:
return path
+def _write_drafter_gguf(path: Path, *, with_token_embd: bool = True) -> Path:
+ import numpy as np
+ from gguf import GGUFWriter
+
+ writer = GGUFWriter(str(path), "qwen35")
+ names = ["output.weight", "blk.64.nextn.eh_proj.weight"]
+ if with_token_embd:
+ names += ["token_embd.weight", "output_norm.weight"]
+ for name in names:
+ writer.add_tensor(name, np.zeros((2, 2), dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ return path
+
+
def _backend(
tmp_path: Path,
*,
@@ -76,7 +93,7 @@ def _backend(
backend = LlamaCppBackend()
gguf = _write_gguf(tmp_path / "model.gguf")
mmproj = _write_gguf(tmp_path / "mmproj-F16.gguf")
- drafter = _write_gguf(tmp_path / "mtp.gguf")
+ drafter = _write_drafter_gguf(tmp_path / "mtp.gguf")
def read_metadata(_path):
backend._context_length = native_ctx
@@ -1599,3 +1616,18 @@ def test_a_gpu_drafter_holds_the_deferred_pin_back(tmp_path):
assert "--mmproj" in cmd
assert "--no-mmproj-offload" not in cmd
+
+
+def test_an_unloadable_drafter_is_not_charged_before_it_is_dropped(tmp_path):
+ """Judged after the fit, the drafter's 2 GiB would push model + drafter + projector
+ past this 10550 MiB budget and pin the projector for a file that is dropped anyway."""
+ backend, gguf = _drafter_backend(tmp_path, [(0, 12_470, 24_000)])
+ _write_drafter_gguf(tmp_path / "mtp.gguf", with_token_embd = False)
+ # The harness resolver re-supplies the drafter whatever the load decided.
+ del backend._resolve_launch_mtp_path
+
+ cmd = _launch_with_drafter(backend, gguf, tmp_path)
+
+ assert "--model-draft" not in cmd
+ assert "--no-mmproj-offload" not in cmd
+ assert backend.mtp_draft_suppressed_path == str(tmp_path / "mtp.gguf")
diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py
index aa7dd03c529..f629ff48aba 100644
--- a/studio/backend/tests/test_mtp_drafter_companion.py
+++ b/studio/backend/tests/test_mtp_drafter_companion.py
@@ -2189,6 +2189,80 @@ def test_forced_dflash_without_a_sidecar_falls_back(monkeypatch):
assert backend._spec_drafter_kind == "dflash"
+def test_a_dropped_unloadable_drafter_reports_its_own_reason(monkeypatch):
+ """ "Present but unopenable" is not "not found", and the remedies differ.
+
+ drafter_not_found tells a local load to place an mtp-*.gguf that is already on disk,
+ and offers a remote load a refetch that returns the same file -- which this branch
+ deliberately stands down. It is also in the frontend's RETRYABLE_SPEC_FALLBACKS, so
+ Apply kept sending a reload the backend then deduped.
+ """
+ backend = _spec_backend(monkeypatch)
+ flags = _spec_flags(
+ backend,
+ # The arm is _mtp_drafter_missing, which is the name-only (Gemma) MTP shape.
+ model_identifier = "unsloth/gemma-4-12b-it-GGUF",
+ speculative_type = "mtp",
+ mtp_draft_path = None,
+ mtp_drafter_unloadable = True,
+ )
+
+ assert backend._spec_fallback_reason == "drafter_unloadable"
+ assert "--model-draft" not in flags
+
+
[email protected]("mode", ["auto", "mtp"])
+def test_a_dropped_sidecar_is_explained_on_a_non_gemma_quant(monkeypatch, mode):
+ """The PR's own motivating case, which the Gemma-only fallback used to miss.
+
+ RVN-Q6_K.gguf reports no nextn_predict_layers and carries no -mtp in its name, so
+ the sidecar beside it was its ONLY MTP signal. Clearing mtp_draft_path made
+ is_mtp_model read false, _mtp_drafter_missing recognised Gemma alone, and neither
+ Auto nor forced MTP reached the fallback: MTP went off with spec_fallback_reason
+ null, so the panel had nothing to show. No --model-draft either way; emitting MTP
+ without a drafter is what aborts llama-server.
+ """
+ backend = _spec_backend(monkeypatch)
+ flags = _spec_flags(
+ backend,
+ model_identifier = "0bserverx/Qwen3.8-27B-Heretic-Abliterated-Uncensored-GGUF",
+ speculative_type = mode,
+ mtp_draft_path = None,
+ mtp_drafter_unloadable = True,
+ )
+
+ assert backend._spec_fallback_reason == "drafter_unloadable"
+ assert "--model-draft" not in flags
+ assert "draft-mtp" not in flags
+
+
+def test_a_plain_quant_with_no_sidecar_at_all_is_untouched(monkeypatch):
+ """The negative: keeping the dropped sidecar as a signal must not invent MTP for a
+ model that never had any. Nothing was dropped here, so nothing is explained."""
+ backend = _spec_backend(monkeypatch)
+ _spec_flags(
+ backend,
+ model_identifier = "0bserverx/Qwen3.8-27B-Heretic-Abliterated-Uncensored-GGUF",
+ speculative_type = "auto",
+ mtp_draft_path = None,
+ )
+
+ assert backend._spec_fallback_reason is None
+
+
+def test_a_genuinely_absent_drafter_still_reports_not_found(monkeypatch):
+ """The negative: the new reason must not swallow the case it was split out of."""
+ backend = _spec_backend(monkeypatch)
+ _spec_flags(
+ backend,
+ model_identifier = "unsloth/gemma-4-12b-it-GGUF",
+ speculative_type = "mtp",
+ mtp_draft_path = None,
+ )
+
+ assert backend._spec_fallback_reason == "drafter_not_found"
+
+
def test_dspark_keeps_first_refusal_when_a_repo_ships_both(monkeypatch):
"""Mirrors llama.cpp's own downloader, which ranks dspark ahead of dflash.
In practice a repo ships one kind or neither; this pins that adding DFlash
@@ -3902,3 +3976,153 @@ def test_split_completeness_is_scoped_to_the_files_own_directory():
assert not split_listing_is_complete(names, names[1])
whole = ["Q4/model-00001-of-00002.gguf", "Q4/model-00002-of-00002.gguf"]
assert split_listing_is_complete(whole, whole[0])
+
+
+def _write_drafter_gguf(
+ path: Path,
+ *,
+ tensors: list[str],
+ arch: str = "qwen35",
+ shared: bool = False,
+ split_count: int = 0,
+) -> Path:
+ import numpy as np
+ from gguf import GGUFWriter
+
+ writer = GGUFWriter(str(path), arch)
+ if shared:
+ writer.add_bool(f"{arch}.nextn_shared_target_tensors", True)
+ if split_count:
+ writer.add_uint16("split.count", split_count)
+ for name in tensors:
+ writer.add_tensor(name, np.zeros((2, 2), dtype = np.float32))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+ writer.write_tensors_to_file()
+ writer.close()
+ return path
+
+
+_HEAD_EXTRACT = ["output.weight", "blk.64.attn_norm.weight", "blk.64.nextn.eh_proj.weight"]
+
+
+ "tensors,shared,expected",
+ [
+ # Published shapes: a bare head extract, unsloth's MTP/ head, a -shared- head. Each
+ # verdict is what llama-server b10909-mix-bea84f7 gives for that file as
+ # --model-draft: the first ends the launch on a missing token_embd.weight, the
+ # other two serve.
+ (_HEAD_EXTRACT, False, False),
+ (["token_embd.weight", "output_norm.weight", *_HEAD_EXTRACT], False, True),
+ (_HEAD_EXTRACT, True, True),
+ ],
+)
+def test_mtp_drafter_loads_standalone(tmp_path, tensors, shared, expected):
+ from core.inference.llama_cpp import _mtp_drafter_loads_standalone
+ drafter = _write_drafter_gguf(tmp_path / "mtp-model.gguf", tensors = tensors, shared = shared)
+ assert _mtp_drafter_loads_standalone(str(drafter)) is expected
+
+
+def test_a_lone_file_claiming_to_be_a_split_set_is_still_judged(tmp_path):
+ """``split.count`` alone is not an excuse: llama-server opens shards by FILENAME.
+
+ Exempting anything whose header said ``split.count > 1`` let a head-only file
+ through untested, and llama-server then ended the launch on it anyway (measured).
+ The exemption belongs to a set whose shards cannot all be inspected, not to a
+ single file that merely declares one.
+ """
+ from core.inference.llama_cpp import _mtp_drafter_loads_standalone
+
+ drafter = _write_drafter_gguf(tmp_path / "mtp-model.gguf", tensors = _HEAD_EXTRACT, split_count = 2)
+ assert _mtp_drafter_loads_standalone(str(drafter)) is False
+
+
+def test_a_complete_split_drafter_is_judged_across_every_shard(tmp_path):
+ """Shard 1 need not list every tensor, so the whole set answers, as for cls.*."""
+ from core.inference.llama_cpp import _mtp_drafter_loads_standalone
+
+ _write_drafter_gguf(
+ tmp_path / "mtp-model-00001-of-00002.gguf", tensors = _HEAD_EXTRACT, split_count = 2
+ )
+ _write_drafter_gguf(
+ tmp_path / "mtp-model-00002-of-00002.gguf",
+ tensors = ["token_embd.weight"],
+ split_count = 2,
+ )
+ # The embeddings live in shard 2; judging shard 1 alone would drop a working set.
+ assert _mtp_drafter_loads_standalone(str(tmp_path / "mtp-model-00001-of-00002.gguf")) is True
+
+ # The same set with the embeddings nowhere in it: every shard is readable and none
+ # carries them, so this one really cannot be opened as a draft.
+ headless = tmp_path / "headless"
+ headless.mkdir()
+ _write_drafter_gguf(
+ headless / "mtp-model-00001-of-00002.gguf", tensors = _HEAD_EXTRACT, split_count = 2
+ )
+ _write_drafter_gguf(
+ headless / "mtp-model-00002-of-00002.gguf",
+ tensors = ["blk.65.nextn.eh_proj.weight"],
+ split_count = 2,
+ )
+ assert _mtp_drafter_loads_standalone(str(headless / "mtp-model-00001-of-00002.gguf")) is False
+
+
+def test_an_incomplete_split_drafter_fails_open(tmp_path):
+ """A shard that is not here could hold the embeddings, so llama-server decides."""
+ from core.inference.llama_cpp import _mtp_drafter_loads_standalone
+
+ lone = _write_drafter_gguf(
+ tmp_path / "mtp-model-00001-of-00003.gguf", tensors = _HEAD_EXTRACT, split_count = 3
+ )
+ assert _mtp_drafter_loads_standalone(str(lone)) is True
+
+
+def test_the_drafter_verdict_is_cached_per_file_version(tmp_path):
+ """The estimate route asks on every settings change; it must not reparse each time."""
+ from core.inference.llama_cpp import _mtp_drafter_loads_standalone
+ from utils.models import gguf_metadata
+
+ drafter = _write_drafter_gguf(tmp_path / "mtp-model.gguf", tensors = _HEAD_EXTRACT)
+ calls: list[str] = []
+ real = gguf_metadata._parse_gguf_has_named_tensor
+
+ def counting(path, wanted_name):
+ calls.append(path)
+ return real(path, wanted_name)
+
+ gguf_metadata._parse_gguf_has_named_tensor = counting
+ try:
+ assert _mtp_drafter_loads_standalone(str(drafter)) is False
+ assert _mtp_drafter_loads_standalone(str(drafter)) is False
+ assert _mtp_drafter_loads_standalone(str(drafter)) is False
+ assert len(calls) == 1
+
+ # Rewritten in place with its embeddings: a new (mtime, size) is a new answer, so
+ # a repaired sidecar is picked up rather than served from the cache.
+ _write_drafter_gguf(
+ tmp_path / "mtp-model.gguf", tensors = ["token_embd.weight", *_HEAD_EXTRACT]
+ )
+ assert _mtp_drafter_loads_standalone(str(drafter)) is True
+ finally:
+ gguf_metadata._parse_gguf_has_named_tensor = real
+
+
+def test_mtp_drafter_loads_standalone_fails_open_on_an_unreadable_header(tmp_path):
+ """A refusal that rejects working input is worse than the crash it prevents."""
+ from core.inference.llama_cpp import _mtp_drafter_loads_standalone
+
+ junk = tmp_path / "mtp-model.gguf"
+ junk.write_bytes(b"not a gguf")
+ assert _mtp_drafter_loads_standalone(str(junk))
+ assert _mtp_drafter_loads_standalone(str(tmp_path / "missing.gguf"))
+
+
+def test_mtp_drafter_loads_standalone_needs_only_token_embd(tmp_path):
+ from core.inference.llama_cpp import _mtp_drafter_loads_standalone
+ drafter = _write_drafter_gguf(
+ tmp_path / "mtp-model.gguf",
+ tensors = ["token_embd.weight", "output.weight", "blk.48.nextn.eh_proj.weight"],
+ arch = "qwen4exp",
+ )
+ assert _mtp_drafter_loads_standalone(str(drafter))
diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.py
index 52d82067e99..3b6144aba24 100644
--- a/studio/backend/utils/models/gguf_metadata.py
+++ b/studio/backend/utils/models/gguf_metadata.py
@@ -55,6 +55,9 @@ _TTS_AUDIO_TYPE_CACHE: Dict[_CacheKey, Optional[str]] = {}
# Whether the GGUF tensor table contains a sequence-classification head. None means the file could not be read or parsed, so callers can fail closed.
_CLASSIFIER_HEAD_CACHE: Dict[_CacheKey, Optional[bool]] = {}
+# Whether the GGUF tensor table lists one exact tensor name, keyed by (file cache key, tensor name). None = unreadable, so callers can fail open. Backs the MTP drafter launchability check, which the memory-estimate route asks on every settings change.
+_NAMED_TENSOR_CACHE: Dict[Tuple[_CacheKey, str], Optional[bool]] = {}
+
# GGUF header dims for the staged UI in one cached pass (context_length, layer_count, moe_layer_count) so the staged sheet can size every slider before the model loads. None = unreadable / not a GGUF; the native ``{arch}.context_length`` the UI shows before a load is read from here via read_gguf_context_length.
_DIMS_CACHE: Dict[_CacheKey, Optional[Dict[str, Optional[int]]]] = {}
@@ -457,6 +460,122 @@ def _gguf_has_classifier_head(path: str) -> Optional[bool]:
return False if results and all(result is False for result in results) else None
+def _parse_gguf_has_named_tensor(path: str, wanted_name: str) -> Optional[bool]:
+ """Whether the GGUF tensor table lists exactly ``wanted_name``.
+
+ Same stream and bounds as ``_parse_gguf_has_classifier_head``, which matches a
+ prefix; this matches a full name, without mapping the tensor data in.
+ """
+ try:
+ with open(path, "rb") as f:
+ head = f.read(24)
+ if len(head) < 24:
+ return None
+ magic, _version, tensor_count, kv_count = struct.unpack("<IIQQ", head)
+ if magic != _GGUF_MAGIC or tensor_count > 1 << 20 or kv_count > 1 << 20:
+ return None
+
+ for _ in range(kv_count):
+ klen_bytes = f.read(8)
+ if len(klen_bytes) < 8:
+ return None
+ klen = struct.unpack("<Q", klen_bytes)[0]
+ if klen > 1 << 20 or len(f.read(klen)) < klen:
+ return None
+ vtype_bytes = f.read(4)
+ if len(vtype_bytes) < 4 or not _skip_gguf_value(
+ f, struct.unpack("<I", vtype_bytes)[0]
+ ):
+ return None
+
+ wanted = wanted_name.encode("utf-8")
+ for _ in range(tensor_count):
+ nlen_bytes = f.read(8)
+ if len(nlen_bytes) < 8:
+ return None
+ nlen = struct.unpack("<Q", nlen_bytes)[0]
+ if nlen > 1 << 20:
+ return None
+ name_bytes = f.read(nlen)
+ ndim_bytes = f.read(4)
+ if len(name_bytes) < nlen or len(ndim_bytes) < 4:
+ return None
+ n_dimensions = struct.unpack("<I", ndim_bytes)[0]
+ if n_dimensions > 16:
+ return None
+ # dimensions (u64 each), ggml type (u32), and data offset (u64)
+ trailer_size = n_dimensions * 8 + 4 + 8
+ if len(f.read(trailer_size)) < trailer_size:
+ return None
+ if name_bytes == wanted:
+ return True
+ except OSError as e:
+ logger.debug(f"_parse_gguf_has_named_tensor: cannot open {path}: {e}")
+ return None
+ except Exception as e:
+ logger.debug(f"_parse_gguf_has_named_tensor: parse failure on {path}: {e}")
+ return None
+ return False
+
+
+def _gguf_shard_has_named_tensor(path: str, wanted_name: str) -> Optional[bool]:
+ """Cached single-shard tensor-name lookup, keyed by (path, mtime, size, name)."""
+ fkey = _cache_key(path)
+ if fkey is None:
+ return None
+ ckey = (fkey, wanted_name)
+ with _CACHE_LOCK:
+ if ckey in _NAMED_TENSOR_CACHE:
+ return _NAMED_TENSOR_CACHE[ckey]
+ result = _parse_gguf_has_named_tensor(path, wanted_name)
+ with _CACHE_LOCK:
+ while len(_NAMED_TENSOR_CACHE) >= _CACHE_MAX_ENTRIES:
+ try:
+ _NAMED_TENSOR_CACHE.pop(next(iter(_NAMED_TENSOR_CACHE)))
+ except StopIteration:
+ break
+ _NAMED_TENSOR_CACHE[ckey] = result
+ return result
+
+
+def mtp_drafter_loads_standalone(path: str) -> bool:
+ """Can llama-server open *path* as a ``--model-draft``?
+
+ A draft head is opened as a complete model unless it can borrow the target's
+ embeddings, so one carrying neither its own ``token_embd.weight`` nor
+ ``<arch>.nextn_shared_target_tensors`` ends the launch with
+ ``check_tensor_dims: tensor 'token_embd.weight' not found``. Measured against the
+ shipped llama.cpp: the metadata flag is what admits a head into
+ ``borrow_shared_tensor``. Only that one tensor decides, since the rest of the
+ borrowable set is per-architecture (``qwen4exp`` creates no ``output_norm``), so a
+ stricter refusal would reject heads that work.
+
+ Split-aware like ``_gguf_has_classifier_head``: llama-server opens sibling shards
+ implicitly, so a tensor absent from shard 1 may live in shard 2. Fails open on
+ anything it cannot see, leaving the verdict to llama-server.
+ """
+ from utils.models.model_config import colocated_split_shards
+
+ try:
+ shards, complete = colocated_split_shards(Path(path))
+ except Exception as e:
+ logger.debug(f"mtp_drafter_loads_standalone: cannot enumerate shards for {path}: {e}")
+ return True
+ if not complete or not shards:
+ return True
+
+ carries = [_gguf_shard_has_named_tensor(str(shard), "token_embd.weight") for shard in shards]
+ if any(result is True for result in carries):
+ return True
+ if any(result is None for result in carries):
+ return True
+
+ architecture = read_gguf_architecture(path)
+ if architecture and _read_gguf_bool(path, f"{architecture}.nextn_shared_target_tensors"):
+ return True
+ return False
+
+
def _parse_gguf_bool(path: str, wanted_key: str) -> Optional[bool]:
"""Bool value of ``wanted_key`` (GGUF vtype 7), or ``None`` if absent or unreadable. Mirrors ``_parse_gguf_header`` for a single bool key."""
try:
diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
index bf110c666f7..48ff54f1b0d 100644
--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx
+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
@@ -436,6 +436,11 @@ function specFallbackMessage({
return `This model fits in VRAM but its ${drafter} drafter does not, so Auto kept your context length and turned ${drafter} off for this load. Choose ${drafter} in Settings to force it, at a smaller context.`;
case "runtime_error":
return `${drafter} could not start for this model on the installed llama.cpp build, so it is running without speculative decoding.`;
+ case "drafter_unloadable":
+ // The file IS beside the model; it just cannot be opened as a draft. So no "place
+ // the sidecar" and no "check your network": both name a remedy that changes
+ // nothing, and the backend stands the refetch down for exactly this case.
+ return "This model's MTP drafter file cannot be loaded as a draft model, so it is running without MTP. The sidecar beside the model is a head-only file that llama.cpp cannot open on its own. Replace it with a self-contained mtp-*.gguf, then reload the model.";
case "drafter_not_found":
if (drafter === "DSpark") {
return isLocalGguf
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index eda05a4fa11..dcbd82445c2 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -132,9 +132,17 @@ type SettingCheck = {
/** Fallback reasons the backend retries on an IDENTICAL next load, from the arms of
* `LlamaCppBackend._runtime_matches_intent` that return False to force a repair. The rest are
* excluded on purpose: "drafter_no_vram" and "mla_mtp_disabled" are Auto-mode policy, and
- * "runtime_error" only reopens when the draft count changes, which the comparison sees. */
+ * "runtime_error" only reopens when the draft count changes, which the comparison sees.
+ *
+ * "drafter_unloadable" is here because the sheet's remedy is to replace the sidecar in place,
+ * and `adoptable` in use-chat-model-runtime would otherwise skip `/load`, so the backend's
+ * content re-check never runs. An unchanged file still dedupes through already_loaded. It
+ * needs no `sendsGgufPath` exclusion, unlike "drafter_not_found", because the re-check lives
+ * in `_runtime_matches_intent`'s drafter comparison rather than the refetch arm that requires
+ * `intent.gguf_path is None`. */
const RETRYABLE_SPEC_FALLBACKS = new Set([
"drafter_not_found",
+ "drafter_unloadable",
"binary_no_mtp",
"binary_outdated",
]);
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index 2d1314b62d8..3f2beb50f9b 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -1487,6 +1487,42 @@ test("a retryable drafter failure declines the shortcut", () => {
}
});
+test("a repaired drafter has to reach the backend to be re-checked", () => {
+ // The sheet's remedy is to replace the sidecar in place, and `adoptable` skips /load
+ // when this returns false, so the re-check would never run. An unchanged file still
+ // dedupes server-side, so declining the shortcut is cheap rather than a teardown.
+ for (const mode of ["auto", "mtp", "mtp+ngram"]) {
+ assert.equal(
+ residentSpeculativeNeedsRepair(
+ { spec_fallback_reason: "drafter_unloadable", spec_drafter_kind: "mtp" },
+ mode,
+ ),
+ true,
+ `drafter_unloadable under ${mode} must reload`,
+ );
+ }
+ // No sendsGgufPath exclusion, unlike drafter_not_found: the re-check sits in the drafter
+ // comparison, which a standalone .gguf load reaches, not in the gguf_path-gated refetch.
+ assert.equal(
+ residentSpeculativeNeedsRepair(
+ { spec_fallback_reason: "drafter_unloadable", spec_drafter_kind: "mtp" },
+ "auto",
+ true,
+ ),
+ true,
+ "a standalone .gguf load must still reload for a repaired drafter",
+ );
+ // And the mode still has to be one that asked for a drafter at all.
+ assert.equal(
+ residentSpeculativeNeedsRepair(
+ { spec_fallback_reason: "drafter_unloadable", spec_drafter_kind: "mtp" },
+ "off",
+ ),
+ false,
+ "spec off asked for no drafter, so there is nothing to repair",
+ );
+});
+
test("an Auto-mode policy downgrade is not a repair the load can make", () => {
for (const reason of [
"drafter_no_vram",