Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

studio: fall back to the MTP/ folder when a repo publishes no root drafter (#10124)

Daniel Han9d4ee613acff
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 459e4c8f311..e0b3c40e1f2 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2758,20 +2758,86 @@ def _pick_dspark(candidates: list[str]) -> Optional[str]:
return files[0] if files else None
-def _pick_mtp(candidates: list[str]) -> Optional[str]:
+_MTP_SHARD_SUFFIX_RE = re.compile(r"-[0-9]{5}-of-[0-9]{5}$")
+
+
+def _is_published_mtp_drafter_name(path: str) -> bool:
+ """Does *path*'s BASENAME name a published MTP head?
+
+ ``_is_mtp_only_drafter_path`` accepts anything under ``MTP/``, which is right
+ for excluding companions from menus and too broad for choosing what to launch:
+ an mmproj, an imatrix or a stray weight copy would go to ``--model-draft``.
+ Same rule as ``detect_mtp_file`` -- ``mtp-<model>`` or the older
+ ``<model>-MTP`` -- shard suffix stripped first, since an old-scheme split copy
+ is ``<model>-Q8_0-MTP-00001-of-00002.gguf``, whose stem lacks ``-mtp``."""
+ lower = Path(path).name.lower()
+ if not lower.endswith(".gguf"):
+ return False
+ stem = _MTP_SHARD_SUFFIX_RE.sub("", Path(lower).stem)
+ return lower.startswith("mtp-") or stem.endswith("-mtp")
+
+
+def _pick_mtp(candidates: list[str], *, allow_nested: bool = True) -> Optional[str]:
"""The MTP drafter a listing offers, or None. Module level for the same reason
``_pick_dspark`` is: both are handed a live repo listing as well as a snapshot,
- and ``/kv-cache-estimate`` has to price the drafter the launch will open."""
- # Root-level only: MTP/ subdir copies now share the mtp- prefix but
- # are explicit-selection, not auto-fetch (they'd sort ahead of root).
- # The mtp- prefix also excludes AppleDouble shadows ("._mtp-x.gguf"), which
- # is why this picker needs no drop_shadowed_appledouble_names of its own.
+ and ``/kv-cache-estimate`` has to price the drafter the launch will open.
+
+ ``allow_nested=False`` restricts the answer to a root mirror, which is what
+ every architecture but qwen4exp gets -- see ``_pick_mtp_root_only``."""
+ from hub.utils.gguf import drop_shadowed_appledouble_names
+ from utils.models.drafters import split_listing_is_complete
+ from utils.models.drafters.preference import mtp_preference_key
+
+ names = drop_shadowed_appledouble_names(list(candidates))
+
+ def _launchable(name: str) -> bool:
+ # Settled before ranking, as detect_mtp_file settles it at collection:
+ # llama.cpp resolves sibling shards from the first one's directory, so half
+ # a set is unusable and _download_companion_gguf answers None to it. Ranked
+ # first and rejected after, it would shadow a complete lower-ranked head.
+ return split_listing_is_complete(names, name)
+
+ # Root first, so a repo mirroring one head at the root (Gemma 4) still resolves to it
+ # rather than to a subdir copy, which would sort ahead. The mtp- prefix also excludes
+ # AppleDouble shadows ("._mtp-x.gguf"), so this bucket needs no filtering of its own.
mtp_files = sorted(
f
- for f in candidates
- if f.lower().endswith(".gguf") and "/" not in f and Path(f).name.lower().startswith("mtp-")
+ for f in names
+ if f.lower().endswith(".gguf")
+ and "/" not in f
+ and Path(f).name.lower().startswith("mtp-")
+ and _launchable(f)
)
- return mtp_files[0] if mtp_files else None
+ if mtp_files:
+ return mtp_files[0]
+
+ # No root mirror, or none of them complete: fall back to the MTP/ folder, which
+ # is the only place Qwen3.8-Flash-Next publishes its heads. This is the policy
+ # _cached_repo_mtp_drafter already applies to the offline cache, so without it a
+ # user holding a cached copy gets speculation and a fresh install does not.
+ if not allow_nested:
+ return None
+ nested = sorted(
+ (
+ name
+ for name in names
+ if "/" in name and _is_published_mtp_drafter_name(name) and _launchable(name)
+ ),
+ key = mtp_preference_key,
+ )
+ return nested[0] if nested else None
+
+
+def _pick_mtp_root_only(candidates: list[str]) -> Optional[str]:
+ """``_pick_mtp`` for everything but qwen4exp: the behaviour every model had
+ before the ``MTP/`` fallback existed.
+
+ llama.cpp loads the draft model whenever one is passed, so a sidecar displaces
+ an embedded head rather than adding to it. On Qwen3.8-27B UD-Q4_K_XL the two
+ draft identically (143 of 223, byte-identical output), so the 1.37 GB copy buys
+ nothing. A root mirror is still taken: publishing one beside the weights says
+ it is the one to use."""
+ return _pick_mtp(candidates, allow_nested = False)
def _pick_mmproj(candidates: list[str]) -> Optional[str]:
@@ -12522,6 +12588,32 @@ class LlamaCppBackend:
probe._read_gguf_metadata(gguf_path)
return probe._is_diffusion
+ @classmethod
+ def _gguf_path_wants_nested_mtp(cls, gguf_path: str) -> bool:
+ """May this target take a drafter from the ``MTP/`` folder?
+
+ Only ``qwen4exp``, and only with no head of its own. Both come from one
+ header read, via the same probe-instance trick as
+ ``_gguf_path_is_diffusion``: this runs before the load reads metadata into
+ ``self`` and must not overwrite the live model's.
+
+ Architecture gates it rather than the head count alone, because a repo can
+ publish an ``MTP/`` folder for a subset of its quants: Qwen3.8-27B writes
+ ``qwen35.nextn_predict_layers`` on all 24, 0 on the four whose head was
+ dropped, and ships one sidecar for those four. Head count alone would hand
+ it to them, a wider change than intended.
+
+ Fails closed on an unreadable header, onto the path every model took before
+ this fallback existed."""
+ try:
+ probe = object.__new__(cls)
+ probe._model_identifier = "mtp-head-probe"
+ probe._read_gguf_metadata(gguf_path)
+ return probe._architecture == "qwen4exp" and not probe._nextn_predict_layers
+ except Exception as e:
+ logger.debug("Nested MTP eligibility probe failed for %s: %s", gguf_path, e)
+ return False
+
def _reject_vulkan_diffusion_gpu_ids_before_teardown(
self, gguf_path: str, model_identifier: str
) -> None:
@@ -13978,13 +14070,25 @@ class LlamaCppBackend:
hf_repo: str,
*,
cache_dir: Optional[str] = None,
+ allow_nested: bool = True,
) -> Optional[str]:
"""A drafter already in this repo's local HF cache, reused offline when a
fresh copy can't be fetched. Prefers a repo-root ``mtp-*.gguf`` across all
- cached snapshots; else an existing ``MTP/`` copy (any precision -- the
- target verifies every drafted token). None if none is cached."""
+ cached snapshots; else an existing ``MTP/`` copy. None if none is cached.
+
+ Ranked with ``_pick_mtp``'s keys, or offline and online disagree about one
+ cache: lexical order put ``mtp-Qwen3.8-Flash-Next-BF16.gguf`` first, so a
+ cached user got the 7.77 GB slowest head while a fresh install downloaded
+ the 2.79 GB shared Q8_0 one.
+
+ ``allow_nested=False`` drops the ``MTP/`` half, so a non-qwen4exp target
+ resolves the same way offline as online (``_pick_mtp_root_only``)."""
try:
- from utils.models.model_config import _iter_hf_cache_snapshots
+ from utils.models.drafters.preference import mtp_preference_key
+ from utils.models.model_config import (
+ _drafter_split_is_complete,
+ _iter_hf_cache_snapshots,
+ )
roots: list[Path] = []
subdirs: list[Path] = []
@@ -13994,15 +14098,27 @@ class LlamaCppBackend:
else _iter_hf_cache_snapshots(hf_repo, cache_dir)
)
for snap in snapshots: # newest first
- for f in sorted(_gguf_snapshot_files(snap)):
- # MTP only: a DSpark drafter needs --spec-type draft-dspark,
- # so it must never be launched as an MTP one.
- if _is_mtp_only_drafter_path(f):
- (roots if "/" not in f else subdirs).append(snap / f)
+ snap_roots: list[str] = []
+ snap_subdirs: list[str] = []
+ for f in _gguf_snapshot_files(snap):
+ # MTP only: a DSpark drafter needs --spec-type draft-dspark.
+ # The nested tier needs a published drafter NAME too, since
+ # everything under MTP/ classifies as one and an mmproj or
+ # imatrix there would otherwise go to --model-draft.
+ if "/" not in f:
+ if _is_mtp_only_drafter_path(f):
+ snap_roots.append(f)
+ elif _is_mtp_only_drafter_path(f) and _is_published_mtp_drafter_name(f):
+ snap_subdirs.append(f)
+ # Root lexical, nested by preference: the tiers as _pick_mtp orders them.
+ roots.extend(snap / f for f in sorted(snap_roots))
+ subdirs.extend(snap / f for f in sorted(snap_subdirs, key = mtp_preference_key))
# Keep snapshot order (newest first), root before any MTP/ copy, so a
# newer main GGUF pairs with the newest cached drafter, not a stale one.
- for cand in roots + subdirs:
- if cand.is_file():
+ for cand in roots + subdirs if allow_nested else roots:
+ # Half a split set is not a drafter, and offline there is no fetch
+ # to complete it: the rule _download_companion_gguf applies too.
+ if cand.is_file() and _drafter_split_is_complete(cand):
return str(cand)
except Exception as e:
logger.debug("Cached MTP drafter lookup failed for %s: %s", hf_repo, e)
@@ -14015,23 +14131,30 @@ class LlamaCppBackend:
hf_token: Optional[str] = None,
cancel_event: Optional[threading.Event] = None,
near_path: Optional[str] = None,
+ allow_nested: bool = True,
) -> Optional[str]:
"""Download the separate MTP drafter (speculative head) from a GGUF repo.
- Targets the repo-root ``mtp-*.gguf`` companion -- the Q8_0 drafter
+ Prefers the repo-root ``mtp-*.gguf`` companion -- the Q8_0 drafter
unsloth mirrors there for llama.cpp ``-hf`` auto-discovery (smallest,
- recommended for speculation). Repos that bake the MTP head into the
- main GGUF (e.g. Qwen) ship no such sibling and this returns None. The
- higher-precision copies under ``MTP/`` are for explicit selection and
- are intentionally skipped. Returns the local path, or None.
+ recommended for speculation) -- and falls back to the ``MTP/`` folder for
+ a repo that publishes no root mirror, as Qwen3.8-Flash-Next and
+ Qwen3.8-27B do. Repos that bake the MTP head into the main GGUF ship
+ neither and this returns None. Returns the local path, or None.
+
+ ``allow_nested=False`` skips the ``MTP/`` fallback, which is what every
+ architecture but qwen4exp gets: elsewhere the sidecar would only displace a
+ head the file already carries.
"""
cancel_event = cancel_event if cancel_event is not None else self._cancel_event
if cancel_event.is_set():
return None
+ pick = _pick_mtp if allow_nested else _pick_mtp_root_only
+
if near_path:
- cached = _companion_snapshot_sibling(near_path, _pick_mtp)
+ cached = _companion_snapshot_sibling(near_path, pick)
if cached:
logger.info("Reusing cached MTP drafter: %s", cached)
return cached
@@ -14044,6 +14167,7 @@ class LlamaCppBackend:
cached = self._cached_repo_mtp_drafter(
hf_repo,
cache_dir = _hub_cache_dir_for_snapshot_path(near_path),
+ allow_nested = allow_nested,
)
if cached:
logger.info(f"Reusing cached MTP drafter (offline): {cached}")
@@ -14052,7 +14176,7 @@ class LlamaCppBackend:
return self._download_companion_gguf(
hf_repo = hf_repo,
hf_token = hf_token,
- pick = _pick_mtp,
+ pick = pick,
label = "MTP drafter",
cancel_event = cancel_event,
near_path = near_path,
@@ -17866,12 +17990,19 @@ class LlamaCppBackend:
near_path = model_path,
)
# Auto-download the separate MTP drafter (e.g. Gemma) when
- # the requested spec mode can use it. Repos with the head
- # baked into the main GGUF (Qwen) have no mtp- sibling and
- # this no-ops, so the size gate stays out of it: a separate
- # drafter speeds up even sub-3B (Gemma E2B), and the resolver
- # below decides the final emission. Skipped only when the
- # user disabled MTP or drives --spec-type manually.
+ # the requested spec mode can use it. The size gate stays out
+ # of it: a separate drafter speeds up even sub-3B (Gemma E2B),
+ # and the resolver below decides the final emission. Skipped
+ # only when the user disabled MTP or drives --spec-type
+ # manually.
+ #
+ # The MTP/ fallback is qwen4exp only, whose published GGUFs
+ # carry no head at all; every other arch keeps its root-mirror
+ # behaviour. Qwen3.8-27B bakes the head into 20 of 24 quants,
+ # and llama.cpp prefers a -md drafter over an embedded one, so
+ # on UD-Q4_K_XL the two accept identically (143 of 223) for
+ # byte-identical output: 1.37 GB to displace what is loaded.
+ # Read from the file, not self, whose metadata is set below.
if (
not mtp_draft_path
and _spec_canon in ("auto", "mtp", "mtp+ngram")
@@ -17882,6 +18013,7 @@ class LlamaCppBackend:
hf_token = hf_token,
cancel_event = download_cancel_event,
near_path = model_path,
+ allow_nested = self._gguf_path_wants_nested_mtp(model_path),
)
# "auto" is included: DSpark is the default whenever the repo
# ships a sidecar. Repos without one no-op, exactly like the
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index 349af239b2a..c708de7a807 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -3746,8 +3746,8 @@ def _resolve_mtp_drafter(
loader's own ``_pick_mtp`` for an HF snapshot, and ``detect_mtp_file`` for a
local folder, which is what ``model_config`` calls when it builds the launch.
A bespoke scan here is how the estimate ends up pricing a different file from
- the one llama-server opens: ``_pick_mtp`` is root-level and prefix-matched, so
- it cannot be fooled by a directory that happens to be named ``mtp``, it finds
+ the one llama-server opens: ``_pick_mtp`` is prefix-matched, so it cannot be
+ fooled by a directory that happens to be named ``mtp``, it finds
the snapshot-root companion when the weights sit in a quant subdirectory, it
sorts on relative strings rather than ``Path`` objects (whose ordering is
case-folded on Windows and not on POSIX, so two hosts really can disagree),
@@ -3756,17 +3756,24 @@ def _resolve_mtp_drafter(
"""
try:
from core.inference.llama_cpp import (
+ LlamaCppBackend,
_companion_snapshot_sibling,
_pick_mtp,
+ _pick_mtp_root_only,
_snapshot_dir_of,
)
if _snapshot_dir_of(main_gguf_path) is not None:
- # An HF snapshot. ``_download_mtp`` resolves through ``_pick_mtp``,
- # which is root-level only, so the ``MTP/`` precision copies are not
- # auto-fetched and must not be priced: charging one would report a
- # reserve for a drafter the load will not open.
- drafter = _companion_snapshot_sibling(main_gguf_path, _pick_mtp)
+ # An HF snapshot. ``_download_mtp`` takes the ``MTP/`` fallback only
+ # for qwen4exp with no head of its own, so the same gate applies here:
+ # pricing a nested copy for any other model reports a reserve for a
+ # drafter the load will not open.
+ pick = (
+ _pick_mtp
+ if LlamaCppBackend._gguf_path_wants_nested_mtp(main_gguf_path)
+ else _pick_mtp_root_only
+ )
+ drafter = _companion_snapshot_sibling(main_gguf_path, pick)
else:
# A local folder, where the load path (model_config) pairs the drafter
# to the weight by name so a multi-model folder cannot attach a foreign
@@ -3781,8 +3788,6 @@ def _resolve_mtp_drafter(
# the load planner sizes the drafter with _get_gguf_size_bytes, and a
# split companion reserves every shard. Billing shard 1 alone reports a
# fit for a launch that allocates several times as much.
- from core.inference.llama_cpp import LlamaCppBackend
-
return drafter, LlamaCppBackend._get_gguf_size_bytes(drafter)
except Exception:
return None, 0
diff --git a/studio/backend/tests/test_kv_cache_estimate_route.py b/studio/backend/tests/test_kv_cache_estimate_route.py
index 7fc53841d46..5d9e3e274f0 100644
--- a/studio/backend/tests/test_kv_cache_estimate_route.py
+++ b/studio/backend/tests/test_kv_cache_estimate_route.py
@@ -254,15 +254,163 @@ class TestDrafterDiscoveryMatchesTheLoader:
got, _ = models_routes._resolve_mtp_drafter(str(main), search_root = str(root))
assert got != str(main)
- def test_nested_mtp_subdir_copies_are_skipped(self, tmp_path):
- """MTP/ copies are explicit-selection; the loader does not auto-fetch them."""
+ def test_nested_mtp_subdir_copy_is_used_when_no_root_mirror(self, tmp_path):
+ """A repo publishing heads only under MTP/, as Qwen3.8-Flash-Next and
+ Qwen3.8-27B do, still gets a drafter. _cached_repo_mtp_drafter already
+ reuses such a copy offline, so skipping it here gave a cached user
+ speculation and a fresh one none."""
snap = self._snapshot(tmp_path)
- main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ nested = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(nested)
+
+ def test_nested_fallback_prefers_q8_over_bf16(self, tmp_path):
+ """bf16 sorts first by name and is the worst head: larger and slower,
+ because a draft step is dominated by the LM head."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ _write_gguf(snap / "MTP" / "mtp-model-BF16.gguf", _MLA_NO_HEAD)
+ q8 = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(q8)
+
+ def test_nested_fallback_prefers_the_shared_head(self, tmp_path):
+ """A -shared- head borrows the target's token_embd/output instead of
+ carrying its own: 1.35 GB smaller at Q8_0 and no worse, accepting
+ identically to the full head (159 of 284) on the shipped prebuilt. Only
+ qwen4exp reaches this path, and its MTP graph and the borrow ship in the
+ same fork, so a build that can draft one carries the other."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
+ shared = _write_gguf(snap / "MTP" / "mtp-model-shared-Q8_0.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(shared)
+
+ def test_precision_outranks_the_shared_preference(self, tmp_path):
+ """Q8_0 first is the stronger rule: a shared bf16 head is both larger than
+ a full Q8_0 one and slower, since a draft step is dominated by the LM head
+ and that head is cheaper to execute at 8 bits."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ q8 = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
+ _write_gguf(snap / "MTP" / "mtp-model-shared-BF16.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(q8)
+
+ def test_nested_fallback_is_skipped_for_other_architectures(self, tmp_path):
+ """The fallback is qwen4exp only. Qwen3.8-27B bakes the head into 20 of
+ its 24 quants, and llama.cpp prefers a -md drafter over an embedded one, so
+ pricing the sidecar would reserve for a file the load will not open."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen35")
+ _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
+
+ got, size = models_routes._resolve_mtp_drafter(str(main))
+ assert got is None, f"billed a sidecar the load will not fetch: {got} ({size} bytes)"
+
+ def test_nested_fallback_is_skipped_for_a_qwen4exp_with_its_own_head(self, tmp_path):
+ """Architecture is not the whole gate: a qwen4exp GGUF converted with the
+ block kept needs no sidecar either. Nothing published does this, so this
+ pins the intent."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(
+ snap / "model-Q4_K_M.gguf",
+ {**_MLA_NO_HEAD, "nextn_predict_layers": 1},
+ arch = "qwen4exp",
+ )
_write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
got, _ = models_routes._resolve_mtp_drafter(str(main))
assert got is None
+ def test_a_zero_head_count_still_takes_the_nested_fallback(self, tmp_path):
+ """Qwen3.8-27B writes the key on every quant and sets it to 0 on the four
+ with no head, so presence of the key must not read as a head."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(
+ snap / "model-UD-IQ1_S.gguf",
+ {**_MLA_NO_HEAD, "nextn_predict_layers": 0},
+ arch = "qwen4exp",
+ )
+ nested = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(nested)
+
+ def test_a_root_mirror_is_taken_by_every_architecture(self, tmp_path):
+ """Only the nested fallback is gated. A root mtp- companion beside the
+ weights says it is the one to use, which is how Gemma 4 ships."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", {**_MLA_NO_HEAD, "nextn_predict_layers": 1})
+ companion = _write_gguf(snap / "mtp-model.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(companion)
+
+ def test_a_non_drafter_parked_under_mtp_is_not_launched(self, tmp_path):
+ """Everything under MTP/ classifies as a drafter, which keeps companions
+ out of variant menus and is too broad for choosing what to launch: an
+ mmproj or an imatrix would be handed to --model-draft."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ _write_gguf(snap / "MTP" / "mmproj-BF16.gguf", _MLA_NO_HEAD)
+ _write_gguf(snap / "MTP" / "imatrix_unsloth.gguf", _MLA_NO_HEAD)
+ _write_gguf(snap / "MTP" / "Qwen3.8-Flash-Next-Q8_0.gguf", _MLA_NO_HEAD)
+
+ got, size = models_routes._resolve_mtp_drafter(str(main))
+ assert got is None, f"would launch a non-drafter as the draft model: {got} ({size} bytes)"
+
+ def test_the_older_mtp_suffix_naming_still_resolves(self, tmp_path):
+ """Gemma 4 published <model>-MTP.gguf before the mtp- prefix, and the
+ local scan still accepts it, so the hub path must too."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ old = _write_gguf(snap / "MTP" / "model-Q8_0-MTP.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(old)
+
+ def test_an_incomplete_nested_split_does_not_shadow_a_complete_head(self, tmp_path):
+ """llama.cpp resolves sibling shards from the first one's directory, so
+ half a set is unusable and _download_companion_gguf answers None to it.
+ Ranked first and rejected after, it would disable speculation with a usable
+ lower-ranked head present."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ _write_gguf(snap / "MTP" / "mtp-model-Q8_0-00001-of-00002.gguf", _MLA_NO_HEAD)
+ complete = _write_gguf(snap / "MTP" / "mtp-model-Q4_K_M.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(complete)
+
+ def test_a_complete_nested_split_is_still_chosen(self, tmp_path):
+ """The filter drops incomplete sets, not split sets."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ first = _write_gguf(snap / "MTP" / "mtp-model-Q8_0-00001-of-00002.gguf", _MLA_NO_HEAD)
+ _write_gguf(snap / "MTP" / "mtp-model-Q8_0-00002-of-00002.gguf", _MLA_NO_HEAD)
+ _write_gguf(snap / "MTP" / "mtp-model-Q4_K_M.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(first)
+
+ def test_an_incomplete_root_split_falls_through_to_the_nested_tier(self, tmp_path):
+ """The root tier returns early, so an incomplete root set used to hide a
+ usable nested head and leave the load with no drafter at all."""
+ snap = self._snapshot(tmp_path)
+ main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD, arch = "qwen4exp")
+ _write_gguf(snap / "mtp-model-Q8_0-00001-of-00002.gguf", _MLA_NO_HEAD)
+ nested = _write_gguf(snap / "MTP" / "mtp-model-Q8_0.gguf", _MLA_NO_HEAD)
+
+ got, _ = models_routes._resolve_mtp_drafter(str(main))
+ assert got == str(nested)
+
def test_root_companion_wins_over_a_nested_copy(self, tmp_path):
snap = self._snapshot(tmp_path)
main = _write_gguf(snap / "model-Q4_K_M.gguf", _MLA_NO_HEAD)
diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py
index 355a0ff7fa2..faf46bad703 100644
--- a/studio/backend/tests/test_mtp_drafter_companion.py
+++ b/studio/backend/tests/test_mtp_drafter_companion.py
@@ -1440,6 +1440,89 @@ def test_a_cached_dspark_drafter_is_never_launched_as_an_mtp_drafter(tmp_path, m
assert found is not None and found.endswith("mtp-model.gguf")
+def test_cached_mtp_lookup_ranks_nested_copies_like_the_download(tmp_path, monkeypatch):
+ """Offline reuse must name the file the online picker names.
+
+ Lexical order put mtp-Qwen3.8-Flash-Next-BF16.gguf first, so a cached user got
+ the 7.77 GB slowest head while a fresh install downloaded the 2.79 GB shared
+ Q8_0 one.
+ """
+ import core.inference.llama_cpp as llama_cpp_module
+
+ published = [
+ f"MTP/mtp-Qwen3.8-Flash-Next-{tier}.gguf"
+ for tier in ("BF16", "Q4_K_M", "Q8_0", "shared-BF16", "shared-Q4_K_M", "shared-Q8_0")
+ ]
+ snap = tmp_path / "snap"
+ for rel in [*published, "UD-IQ1_S/model.gguf"]:
+ (snap / rel).parent.mkdir(parents = True, exist_ok = True)
+ (snap / rel).write_bytes(b"x")
+
+ monkeypatch.setattr(
+ "utils.models.model_config._iter_hf_cache_snapshots", lambda *a, **k: [snap]
+ )
+ backend = llama_cpp_module.LlamaCppBackend.__new__(llama_cpp_module.LlamaCppBackend)
+
+ found = backend._cached_repo_mtp_drafter("unsloth/Qwen3.8-Flash-Next-GGUF")
+ assert found is not None
+ assert Path(found).name == "mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf", (
+ f"offline reuse picked {Path(found).name}; the online picker takes "
+ f"{llama_cpp_module._pick_mtp(published)}"
+ )
+ # Same listing, same answer, whichever path reaches it first.
+ assert Path(found).name == Path(llama_cpp_module._pick_mtp(published)).name
+
+
+def test_cached_mtp_lookup_rejects_non_drafters_parked_under_mtp(tmp_path, monkeypatch):
+ """Everything under MTP/ classifies as an mtp drafter: right for excluding
+ companions from menus, wrong for choosing what to launch."""
+ import core.inference.llama_cpp as llama_cpp_module
+
+ snap = tmp_path / "snap"
+ for rel in ("MTP/mmproj-BF16.gguf", "MTP/imatrix_unsloth.gguf", "model-Q4_K_M.gguf"):
+ (snap / rel).parent.mkdir(parents = True, exist_ok = True)
+ (snap / rel).write_bytes(b"x")
+
+ monkeypatch.setattr(
+ "utils.models.model_config._iter_hf_cache_snapshots", lambda *a, **k: [snap]
+ )
+ backend = llama_cpp_module.LlamaCppBackend.__new__(llama_cpp_module.LlamaCppBackend)
+ assert backend._cached_repo_mtp_drafter("some/repo") is None
+
+
+def test_a_shared_head_pairs_with_its_target_in_the_local_scan(tmp_path):
+ """-shared marks the head's FORM, not its family.
+
+ mtp-<model>-shared-<quant>.gguf left a pairing stem of <model>-shared, which
+ never prefixes <model>-<quant>, so detect_mtp_file could not pair the head the
+ hub picker prefers: a local checkout of the files Studio had just downloaded
+ resolved differently from the download.
+ """
+ from utils.models.drafters.common import _drafter_matches_weight, _drafter_pairing_stem
+ from utils.models.model_config import detect_mtp_file
+
+ weight = "qwen3.8-flash-next-ud-iq1_s-00001-of-00003.gguf"
+ for tier in ("Q8_0", "BF16", "Q4_K_M"):
+ name = f"mtp-Qwen3.8-Flash-Next-shared-{tier}.gguf"
+ assert _drafter_pairing_stem(name, kind = "mtp") == "qwen3.8-flash-next"
+ assert _drafter_matches_weight(name, weight, kind = "mtp"), name
+ # A different family is still rejected, which is the whole point of pairing.
+ assert not _drafter_matches_weight("mtp-Some-Other-shared-Q8_0.gguf", weight, kind = "mtp")
+ # Only MTP publishes a borrowed form, so no other kind changes meaning.
+ assert _drafter_pairing_stem("dspark-Model-shared-Q8_0.gguf", kind = "dspark") == "model-shared"
+
+ root = tmp_path / "local"
+ (root / "MTP").mkdir(parents = True)
+ for i in (1, 2, 3):
+ (root / f"Qwen3.8-Flash-Next-UD-IQ1_S-0000{i}-of-00003.gguf").write_bytes(b"x" * 64)
+ shared = root / "MTP" / "mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
+ shared.write_bytes(b"x" * 64)
+ found = detect_mtp_file(
+ str(root / "Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf"), search_root = str(root)
+ )
+ assert found is not None and Path(found).name == shared.name
+
+
def test_cached_dspark_lookup_prefers_q8_and_excludes_dflash(tmp_path, monkeypatch):
import core.inference.llama_cpp as llama_cpp_module
diff --git a/studio/backend/tests/test_non_gguf_reload_settings.py b/studio/backend/tests/test_non_gguf_reload_settings.py
index 549684ae6de..67b22b182d3 100644
--- a/studio/backend/tests/test_non_gguf_reload_settings.py
+++ b/studio/backend/tests/test_non_gguf_reload_settings.py
@@ -41,6 +41,7 @@ class _Backend:
class _Request:
"""model_fields_set is what pydantic records."""
+
def __init__(self, **fields):
self.model_fields_set = set(fields)
self.force_reload = fields.pop("force_reload", False)
diff --git a/studio/backend/utils/models/drafters/common.py b/studio/backend/utils/models/drafters/common.py
index b79795e22d2..93120ca6baf 100644
--- a/studio/backend/utils/models/drafters/common.py
+++ b/studio/backend/utils/models/drafters/common.py
@@ -39,12 +39,19 @@ def _drafter_pairing_stem(name: str, *, kind: str) -> str:
stem = stem[: -(len(kind) + 1)]
from utils.models.model_config import _GGUF_KNOWN_QUANT_RE
- return re.sub(
+ stem = re.sub(
rf"-(?:{_GGUF_KNOWN_QUANT_RE.pattern})(?:-[0-9]+(?:\.[0-9]+)?bpw)?$",
"",
stem,
flags = re.IGNORECASE,
)
+ # A borrowed MTP head is published as mtp-<model>-shared-<quant>.gguf, so
+ # -shared marks the head's FORM, not the family. Left in, the stem is
+ # <model>-shared, which never prefixes <model>-<quant>, so the local scan could
+ # not pair the head the hub picker prefers. MTP only: no other kind borrows.
+ if kind == "mtp":
+ stem = re.sub(r"-shared$", "", stem)
+ return stem
def _drafter_matches_weight(candidate_name: str, weight_name: Optional[str], *, kind: str) -> bool:
diff --git a/studio/backend/utils/models/drafters/preference.py b/studio/backend/utils/models/drafters/preference.py
index 34cad169837..e11080dc69f 100644
--- a/studio/backend/utils/models/drafters/preference.py
+++ b/studio/backend/utils/models/drafters/preference.py
@@ -3,9 +3,11 @@
"""Ranking keys that decide which sidecar a load prefers.
-The download, the snapshot reuse, the offline cache lookup and the local scan
-all order candidates with these, so a repo resolves to the same file whichever
-path reaches it first.
+The download, the snapshot reuse and the offline cache lookup share them, so a
+repo resolves the same way whichever reaches it first. The local scan does not:
+detect_mtp_file ranks size-first (``_smallest_first``) because a folder holding
+several copies costs disk, while these rank speed-first because the hub path is
+choosing what to spend a download on. Same family, sometimes a different copy.
"""
from pathlib import Path
@@ -37,6 +39,38 @@ def dspark_preference_key(name: str) -> tuple[int, str]:
return dspark_precision_rank(name), Path(name).name.lower()
+def mtp_precision_rank(name: str) -> int:
+ """MTP head precision preference: Q8_0 first. Correctness does not enter into
+ it, since the target verifies every drafted token, so this is purely which
+ head drafts fastest. Q8_0 measured both quicker and more accepted than bf16 on
+ Qwen3.8-Flash-Next: a draft step is dominated by the LM head, and that head is
+ cheaper to execute at 8 bits, so bf16 is larger and slower for no gain."""
+ base = Path(name).name.lower()
+ if "-q8_0" in base:
+ return 0
+ if "-q6_k" in base:
+ return 1
+ if "-q5_k" in base:
+ return 2
+ if "-q4_k" in base or "-q4_0" in base:
+ return 3
+ if "-bf16" in base or "-f16" in base:
+ return 4
+ return 5
+
+
+def mtp_preference_key(name: str) -> tuple[int, int, str]:
+ """Sort key picking the preferred MTP head by name alone.
+
+ A head borrowing the target's token_embd/output wins the tie: 1.35 GB smaller
+ at Q8_0 and no worse, accepting identically (159 of 284) on the shipped
+ prebuilt. Only the qwen4exp path reaches a repo publishing both forms, and
+ qwen4exp MTP and the borrow ship in one fork, so the borrow always resolves.
+ """
+ borrows = 0 if "shared" in Path(name).name.lower() else 1
+ return mtp_precision_rank(name), borrows, Path(name).name.lower()
+
+
# DFlash publishes the same precision vocabulary (and the published sidecar
# carries no precision token at all, which lands in the catch-all rank), so the
# ordering is shared rather than duplicated.
diff --git a/tests/kaggle/test_t4_smoke_harness.py b/tests/kaggle/test_t4_smoke_harness.py
index 5dd87aba271..91e4a42655f 100644
--- a/tests/kaggle/test_t4_smoke_harness.py
+++ b/tests/kaggle/test_t4_smoke_harness.py
@@ -2160,19 +2160,22 @@ def test_the_files_the_payload_carries_are_byte_identical_to_the_repo(tmp_path):
payload = _payload_notebooks(_build(tmp_path))["t4_control.ipynb"]
blob = re.search(r"^FILES = (\{.*?\})$", _cell(payload, 0), re.M | re.S).group(1)
files = json.loads(blob)
- expected = {
- "versions.py",
- "canary_dataset.jsonl",
- "training_evidence.py",
- "phase_timers.py",
- "run_t4_smoke.py",
- "determinism.py",
- "gguf_export.py",
- "naive_trl_compare.py",
- "kernel_provenance.py",
- "pins/control.txt",
- "references/t4_qwen2.5-0.5b.json",
- }, sorted(files)
+ expected = (
+ {
+ "versions.py",
+ "canary_dataset.jsonl",
+ "training_evidence.py",
+ "phase_timers.py",
+ "run_t4_smoke.py",
+ "determinism.py",
+ "gguf_export.py",
+ "naive_trl_compare.py",
+ "kernel_provenance.py",
+ "pins/control.txt",
+ "references/t4_qwen2.5-0.5b.json",
+ },
+ sorted(files),
+ )
# Cross-checked against the registry as well as the literal above. The
# literal is what makes an accidental addition visible in a diff; the
# registry check is what stops the two drifting, which is how a leg ends up
diff --git a/tests/studio/test_cache_budget_discipline.py b/tests/studio/test_cache_budget_discipline.py
index 86da84ae1fb..0d94dca90f8 100644
--- a/tests/studio/test_cache_budget_discipline.py
+++ b/tests/studio/test_cache_budget_discipline.py
@@ -155,8 +155,7 @@ def _balanced(expr: str) -> bool:
_MAIN_ONLY = re.compile(r"github\.ref\s*==\s*['\"]refs/heads/main['\"]")
# A whole leaf that is nothing but the equality, in either operand order.
_LEAF_MAIN = re.compile(
- r"github\.ref\s*==\s*['\"]refs/heads/main['\"]"
- r"|['\"]refs/heads/main['\"]\s*==\s*github\.ref"
+ r"github\.ref\s*==\s*['\"]refs/heads/main['\"]|['\"]refs/heads/main['\"]\s*==\s*github\.ref"
)
@@ -521,8 +520,7 @@ def test_every_setup_python_step_still_pins_an_interpreter():
f"{name}:{jid}"
for name, jid, job in _jobs()
for step in job.get("steps") or []
- if "setup-python" in _uses(step)
- and not (step.get("with") or {}).get("python-version")
+ if "setup-python" in _uses(step) and not (step.get("with") or {}).get("python-version")
]
assert not offenders, f"setup-python without an explicit python-version: {offenders}"
@@ -729,6 +727,7 @@ def _uses(step):
"""
return str(step.get("uses", "")).casefold()
+
def _matrix_rows(job) -> list[dict]:
"""One substitution map per job the matrix can actually produce.
@@ -749,8 +748,9 @@ def _matrix_rows(job) -> list[dict]:
return [{}]
include = [r for r in (matrix.get("include") or []) if isinstance(r, dict)]
exclude = [r for r in (matrix.get("exclude") or []) if isinstance(r, dict)]
- base_keys = [k for k, v in matrix.items() if k not in ("include", "exclude")
- and isinstance(v, list)]
+ base_keys = [
+ k for k, v in matrix.items() if k not in ("include", "exclude") and isinstance(v, list)
+ ]
combos = [{}]
for k in base_keys:
@@ -759,9 +759,8 @@ def _matrix_rows(job) -> list[dict]:
# Exclude first, and it is processed before include, so include can add a
# combination back. A partial exclude drops every row agreeing on the keys it names.
def excluded(combo):
- return any(
- all(str(combo.get(k)) == str(v) for k, v in ex.items()) for ex in exclude
- )
+ return any(all(str(combo.get(k)) == str(v) for k, v in ex.items()) for ex in exclude)
+
combos = [c for c in combos if not excluded(c)]
rows, matched = [], set()
diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.py
index 9182f7c7507..c16ee80a8b3 100644
--- a/tests/studio/test_cli_attach_load_knobs.py
+++ b/tests/studio/test_cli_attach_load_knobs.py
@@ -17,6 +17,7 @@ RESIDENT = {"id": "unsloth/Qwen3-8B", "loaded": True}
class FakeServer:
"""Serves /v1/models and /api/inference/status; records every load payload."""
+
def __init__(self, models, status):
self.models = models
self.status = status
@@ -405,6 +406,7 @@ def test_omitted_default_flags_are_not_forwarded(monkeypatch):
class TestExplicitFlagsThroughTheRealCli:
"""`supplied` tracking through the real Typer and Click stack."""
+
@staticmethod
def _load_for(argv):
from typer.testing import CliRunner
@@ -1096,9 +1098,7 @@ def test_a_diffusion_resident_is_never_an_attach_target(monkeypatch):
)
def test_a_pending_retry_is_not_a_no_op(monkeypatch, capsys, field):
"""_runtime_matches_intent reloads on an identical intent while a retry is pending."""
- server = FakeServer(
- [dict(RESIDENT)], _gguf_status(**{field: True})
- ).install(monkeypatch)
+ server = FakeServer([dict(RESIDENT)], _gguf_status(**{field: True})).install(monkeypatch)
start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 8192))
diff --git a/tests/studio/test_mlx_context_platform_matrix.py b/tests/studio/test_mlx_context_platform_matrix.py
index 9d22c91327e..1638ad0d7d2 100644
--- a/tests/studio/test_mlx_context_platform_matrix.py
+++ b/tests/studio/test_mlx_context_platform_matrix.py
@@ -311,9 +311,9 @@ def test_detected_device_per_cell(os_key, vendor, spoof_cell):
if expected.chat_only_reason == "no_gpu":
assert hw.CHAT_ONLY_REASON in _CPU_ONLY_REASONS, f"{os_key}/{vendor}: chat-only reason"
else:
- assert hw.CHAT_ONLY_REASON == expected.chat_only_reason, (
- f"{os_key}/{vendor}: chat-only reason"
- )
+ assert (
+ hw.CHAT_ONLY_REASON == expected.chat_only_reason
+ ), f"{os_key}/{vendor}: chat-only reason"
# ======================================================================================
diff --git a/tests/test_rl_config_compat.py b/tests/test_rl_config_compat.py
index e6b4294e9c9..4b002365948 100644
--- a/tests/test_rl_config_compat.py
+++ b/tests/test_rl_config_compat.py
@@ -357,7 +357,11 @@ def test_setting_the_new_name_to_its_own_default_is_reported_as_ambiguous():
warmup_steps: float = 0.0
class UnslothSFTConfig(ModernSFTConfig):
- def __init__(self, warmup_steps = 0.1, **kwargs):
+ def __init__(
+ self,
+ warmup_steps = 0.1,
+ **kwargs,
+ ):
pass
messages = []
@@ -374,7 +378,9 @@ def test_setting_the_new_name_to_its_own_default_is_reported_as_ambiguous():
# No `mirrored_from` means no mirrored parameter, so no ambiguity to report.
messages = []
filter_config_init_kwargs(
- ModernSFTConfig, {"warmup_ratio": 0.03}, notify = messages.append,
+ ModernSFTConfig,
+ {"warmup_ratio": 0.03},
+ notify = messages.append,
)
assert "cannot be distinguished" not in messages[0]
diff --git a/tests/test_warnings_issued_guard.py b/tests/test_warnings_issued_guard.py
index f0b9f4614d2..9439eddb4cb 100644
--- a/tests/test_warnings_issued_guard.py
+++ b/tests/test_warnings_issued_guard.py
@@ -188,8 +188,12 @@ def _wrapped():
)
if "trl" not in ns:
pytest.skip("trl not installed")
- for name in ("classify_config_kwarg", "rename_source", "removal_source",
- "rename_value_is_unset"):
+ for name in (
+ "classify_config_kwarg",
+ "rename_source",
+ "removal_source",
+ "rename_value_is_unset",
+ ):
ns[name] = getattr(_rl_config_compat(), name)
class T(_FakeTrainer):
@@ -410,8 +414,12 @@ def _wrapped_recording(trainer_base = None):
if "trl" not in ns:
pytest.skip("trl not installed")
# The relative import in the routed function cannot work in a bare namespace.
- for name in ("classify_config_kwarg", "rename_source", "removal_source",
- "rename_value_is_unset"):
+ for name in (
+ "classify_config_kwarg",
+ "rename_source",
+ "removal_source",
+ "rename_value_is_unset",
+ ):
ns[name] = getattr(_rl_config_compat(), name)
class T(trainer_base or _RecordingTrainer):
diff --git a/unsloth/models/rl_config_compat.py b/unsloth/models/rl_config_compat.py
index f0c01367db8..6bda6756b33 100644
--- a/unsloth/models/rl_config_compat.py
+++ b/unsloth/models/rl_config_compat.py
@@ -359,7 +359,8 @@ def filter_config_init_kwargs(
f" If you also passed `{renamed}` as {existing!r}, that is its "
f"default here and cannot be distinguished from leaving it "
f"unset, so `{key}` was used; drop `{key}` to keep it."
- if ambiguous else ""
+ if ambiguous
+ else ""
)
)
else: