Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep tensor split and MTP when Auto would pick a DFlash drafter that aborts it (#13040)

Daniel Hanbc915c056349
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index cbce05ca0c2..3e6f4fa831d 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -23990,6 +23990,30 @@ class LlamaCppBackend:
# The canonical mode drives which drafter is downloaded, sized and
# launched, so resolve it once before either branch can use it.
_spec_canon = _canonicalize_spec_mode(speculative_type) or "auto"
+ # #11308: DFlash2 + --split-mode tensor aborts at startup (ggml-org/llama.cpp#27819) and the
+ # route falls back to layer split, so Auto prefers a loadable MTP sidecar on tensor instead.
+ # _estimate_gguf_required_gb mirrors this.
+ _auto_tensor_split = _spec_canon == "auto" and _effective_tensor_parallel(
+ extra_args, tensor_parallel
+ )
+
+ def _auto_dflash_blocked_by_tensor() -> bool:
+ if (
+ _auto_tensor_split
+ and mtp_draft_path
+ and not _extra_args_mtp_draft_path(extra_args, env = _child_spec_env(extra_args))
+ ):
+ try:
+ from utils.models.gguf_metadata import read_gguf_nextn_predict_layers
+ return bool(
+ _launch_caps(binary).get("mtp_token")
+ and not (read_gguf_nextn_predict_layers(model_path) or 0) > 0
+ and _mtp_drafter_loads_standalone(mtp_draft_path)
+ )
+ except Exception:
+ return False
+ return False
+
_unloadable_mtp_draft_path: Optional[str] = None
# Scope HF_HUB_OFFLINE to the download block only when DNS is
# dead; cleanup runs even on exception so a transient hiccup
@@ -24101,6 +24125,7 @@ class LlamaCppBackend:
if (
not dflash_draft_path
and _spec_canon in ("auto", "dflash")
+ and not _auto_dflash_blocked_by_tensor()
and not self._dspark_wins_auto(
binary = binary,
dspark_draft_path = dspark_draft_path,
@@ -24152,7 +24177,15 @@ class LlamaCppBackend:
and not _extra_args_set_spec_type(extra_args)
):
try:
- if _launch_caps(binary).get("supports_dflash"):
+ if not _launch_caps(binary).get("supports_dflash"):
+ pass
+ elif _auto_dflash_blocked_by_tensor():
+ logger.info(
+ "Auto: DFlash sidecar available but tensor split is on; "
+ "keeping tensor split with the MTP drafter "
+ "(llama.cpp cannot run DFlash2 with --split-mode tensor yet)."
+ )
+ else:
_spec_canon = "dflash"
logger.info("Auto: DFlash sidecar available, using draft-dflash.")
except Exception as exc:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d12568fbbde..7c63594a46b 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -13222,6 +13222,25 @@ def _estimate_gguf_required_gb(
and not _extras_own_drafter
and not _draft_pinned_to_cpu
)
+ # Mirrors load_model's #11308 rule: Auto on tensor split runs a loadable MTP sidecar, not DFlash.
+ if _auto_dflash:
+ from core.inference.llama_server_args import _effective_tensor_parallel
+ from utils.models.gguf_metadata import read_gguf_nextn_predict_layers
+
+ _tp_mtp = getattr(config, "gguf_mtp_file", None)
+ _tp_main = getattr(config, "gguf_file", None)
+ if (
+ _tp_mtp
+ and _effective_tensor_parallel(llama_extra_args, tensor_parallel)
+ and Path(_tp_mtp).is_file()
+ and not (_tp_main and (read_gguf_nextn_predict_layers(str(_tp_main)) or 0) > 0)
+ and _mtp_drafter_loads_standalone(str(_tp_mtp))
+ ):
+ try:
+ if LlamaCppBackend.probe_server_capabilities().get("mtp_token"):
+ _auto_dflash = False
+ except Exception:
+ pass
_dflash_capable = True
if _forced_dflash or _auto_dflash:
try:
@@ -13886,6 +13905,7 @@ def _gguf_resident_file_gb(
speculative_type: Optional[str] = None,
disable_vision: bool = False,
local_only: bool = False,
+ tensor_parallel: bool = False,
) -> Optional[float]:
"""GB of files a launch would make resident: weights, projector, drafter.
@@ -13911,7 +13931,7 @@ def _gguf_resident_file_gb(
)
# In the key: a local-only answer served to the admission guard would drop a
# remote drafter it must charge for.
- key = (*key, local_only)
+ key = (*key, local_only, bool(tensor_parallel))
now = time.monotonic()
hit = _estimate_files_cache.get(key)
if hit is not None and now - hit[0] < _ESTIMATE_FILES_TTL_SECONDS:
@@ -13923,6 +13943,7 @@ def _gguf_resident_file_gb(
disable_vision = disable_vision,
hf_token = hf_token,
local_only = local_only,
+ tensor_parallel = tensor_parallel,
)
required_gb = _estimate_gguf_required_gb(config, max_seq_length = 0, **priced)
if required_gb is None:
@@ -14510,6 +14531,7 @@ def _gguf_memory_breakdown(
# No Hub listing behind a slider: a --spec-draft-hf repo is left uncharged here
# and marked below, rather than costing a model_info per settings change.
local_only = True,
+ tensor_parallel = tensor_parallel,
)
if files_gb is None:
return None
@@ -14546,6 +14568,7 @@ def _gguf_memory_breakdown(
speculative_type = speculative_type,
disable_vision = disable_vision,
local_only = True,
+ tensor_parallel = tensor_parallel,
)
def _files_bytes_with(extra: tuple[str, ...]) -> Optional[int]:
diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.py
index cc49bf03f8b..d5cdd850cc2 100644
--- a/studio/backend/tests/test_chat_load_during_training.py
+++ b/studio/backend/tests/test_chat_load_during_training.py
@@ -1644,6 +1644,80 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):
classmethod(lambda cls, binary = None: {"supports_dflash": supported}),
)
+ def test_auto_prices_the_mtp_sidecar_under_tensor_split(self):
+ """#11308: Auto on tensor split launches the loadable MTP sidecar, so it is the one charged."""
+ import tempfile
+
+ import core.inference.llama_cpp as llama_cpp_module
+
+ with tempfile.TemporaryDirectory() as d:
+ p = Path(d)
+ target = p / "model.gguf"
+ sidecar = p / "dflash-kquant.gguf"
+ mtp = p / "MTP" / "mtp-model.gguf"
+ mtp.parent.mkdir()
+ target.write_bytes(b"x" * 2000)
+ sidecar.write_bytes(b"y" * 3000)
+ mtp.write_bytes(b"z" * 5000)
+ cfg = _gguf_cfg(
+ gguf_file = str(target),
+ gguf_dflash_file = str(sidecar),
+ gguf_mtp_file = str(mtp),
+ gguf_hf_repo = None,
+ gguf_variant = None,
+ )
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ def _caps(mtp_token = "draft-mtp"):
+ return patch.object(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(
+ lambda cls, binary = None: {"supports_dflash": True, "mtp_token": mtp_token}
+ ),
+ )
+
+ def _estimate(
+ loads,
+ mtp_token = "draft-mtp",
+ **kw,
+ ):
+ with (
+ patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
+ patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
+ patch.object(
+ llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _p: loads
+ ),
+ _caps(mtp_token),
+ ):
+ return self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto", **kw)
+
+ def _resident(**kw):
+ with (
+ patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
+ patch.object(
+ llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _p: True
+ ),
+ _caps(),
+ ):
+ return self.route._gguf_resident_file_gb(cfg, speculative_type = "auto", **kw)
+
+ resident_tensor = _resident(tensor_parallel = True)
+ resident_layer = _resident()
+ tensor = _estimate(True, tensor_parallel = True)
+ tensor_extras = _estimate(True, llama_extra_args = ["-sm", "tensor"])
+ layer = _estimate(True)
+ unloadable = _estimate(False, tensor_parallel = True)
+ no_mtp_binary = _estimate(True, mtp_token = None, tensor_parallel = True)
+ self.assertAlmostEqual(tensor, 7000 / (1024**3), places = 9)
+ self.assertAlmostEqual(tensor_extras, 7000 / (1024**3), places = 9)
+ self.assertAlmostEqual(layer, 5000 / (1024**3), places = 9)
+ self.assertAlmostEqual(unloadable, 5000 / (1024**3), places = 9)
+ self.assertAlmostEqual(no_mtp_binary, 5000 / (1024**3), places = 9)
+ self.assertAlmostEqual(resident_tensor, 7000 / (1024**3), places = 9)
+ self.assertAlmostEqual(resident_layer, 5000 / (1024**3), places = 9)
+
def test_extra_args_drafter_is_charged_once_when_it_is_the_local_sidecar(self):
"""--model-draft usually names the very sidecar discovery already found,
and charging it on both paths billed a 1.5 GiB drafter as 3 GiB, so the
diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py
index fdc2c2fe92b..3662454d520 100644
--- a/studio/backend/tests/test_mtp_drafter_companion.py
+++ b/studio/backend/tests/test_mtp_drafter_companion.py
@@ -3083,7 +3083,21 @@ class _StopAfterDownloads(Exception):
"""Ends the load once Phase 2 is done, which is all these tests observe."""
-def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dflash, dspark_cached):
+def _dflash_fetch_during_auto_load(
+ monkeypatch,
+ *,
+ supports_dspark,
+ supports_dflash,
+ dspark_cached,
+ speculative_type = "auto",
+ tensor_parallel = False,
+ extra_args = None,
+ gguf_path = None,
+ dflash_draft_path = None,
+ mtp_draft_path = None,
+ mtp_loads = True,
+ mtp_token = "draft-mtp",
+):
"""Whether an Auto load fetches the DFlash sidecar, and what it resolves to.
Drives the real load path: the suppression lives inline in load_model's
@@ -3102,6 +3116,7 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dfl
"found": True,
"supports_dspark": supports_dspark,
"supports_dflash": supports_dflash,
+ "mtp_token": mtp_token,
}
),
)
@@ -3122,7 +3137,8 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dfl
monkeypatch.setattr(
backend, "_download_gguf", lambda **_kwargs: "/cache/snap/model-Q4_K_M.gguf"
)
- monkeypatch.setattr(backend, "_download_mtp", lambda **_kwargs: None)
+ monkeypatch.setattr(backend, "_download_mtp", lambda **_kwargs: mtp_draft_path)
+ monkeypatch.setattr(llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _path: mtp_loads)
# Exactly what _download_dspark does for a cached sidecar on a binary that
# cannot run it: the path comes back regardless of the capability.
monkeypatch.setattr(backend, "_download_dspark", lambda **_kwargs: dspark_cached)
@@ -3132,6 +3148,14 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dfl
return "/cache/snap/dflash-kquant.gguf"
monkeypatch.setattr(backend, "_download_dflash", _fetch_dflash)
+ seen["log"] = []
+ _real_info = llama_cpp_module.logger.info
+
+ def _info(msg, *args, **kwargs):
+ seen["log"].append(str(msg))
+ return _real_info(msg, *args, **kwargs)
+
+ monkeypatch.setattr(llama_cpp_module.logger, "info", _info)
def _stop(*_args, **_kwargs):
raise _StopAfterDownloads
@@ -3140,15 +3164,23 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dfl
# settled by then.
monkeypatch.setattr(backend, "_read_gguf_metadata", _stop)
+ source = (
+ {"gguf_path": gguf_path, "model_identifier": gguf_path}
+ if gguf_path
+ else {"hf_repo": "org/repo", "hf_variant": "Q4_K_M", "model_identifier": "org/repo"}
+ )
with pytest.raises(_StopAfterDownloads):
backend.load_model(
GgufLoadIntent(
- hf_repo = "org/repo",
- hf_variant = "Q4_K_M",
- model_identifier = "org/repo",
- speculative_type = "auto",
+ **source,
+ speculative_type = speculative_type,
+ tensor_parallel = tensor_parallel,
+ extra_args = tuple(extra_args) if extra_args is not None else None,
+ dflash_draft_path = dflash_draft_path,
+ mtp_draft_path = mtp_draft_path if gguf_path else None,
)
)
+ seen["dflash_promoted"] = any("using draft-dflash" in m for m in seen["log"])
return seen
@@ -3188,6 +3220,160 @@ def test_auto_fetches_dflash_when_the_repo_ships_no_dspark_sidecar(monkeypatch):
assert seen["dflash_fetched"] is True
+# #11308: DFlash2 + --split-mode tensor aborts at startup, so Auto keeps tensor split with MTP.
+
+_MTP = "/cache/snap/MTP/mtp-model-Q4_0.gguf"
+
+
+ "tensor_parallel, extra_args",
+ [(True, None), (False, ["-sm", "tensor"]), (False, ["--split-mode", "tensor"])],
+)
+def test_auto_skips_dflash_under_tensor_split_when_mtp_is_available(
+ monkeypatch, tensor_parallel, extra_args
+):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ tensor_parallel = tensor_parallel,
+ extra_args = extra_args,
+ mtp_draft_path = _MTP,
+ )
+ assert seen["dflash_fetched"] is False
+ assert seen["dflash_promoted"] is False
+
+
+def test_auto_skips_dflash_under_tensor_split_from_env(monkeypatch):
+ monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "tensor")
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ mtp_draft_path = _MTP,
+ )
+ assert seen["dflash_fetched"] is False
+
+
+def test_auto_keeps_dflash_under_tensor_split_when_the_mtp_sidecar_cannot_load(monkeypatch):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ tensor_parallel = True,
+ mtp_draft_path = _MTP,
+ mtp_loads = False,
+ )
+ assert seen["dflash_fetched"] is True
+ assert seen["dflash_promoted"] is True
+
+
+def test_auto_keeps_dflash_under_tensor_split_when_the_binary_cannot_run_mtp(monkeypatch):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ tensor_parallel = True,
+ mtp_draft_path = _MTP,
+ mtp_token = None,
+ )
+ assert seen["dflash_promoted"] is True
+
+
+def test_auto_keeps_dflash_under_tensor_split_for_an_extras_drafter(monkeypatch):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ extra_args = ["-sm", "tensor", "--model-draft", "/models/dflash-model.gguf"],
+ mtp_draft_path = _MTP,
+ )
+ assert seen["dflash_promoted"] is True
+
+
+def test_auto_keeps_dflash_under_tensor_split_for_a_target_with_an_embedded_head(monkeypatch):
+ import utils.models.gguf_metadata as gguf_metadata
+
+ monkeypatch.setattr(gguf_metadata, "read_gguf_nextn_predict_layers", lambda _path: 1)
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ tensor_parallel = True,
+ mtp_draft_path = _MTP,
+ )
+ assert seen["dflash_promoted"] is True
+
+
+def test_auto_keeps_dflash_under_tensor_split_without_an_mtp_drafter(monkeypatch):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ tensor_parallel = True,
+ )
+ assert seen["dflash_fetched"] is True
+ assert seen["dflash_promoted"] is True
+
+
+def test_auto_does_not_promote_a_local_dflash_sidecar_under_tensor_split(monkeypatch, tmp_path):
+ model = tmp_path / "Qwen3.8-27B-UD-Q6_K_XL.gguf"
+ model.write_bytes(b"GGUF")
+ sidecar = tmp_path / "dflash-Qwen3.8-27B-Q8_0.gguf"
+ sidecar.write_bytes(b"GGUF")
+ mtp = tmp_path / "MTP" / "mtp-Qwen3.8-27B-Q4_0.gguf"
+ mtp.parent.mkdir()
+ mtp.write_bytes(b"GGUF")
+ kwargs = dict(
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ gguf_path = str(model),
+ dflash_draft_path = str(sidecar),
+ mtp_draft_path = str(mtp),
+ )
+ tensor = _dflash_fetch_during_auto_load(monkeypatch, extra_args = ["-sm", "tensor"], **kwargs)
+ assert tensor["dflash_promoted"] is False
+ assert any("keeping tensor split" in m for m in tensor["log"])
+ layer = _dflash_fetch_during_auto_load(monkeypatch, **kwargs)
+ assert layer["dflash_promoted"] is True
+
+
[email protected]("extra_args", [None, ["-sm", "layer"]])
+def test_auto_still_uses_dflash_without_tensor_split(monkeypatch, extra_args):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ tensor_parallel = extra_args is not None,
+ extra_args = extra_args,
+ mtp_draft_path = _MTP,
+ )
+ assert seen["dflash_fetched"] is True
+ assert seen["dflash_promoted"] is True
+
+
+def test_explicit_dflash_still_fetches_under_tensor_split(monkeypatch):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ speculative_type = "dflash",
+ tensor_parallel = True,
+ mtp_draft_path = _MTP,
+ )
+ assert seen["dflash_fetched"] is True
+
+
# ── The caller's boundary reaches discovery, not just the rescan ──────
#
# ModelConfig.from_identifier runs the local companion scan, and the DFlash scan