Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep tensor split and MTP when Auto would pick a DFlash drafter that aborts it (#13040)
Daniel Hanbc915c056349
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex cbce05ca0c2..3e6f4fa831d 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -23990,6 +23990,30 @@ class LlamaCppBackend:# The canonical mode drives which drafter is downloaded, sized and# launched, so resolve it once before either branch can use it._spec_canon = _canonicalize_spec_mode(speculative_type) or "auto"+ # #11308: DFlash2 + --split-mode tensor aborts at startup (ggml-org/llama.cpp#27819) and the+ # route falls back to layer split, so Auto prefers a loadable MTP sidecar on tensor instead.+ # _estimate_gguf_required_gb mirrors this.+ _auto_tensor_split = _spec_canon == "auto" and _effective_tensor_parallel(+ extra_args, tensor_parallel+ )++ def _auto_dflash_blocked_by_tensor() -> bool:+ if (+ _auto_tensor_split+ and mtp_draft_path+ and not _extra_args_mtp_draft_path(extra_args, env = _child_spec_env(extra_args))+ ):+ try:+ from utils.models.gguf_metadata import read_gguf_nextn_predict_layers+ return bool(+ _launch_caps(binary).get("mtp_token")+ and not (read_gguf_nextn_predict_layers(model_path) or 0) > 0+ and _mtp_drafter_loads_standalone(mtp_draft_path)+ )+ except Exception:+ return False+ return False+_unloadable_mtp_draft_path: Optional[str] = None# Scope HF_HUB_OFFLINE to the download block only when DNS is# dead; cleanup runs even on exception so a transient hiccup@@ -24101,6 +24125,7 @@ class LlamaCppBackend:if (not dflash_draft_pathand _spec_canon in ("auto", "dflash")+ and not _auto_dflash_blocked_by_tensor()and not self._dspark_wins_auto(binary = binary,dspark_draft_path = dspark_draft_path,@@ -24152,7 +24177,15 @@ class LlamaCppBackend:and not _extra_args_set_spec_type(extra_args)):try:- if _launch_caps(binary).get("supports_dflash"):+ if not _launch_caps(binary).get("supports_dflash"):+ pass+ elif _auto_dflash_blocked_by_tensor():+ logger.info(+ "Auto: DFlash sidecar available but tensor split is on; "+ "keeping tensor split with the MTP drafter "+ "(llama.cpp cannot run DFlash2 with --split-mode tensor yet)."+ )+ else:_spec_canon = "dflash"logger.info("Auto: DFlash sidecar available, using draft-dflash.")except Exception as exc:diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex d12568fbbde..7c63594a46b 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -13222,6 +13222,25 @@ def _estimate_gguf_required_gb(and not _extras_own_drafterand not _draft_pinned_to_cpu)+ # Mirrors load_model's #11308 rule: Auto on tensor split runs a loadable MTP sidecar, not DFlash.+ if _auto_dflash:+ from core.inference.llama_server_args import _effective_tensor_parallel+ from utils.models.gguf_metadata import read_gguf_nextn_predict_layers++ _tp_mtp = getattr(config, "gguf_mtp_file", None)+ _tp_main = getattr(config, "gguf_file", None)+ if (+ _tp_mtp+ and _effective_tensor_parallel(llama_extra_args, tensor_parallel)+ and Path(_tp_mtp).is_file()+ and not (_tp_main and (read_gguf_nextn_predict_layers(str(_tp_main)) or 0) > 0)+ and _mtp_drafter_loads_standalone(str(_tp_mtp))+ ):+ try:+ if LlamaCppBackend.probe_server_capabilities().get("mtp_token"):+ _auto_dflash = False+ except Exception:+ pass_dflash_capable = Trueif _forced_dflash or _auto_dflash:try:@@ -13886,6 +13905,7 @@ def _gguf_resident_file_gb(speculative_type: Optional[str] = None,disable_vision: bool = False,local_only: bool = False,+ tensor_parallel: bool = False,) -> Optional[float]:"""GB of files a launch would make resident: weights, projector, drafter.@@ -13911,7 +13931,7 @@ def _gguf_resident_file_gb()# In the key: a local-only answer served to the admission guard would drop a# remote drafter it must charge for.- key = (*key, local_only)+ key = (*key, local_only, bool(tensor_parallel))now = time.monotonic()hit = _estimate_files_cache.get(key)if hit is not None and now - hit[0] < _ESTIMATE_FILES_TTL_SECONDS:@@ -13923,6 +13943,7 @@ def _gguf_resident_file_gb(disable_vision = disable_vision,hf_token = hf_token,local_only = local_only,+ tensor_parallel = tensor_parallel,)required_gb = _estimate_gguf_required_gb(config, max_seq_length = 0, **priced)if required_gb is None:@@ -14510,6 +14531,7 @@ def _gguf_memory_breakdown(# No Hub listing behind a slider: a --spec-draft-hf repo is left uncharged here# and marked below, rather than costing a model_info per settings change.local_only = True,+ tensor_parallel = tensor_parallel,)if files_gb is None:return None@@ -14546,6 +14568,7 @@ def _gguf_memory_breakdown(speculative_type = speculative_type,disable_vision = disable_vision,local_only = True,+ tensor_parallel = tensor_parallel,)def _files_bytes_with(extra: tuple[str, ...]) -> Optional[int]:diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.pyindex cc49bf03f8b..d5cdd850cc2 100644--- a/studio/backend/tests/test_chat_load_during_training.py+++ b/studio/backend/tests/test_chat_load_during_training.py@@ -1644,6 +1644,80 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):classmethod(lambda cls, binary = None: {"supports_dflash": supported}),)+ def test_auto_prices_the_mtp_sidecar_under_tensor_split(self):+ """#11308: Auto on tensor split launches the loadable MTP sidecar, so it is the one charged."""+ import tempfile++ import core.inference.llama_cpp as llama_cpp_module++ with tempfile.TemporaryDirectory() as d:+ p = Path(d)+ target = p / "model.gguf"+ sidecar = p / "dflash-kquant.gguf"+ mtp = p / "MTP" / "mtp-model.gguf"+ mtp.parent.mkdir()+ target.write_bytes(b"x" * 2000)+ sidecar.write_bytes(b"y" * 3000)+ mtp.write_bytes(b"z" * 5000)+ cfg = _gguf_cfg(+ gguf_file = str(target),+ gguf_dflash_file = str(sidecar),+ gguf_mtp_file = str(mtp),+ gguf_hf_repo = None,+ gguf_variant = None,+ )++ from core.inference.llama_cpp import LlamaCppBackend++ def _caps(mtp_token = "draft-mtp"):+ return patch.object(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(+ lambda cls, binary = None: {"supports_dflash": True, "mtp_token": mtp_token}+ ),+ )++ def _estimate(+ loads,+ mtp_token = "draft-mtp",+ **kw,+ ):+ with (+ patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),+ patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),+ patch.object(+ llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _p: loads+ ),+ _caps(mtp_token),+ ):+ return self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto", **kw)++ def _resident(**kw):+ with (+ patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),+ patch.object(+ llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _p: True+ ),+ _caps(),+ ):+ return self.route._gguf_resident_file_gb(cfg, speculative_type = "auto", **kw)++ resident_tensor = _resident(tensor_parallel = True)+ resident_layer = _resident()+ tensor = _estimate(True, tensor_parallel = True)+ tensor_extras = _estimate(True, llama_extra_args = ["-sm", "tensor"])+ layer = _estimate(True)+ unloadable = _estimate(False, tensor_parallel = True)+ no_mtp_binary = _estimate(True, mtp_token = None, tensor_parallel = True)+ self.assertAlmostEqual(tensor, 7000 / (1024**3), places = 9)+ self.assertAlmostEqual(tensor_extras, 7000 / (1024**3), places = 9)+ self.assertAlmostEqual(layer, 5000 / (1024**3), places = 9)+ self.assertAlmostEqual(unloadable, 5000 / (1024**3), places = 9)+ self.assertAlmostEqual(no_mtp_binary, 5000 / (1024**3), places = 9)+ self.assertAlmostEqual(resident_tensor, 7000 / (1024**3), places = 9)+ self.assertAlmostEqual(resident_layer, 5000 / (1024**3), places = 9)+def test_extra_args_drafter_is_charged_once_when_it_is_the_local_sidecar(self):"""--model-draft usually names the very sidecar discovery already found,and charging it on both paths billed a 1.5 GiB drafter as 3 GiB, so thediff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.pyindex fdc2c2fe92b..3662454d520 100644--- a/studio/backend/tests/test_mtp_drafter_companion.py+++ b/studio/backend/tests/test_mtp_drafter_companion.py@@ -3083,7 +3083,21 @@ class _StopAfterDownloads(Exception):"""Ends the load once Phase 2 is done, which is all these tests observe."""-def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dflash, dspark_cached):+def _dflash_fetch_during_auto_load(+ monkeypatch,+ *,+ supports_dspark,+ supports_dflash,+ dspark_cached,+ speculative_type = "auto",+ tensor_parallel = False,+ extra_args = None,+ gguf_path = None,+ dflash_draft_path = None,+ mtp_draft_path = None,+ mtp_loads = True,+ mtp_token = "draft-mtp",+):"""Whether an Auto load fetches the DFlash sidecar, and what it resolves to.Drives the real load path: the suppression lives inline in load_model's@@ -3102,6 +3116,7 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dfl"found": True,"supports_dspark": supports_dspark,"supports_dflash": supports_dflash,+ "mtp_token": mtp_token,}),)@@ -3122,7 +3137,8 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dflmonkeypatch.setattr(backend, "_download_gguf", lambda **_kwargs: "/cache/snap/model-Q4_K_M.gguf")- monkeypatch.setattr(backend, "_download_mtp", lambda **_kwargs: None)+ monkeypatch.setattr(backend, "_download_mtp", lambda **_kwargs: mtp_draft_path)+ monkeypatch.setattr(llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _path: mtp_loads)# Exactly what _download_dspark does for a cached sidecar on a binary that# cannot run it: the path comes back regardless of the capability.monkeypatch.setattr(backend, "_download_dspark", lambda **_kwargs: dspark_cached)@@ -3132,6 +3148,14 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dflreturn "/cache/snap/dflash-kquant.gguf"monkeypatch.setattr(backend, "_download_dflash", _fetch_dflash)+ seen["log"] = []+ _real_info = llama_cpp_module.logger.info++ def _info(msg, *args, **kwargs):+ seen["log"].append(str(msg))+ return _real_info(msg, *args, **kwargs)++ monkeypatch.setattr(llama_cpp_module.logger, "info", _info)def _stop(*_args, **_kwargs):raise _StopAfterDownloads@@ -3140,15 +3164,23 @@ def _dflash_fetch_during_auto_load(monkeypatch, *, supports_dspark, supports_dfl# settled by then.monkeypatch.setattr(backend, "_read_gguf_metadata", _stop)+ source = (+ {"gguf_path": gguf_path, "model_identifier": gguf_path}+ if gguf_path+ else {"hf_repo": "org/repo", "hf_variant": "Q4_K_M", "model_identifier": "org/repo"}+ )with pytest.raises(_StopAfterDownloads):backend.load_model(GgufLoadIntent(- hf_repo = "org/repo",- hf_variant = "Q4_K_M",- model_identifier = "org/repo",- speculative_type = "auto",+ **source,+ speculative_type = speculative_type,+ tensor_parallel = tensor_parallel,+ extra_args = tuple(extra_args) if extra_args is not None else None,+ dflash_draft_path = dflash_draft_path,+ mtp_draft_path = mtp_draft_path if gguf_path else None,))+ seen["dflash_promoted"] = any("using draft-dflash" in m for m in seen["log"])return seen@@ -3188,6 +3220,160 @@ def test_auto_fetches_dflash_when_the_repo_ships_no_dspark_sidecar(monkeypatch):assert seen["dflash_fetched"] is True+# #11308: DFlash2 + --split-mode tensor aborts at startup, so Auto keeps tensor split with MTP.++_MTP = "/cache/snap/MTP/mtp-model-Q4_0.gguf"+++ "tensor_parallel, extra_args",+ [(True, None), (False, ["-sm", "tensor"]), (False, ["--split-mode", "tensor"])],+)+def test_auto_skips_dflash_under_tensor_split_when_mtp_is_available(+ monkeypatch, tensor_parallel, extra_args+):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ tensor_parallel = tensor_parallel,+ extra_args = extra_args,+ mtp_draft_path = _MTP,+ )+ assert seen["dflash_fetched"] is False+ assert seen["dflash_promoted"] is False+++def test_auto_skips_dflash_under_tensor_split_from_env(monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_SPLIT_MODE", "tensor")+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ mtp_draft_path = _MTP,+ )+ assert seen["dflash_fetched"] is False+++def test_auto_keeps_dflash_under_tensor_split_when_the_mtp_sidecar_cannot_load(monkeypatch):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ tensor_parallel = True,+ mtp_draft_path = _MTP,+ mtp_loads = False,+ )+ assert seen["dflash_fetched"] is True+ assert seen["dflash_promoted"] is True+++def test_auto_keeps_dflash_under_tensor_split_when_the_binary_cannot_run_mtp(monkeypatch):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ tensor_parallel = True,+ mtp_draft_path = _MTP,+ mtp_token = None,+ )+ assert seen["dflash_promoted"] is True+++def test_auto_keeps_dflash_under_tensor_split_for_an_extras_drafter(monkeypatch):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ extra_args = ["-sm", "tensor", "--model-draft", "/models/dflash-model.gguf"],+ mtp_draft_path = _MTP,+ )+ assert seen["dflash_promoted"] is True+++def test_auto_keeps_dflash_under_tensor_split_for_a_target_with_an_embedded_head(monkeypatch):+ import utils.models.gguf_metadata as gguf_metadata++ monkeypatch.setattr(gguf_metadata, "read_gguf_nextn_predict_layers", lambda _path: 1)+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ tensor_parallel = True,+ mtp_draft_path = _MTP,+ )+ assert seen["dflash_promoted"] is True+++def test_auto_keeps_dflash_under_tensor_split_without_an_mtp_drafter(monkeypatch):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ tensor_parallel = True,+ )+ assert seen["dflash_fetched"] is True+ assert seen["dflash_promoted"] is True+++def test_auto_does_not_promote_a_local_dflash_sidecar_under_tensor_split(monkeypatch, tmp_path):+ model = tmp_path / "Qwen3.8-27B-UD-Q6_K_XL.gguf"+ model.write_bytes(b"GGUF")+ sidecar = tmp_path / "dflash-Qwen3.8-27B-Q8_0.gguf"+ sidecar.write_bytes(b"GGUF")+ mtp = tmp_path / "MTP" / "mtp-Qwen3.8-27B-Q4_0.gguf"+ mtp.parent.mkdir()+ mtp.write_bytes(b"GGUF")+ kwargs = dict(+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ gguf_path = str(model),+ dflash_draft_path = str(sidecar),+ mtp_draft_path = str(mtp),+ )+ tensor = _dflash_fetch_during_auto_load(monkeypatch, extra_args = ["-sm", "tensor"], **kwargs)+ assert tensor["dflash_promoted"] is False+ assert any("keeping tensor split" in m for m in tensor["log"])+ layer = _dflash_fetch_during_auto_load(monkeypatch, **kwargs)+ assert layer["dflash_promoted"] is True++[email protected]("extra_args", [None, ["-sm", "layer"]])+def test_auto_still_uses_dflash_without_tensor_split(monkeypatch, extra_args):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ tensor_parallel = extra_args is not None,+ extra_args = extra_args,+ mtp_draft_path = _MTP,+ )+ assert seen["dflash_fetched"] is True+ assert seen["dflash_promoted"] is True+++def test_explicit_dflash_still_fetches_under_tensor_split(monkeypatch):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ speculative_type = "dflash",+ tensor_parallel = True,+ mtp_draft_path = _MTP,+ )+ assert seen["dflash_fetched"] is True++# ── The caller's boundary reaches discovery, not just the rescan ──────## ModelConfig.from_identifier runs the local companion scan, and the DFlash scan