Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: skip a DFlash drafter that an image would break under Auto (#13553)
Daniel Han579809c46ccd
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 801c51de6e9..5688b71a094 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -5565,6 +5565,25 @@ def _extra_args_requests_dspark(return "draft-dspark" in _accumulated_spec_types(extra_args, env)+def _dflash_drafter_breaks_on_images(+ model_path: Optional[str], dflash_path: Optional[str], projector: Optional[str]+) -> bool:+ """#13511: llama.cpp skips an M-RoPE image in the DFlash drafter (its rows share one+ position) and a NeoX drafter then rejects the jump, so the first image sent after any+ text fails the slot. Unreadable headers fail open."""+ if not (model_path and dflash_path and projector):+ return False+ try:+ from utils.models.gguf_metadata import gguf_has_mrope_sections, mmproj_accepts_image+ return bool(+ mmproj_accepts_image(projector)+ and gguf_has_mrope_sections(model_path) is True+ and gguf_has_mrope_sections(dflash_path) is False+ )+ except Exception:+ return False++def _extra_args_requests_dflash(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> bool:@@ -25160,6 +25179,17 @@ class LlamaCppBackend:return Falsereturn False+ def _dflash_breaks_on_images() -> bool:+ if not dflash_draft_path or disable_vision or extra_args_disable_mmproj(extra_args):+ return False+ try:+ projector = self._resolve_launch_mmproj_path(+ model_path = model_path, mmproj_path = mmproj_path, extra_args = extra_args+ )+ except Exception:+ return False+ return _dflash_drafter_breaks_on_images(model_path, dflash_draft_path, projector)+_unloadable_mtp_draft_path: Optional[str] = None# Scope HF_HUB_OFFLINE to the download block only when DNS is# dead; cleanup runs even on exception so a transient hiccup@@ -25331,11 +25361,30 @@ class LlamaCppBackend:"keeping tensor split with the MTP drafter ""(llama.cpp cannot run DFlash2 with --split-mode tensor yet).")+ elif _dflash_breaks_on_images():+ logger.info(+ "Auto: DFlash sidecar %s has no M-RoPE sections, so llama.cpp "+ "cannot carry it past an image on this vision model; keeping "+ "vision and skipping DFlash. Choose DFlash in the Speculative "+ "Decoding dropdown to force it, or turn Vision off.",+ dflash_draft_path,+ )else:_spec_canon = "dflash"logger.info("Auto: DFlash sidecar available, using draft-dflash.")except Exception as exc:logger.debug("DFlash capability probe failed during Auto: %s", exc)+ elif (+ _spec_canon == "dflash"+ and not _extra_args_set_spec_type(extra_args)+ and _dflash_breaks_on_images()+ ):+ logger.warning(+ "DFlash forced with sidecar %s, which has no M-RoPE sections: llama.cpp "+ "will fail the first image sent after text in a chat. Use Auto or turn "+ "Vision off to keep images working.",+ dflash_draft_path,+ )# MTP and DSpark are mutually exclusive, and the drafter sizing and# lifecycle path below is architecture agnostic, so it carries thediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 92ecffd3aab..9587539c3e4 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -13252,6 +13252,7 @@ def _estimate_gguf_required_gb(try:from core.inference.llama_cpp import (_canonicalize_spec_mode,+ _dflash_drafter_breaks_on_images,_extra_args_device,_extra_args_draft_offloaded_to_cpu,_extra_args_mtp_draft_path,@@ -13346,6 +13347,17 @@ def _estimate_gguf_required_gb(_auto_dflash = Falseexcept Exception:pass+ # Mirrors load_model's #13511 rule: Auto skips a NeoX DFlash drafter on an M-RoPE vision load.+ if _auto_dflash and not disable_vision and not extra_args_disable_mmproj(llama_extra_args):+ _vd_mmproj = _extra_args_device(llama_extra_args, {"--mmproj", "-mm"}) or getattr(+ config, "gguf_mmproj_file", None+ )+ if _vd_mmproj and Path(_vd_mmproj).is_file():+ _auto_dflash = not _dflash_drafter_breaks_on_images(+ getattr(config, "gguf_file", None),+ getattr(config, "gguf_dflash_file", None),+ str(_vd_mmproj),+ )_dflash_capable = Trueif _forced_dflash or _auto_dflash:try:diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.pyindex b84da076ca7..6e3c73e4007 100644--- a/studio/backend/tests/test_chat_load_during_training.py+++ b/studio/backend/tests/test_chat_load_during_training.py@@ -1772,6 +1772,73 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):self.assertAlmostEqual(resident_tensor, 7000 / (1024**3), places = 9)self.assertAlmostEqual(resident_layer, 5000 / (1024**3), places = 9)+ def test_auto_drops_a_neox_dflash_charge_on_an_mrope_vision_load(self):+ """#13511: Auto launches no DFlash drafter that an image would break, so none is charged."""+ import struct+ import tempfile++ def _header(path, arch, kv, size):+ entries = [("general.architecture", arch), *kv.items()]+ out = [struct.pack("<IIQQ", 0x46554747, 3, 0, len(entries))]+ for key, val in entries:+ out.append(struct.pack("<Q", len(key)) + key.encode())+ if isinstance(val, str):+ out.append(struct.pack("<IQ", 8, len(val)) + val.encode())+ elif isinstance(val, bool):+ out.append(struct.pack("<I?", 7, val))+ else:+ out.append(+ struct.pack("<IIQ", 9, 5, len(val)) + struct.pack(f"<{len(val)}i", *val)+ )+ blob = b"".join(out)+ path.write_bytes(blob + b"\0" * (size - len(blob)))+ return str(path)++ sections = [11, 11, 10, 0]+ with tempfile.TemporaryDirectory() as d:+ p = Path(d)+ target = _header(+ p / "model.gguf", "qwen35", {"qwen35.rope.dimension_sections": sections}, 2000+ )+ neox = _header(p / "dflash-neox.gguf", "dflash", {}, 3000)+ mrope = _header(+ p / "dflash-mrope.gguf",+ "dflash",+ {"dflash.rope.dimension_sections": sections},+ 3000,+ )+ mmproj = _header(p / "mmproj-F16.gguf", "clip", {"clip.has_vision_encoder": True}, 7000)++ from core.inference.llama_cpp import LlamaCppBackend++ def _estimate(dflash, **kw):+ cfg = _gguf_cfg(+ gguf_file = target,+ gguf_dflash_file = dflash,+ gguf_mmproj_file = mmproj,+ gguf_hf_repo = None,+ gguf_variant = None,+ )+ with (+ patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),+ patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),+ patch.object(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(lambda cls, binary = None: {"supports_dflash": True}),+ ),+ ):+ return self.route._estimate_gguf_required_gb(cfg, **kw)++ auto_neox = _estimate(neox, speculative_type = "auto")+ auto_mrope = _estimate(mrope, speculative_type = "auto")+ auto_neox_vision_off = _estimate(neox, speculative_type = "auto", disable_vision = True)+ forced_neox = _estimate(neox, speculative_type = "dflash")+ self.assertAlmostEqual(auto_neox, 9000 / (1024**3), places = 9)+ self.assertAlmostEqual(auto_mrope, 12000 / (1024**3), places = 9)+ self.assertAlmostEqual(auto_neox_vision_off, 5000 / (1024**3), places = 9)+ self.assertAlmostEqual(forced_neox, 12000 / (1024**3), places = 9)+def test_extra_args_drafter_is_charged_once_when_it_is_the_local_sidecar(self):"""--model-draft usually names the very sidecar discovery already found,and charging it on both paths billed a 1.5 GiB drafter as 3 GiB, so thediff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.pyindex b1ec5a23eee..cb5a51673ec 100644--- a/studio/backend/tests/test_mtp_drafter_companion.py+++ b/studio/backend/tests/test_mtp_drafter_companion.py@@ -3131,6 +3131,10 @@ def _dflash_fetch_during_auto_load(mtp_draft_path = None,mtp_loads = True,mtp_token = "draft-mtp",+ model_file = "/cache/snap/model-Q4_K_M.gguf",+ dflash_file = "/cache/snap/dflash-kquant.gguf",+ mmproj_file = None,+ disable_vision = False,):"""Whether an Auto load fetches the DFlash sidecar, and what it resolves to.@@ -3168,9 +3172,8 @@ def _dflash_fetch_during_auto_load(monkeypatch.setattr(backend, "_get_gpu_memory", lambda _binary = None, **_kw: [(0, 4096, 8192)])monkeypatch.setattr(backend, "_gguf_path_is_diffusion", lambda *_args: False)monkeypatch.setattr(backend, "_kill_process", lambda: None)- monkeypatch.setattr(- backend, "_download_gguf", lambda **_kwargs: "/cache/snap/model-Q4_K_M.gguf"- )+ monkeypatch.setattr(backend, "_download_gguf", lambda **_kwargs: model_file)+ monkeypatch.setattr(backend, "_resolve_launch_mmproj_path", lambda **_kwargs: mmproj_file)monkeypatch.setattr(backend, "_download_mtp", lambda **_kwargs: mtp_draft_path)monkeypatch.setattr(llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _path: mtp_loads)# Exactly what _download_dspark does for a cached sidecar on a binary that@@ -3179,7 +3182,7 @@ def _dflash_fetch_during_auto_load(def _fetch_dflash(**_kwargs):seen["dflash_fetched"] = True- return "/cache/snap/dflash-kquant.gguf"+ return dflash_filemonkeypatch.setattr(backend, "_download_dflash", _fetch_dflash)seen["log"] = []@@ -3190,6 +3193,14 @@ def _dflash_fetch_during_auto_load(return _real_info(msg, *args, **kwargs)monkeypatch.setattr(llama_cpp_module.logger, "info", _info)+ seen["warnings"] = []+ _real_warning = llama_cpp_module.logger.warning++ def _warning(msg, *args, **kwargs):+ seen["warnings"].append(str(msg))+ return _real_warning(msg, *args, **kwargs)++ monkeypatch.setattr(llama_cpp_module.logger, "warning", _warning)def _stop(*_args, **_kwargs):raise _StopAfterDownloads@@ -3212,6 +3223,7 @@ def _dflash_fetch_during_auto_load(extra_args = tuple(extra_args) if extra_args is not None else None,dflash_draft_path = dflash_draft_path,mtp_draft_path = mtp_draft_path if gguf_path else None,+ disable_vision = disable_vision,))seen["dflash_promoted"] = any("using draft-dflash" in m for m in seen["log"])@@ -4398,3 +4410,163 @@ def test_mtp_drafter_loads_standalone_needs_only_token_embd(tmp_path):arch = "qwen4exp",)assert _mtp_drafter_loads_standalone(str(drafter))+++# #13511: a NeoX DFlash drafter cannot follow an M-RoPE image, so Auto keeps vision without it.+++def _gguf_header(path, arch, kv):+ import struct++ entries = [("general.architecture", arch), *kv.items()]+ out = [struct.pack("<IIQQ", 0x46554747, 3, 0, len(entries))]+ for key, val in entries:+ out.append(struct.pack("<Q", len(key)) + key.encode())+ if isinstance(val, str):+ out.append(struct.pack("<IQ", 8, len(val)) + val.encode())+ elif isinstance(val, bool):+ out.append(struct.pack("<I?", 7, val))+ elif isinstance(val, list):+ out.append(struct.pack("<IIQ", 9, 5, len(val)) + struct.pack(f"<{len(val)}i", *val))+ else:+ out.append(struct.pack("<II", 4, val))+ path.write_bytes(b"".join(out))+ return str(path)+++def vision_files(tmp_path):+ return {+ "mrope_model": _gguf_header(+ tmp_path / "Qwen3.8-27B-Q4_K_M.gguf",+ "qwen35",+ {"qwen35.rope.dimension_sections": [11, 11, 10, 0], "qwen35.block_count": 64},+ ),+ "neox_model": _gguf_header(+ tmp_path / "Muse-30B-Q4_K_M.gguf", "llama", {"llama.block_count": 2}+ ),+ "neox_dflash": _gguf_header(+ tmp_path / "dflash-Qwen3.8-27B-BF16.gguf",+ "dflash",+ {"dflash.rope.freq_base_u": 1, "tokenizer.ggml.pre": "qwen35"},+ ),+ "mrope_dflash": _gguf_header(+ tmp_path / "dflash-mrope.gguf",+ "dflash",+ {"dflash.rope.dimension_sections": [11, 11, 10, 0]},+ ),+ "zero_dflash": _gguf_header(+ tmp_path / "dflash-zero.gguf",+ "dflash",+ {"dflash.rope.dimension_sections": [0, 0, 0, 0]},+ ),+ "image_mmproj": _gguf_header(+ tmp_path / "mmproj-F16.gguf", "clip", {"clip.has_vision_encoder": True}+ ),+ "audio_mmproj": _gguf_header(+ tmp_path / "mmproj-audio.gguf",+ "clip",+ {"clip.has_vision_encoder": False, "clip.has_audio_encoder": True},+ ),+ }+++def test_mrope_sections_reader(vision_files, tmp_path):+ from utils.models.gguf_metadata import gguf_has_mrope_sections++ assert gguf_has_mrope_sections(vision_files["mrope_model"]) is True+ assert gguf_has_mrope_sections(vision_files["mrope_dflash"]) is True+ assert gguf_has_mrope_sections(vision_files["neox_dflash"]) is False+ assert gguf_has_mrope_sections(vision_files["zero_dflash"]) is False+ assert gguf_has_mrope_sections(vision_files["neox_model"]) is False+ # Another namespace's sections are not this architecture's.+ other = _gguf_header(+ tmp_path / "other.gguf", "dflash", {"qwen35.rope.dimension_sections": [1, 1, 1, 0]}+ )+ assert gguf_has_mrope_sections(other) is False+ garbage = tmp_path / "garbage.gguf"+ garbage.write_bytes(b"not a gguf at all")+ assert gguf_has_mrope_sections(str(garbage)) is None+ truncated = tmp_path / "truncated.gguf"+ truncated.write_bytes(Path(vision_files["mrope_model"]).read_bytes()[:40])+ assert gguf_has_mrope_sections(str(truncated)) is None+ assert gguf_has_mrope_sections(str(tmp_path / "missing.gguf")) is None+++ "model, dflash, mmproj, expected",+ [+ ("mrope_model", "neox_dflash", "image_mmproj", True),+ ("mrope_model", "zero_dflash", "image_mmproj", True),+ ("mrope_model", "mrope_dflash", "image_mmproj", False),+ ("neox_model", "neox_dflash", "image_mmproj", False),+ ("mrope_model", "neox_dflash", "audio_mmproj", False),+ ("mrope_model", "neox_dflash", None, False),+ ("mrope_model", "missing", "image_mmproj", False),+ ],+)+def test_dflash_drafter_breaks_on_images(vision_files, tmp_path, model, dflash, mmproj, expected):+ from core.inference.llama_cpp import _dflash_drafter_breaks_on_images+ files = {**vision_files, "missing": str(tmp_path / "missing.gguf")}+ assert (+ _dflash_drafter_breaks_on_images(+ files[model], files[dflash], files[mmproj] if mmproj else None+ )+ is expected+ )+++def test_auto_skips_neox_dflash_on_an_mrope_vision_load(monkeypatch, vision_files):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ model_file = vision_files["mrope_model"],+ dflash_file = vision_files["neox_dflash"],+ mmproj_file = vision_files["image_mmproj"],+ )+ assert seen["dflash_fetched"] is True+ assert seen["dflash_promoted"] is False+ assert any("no M-RoPE sections" in m for m in seen["log"])+++ "dflash, mmproj, model, disable_vision",+ [+ ("mrope_dflash", "image_mmproj", "mrope_model", False),+ ("neox_dflash", "image_mmproj", "neox_model", False),+ ("neox_dflash", "audio_mmproj", "mrope_model", False),+ ("neox_dflash", None, "mrope_model", False),+ ("neox_dflash", "image_mmproj", "mrope_model", True),+ ],+)+def test_auto_keeps_dflash_when_images_cannot_break_it(+ monkeypatch, vision_files, dflash, mmproj, model, disable_vision+):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ model_file = vision_files[model],+ dflash_file = vision_files[dflash],+ mmproj_file = vision_files[mmproj] if mmproj else None,+ disable_vision = disable_vision,+ )+ assert seen["dflash_promoted"] is True+++def test_forced_dflash_on_an_mrope_vision_load_still_launches_and_warns(monkeypatch, vision_files):+ seen = _dflash_fetch_during_auto_load(+ monkeypatch,+ supports_dspark = False,+ supports_dflash = True,+ dspark_cached = None,+ speculative_type = "dflash",+ model_file = vision_files["mrope_model"],+ dflash_file = vision_files["neox_dflash"],+ mmproj_file = vision_files["image_mmproj"],+ )+ assert seen["dflash_fetched"] is True+ assert any("DFlash forced" in m for m in seen["warnings"])diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.pyindex d23a935f9e5..929bbdc4add 100644--- a/studio/backend/utils/models/gguf_metadata.py+++ b/studio/backend/utils/models/gguf_metadata.py@@ -68,6 +68,8 @@ _N_EMBD_CACHE: Dict[_CacheKey, Optional[int]] = {}# Cache the embedded speculative-head count separately for discovery, launch, and sizing._NEXTN_CACHE: Dict[_CacheKey, Optional[int]] = {}+_MROPE_SECTIONS_CACHE: Dict[_CacheKey, Optional[bool]] = {}+def _cache_key(path: str) -> Optional[_CacheKey]:try:@@ -676,6 +678,82 @@ def _read_gguf_bool(path: str, wanted_key: str) -> Optional[bool]:return result+def _parse_gguf_has_mrope_sections(path: str) -> Optional[bool]:+ arch: Optional[str] = None+ nonzero: Dict[str, bool] = {}+ try:+ with open(path, "rb") as f:+ head = f.read(24)+ if len(head) < 24:+ return None+ magic, _version, _tcount, kv_count = struct.unpack("<IIQQ", head)+ if magic != _GGUF_MAGIC:+ return None+ for _ in range(kv_count):+ klen_bytes = f.read(8)+ if len(klen_bytes) < 8:+ return None+ klen = struct.unpack("<Q", klen_bytes)[0]+ if klen > 1 << 20:+ return None+ kbytes = f.read(klen)+ vt_bytes = f.read(4)+ if len(kbytes) < klen or len(vt_bytes) < 4:+ return None+ key = kbytes.decode("utf-8", "replace")+ vtype = struct.unpack("<I", vt_bytes)[0]+ if vtype == 8 and key == "general.architecture":+ slen_bytes = f.read(8)+ if len(slen_bytes) < 8:+ return None+ slen = struct.unpack("<Q", slen_bytes)[0]+ if slen > 1 << 22:+ return None+ arch = f.read(slen).decode("utf-8", "replace")+ elif vtype == 9 and key.endswith(".rope.dimension_sections"):+ ahead = f.read(12)+ if len(ahead) < 12:+ return None+ atype, alen = struct.unpack("<IQ", ahead)+ # llama.cpp reads these as int32 (get_key_or_arr, at most 4).+ if atype not in (4, 5) or alen > 16:+ return None+ data = f.read(4 * alen)+ if len(data) < 4 * alen:+ return None+ values = struct.unpack(f"<{alen}{'I' if atype == 4 else 'i'}", data)+ nonzero[key] = any(values)+ elif not _skip_gguf_value(f, vtype):+ return None+ except (OSError, struct.error, UnicodeDecodeError) as e:+ logger.debug(f"_parse_gguf_has_mrope_sections: cannot read {path}: {e}")+ return None+ if arch is None:+ return None+ return nonzero.get(f"{arch}.rope.dimension_sections", False)+++def gguf_has_mrope_sections(path: str) -> Optional[bool]:+ """Whether the selected architecture declares a nonzero ``rope.dimension_sections``.++ llama.cpp makes a DFlash drafter M-RoPE exactly when this holds (``llama_model_rope_type``), and an M-RoPE target is what pins an image to one position. ``False`` is a readable header without the key (or all zeros); ``None`` is unreadable, so callers can fail open."""+ fkey = _cache_key(path)+ if fkey is None:+ return None+ with _CACHE_LOCK:+ if fkey in _MROPE_SECTIONS_CACHE:+ return _MROPE_SECTIONS_CACHE[fkey]+ result = _parse_gguf_has_mrope_sections(path)+ with _CACHE_LOCK:+ while len(_MROPE_SECTIONS_CACHE) >= _CACHE_MAX_ENTRIES:+ try:+ _MROPE_SECTIONS_CACHE.pop(next(iter(_MROPE_SECTIONS_CACHE)))+ except StopIteration:+ break+ _MROPE_SECTIONS_CACHE[fkey] = result+ return result++def _parse_gguf_string(path: str, wanted_key: str) -> Optional[str]:try:with open(path, "rb") as f: