Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: skip a DFlash drafter that an image would break under Auto (#13553)

Daniel Han579809c46ccd
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 801c51de6e9..5688b71a094 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -5565,6 +5565,25 @@ def _extra_args_requests_dspark(
return "draft-dspark" in _accumulated_spec_types(extra_args, env)
+def _dflash_drafter_breaks_on_images(
+ model_path: Optional[str], dflash_path: Optional[str], projector: Optional[str]
+) -> bool:
+ """#13511: llama.cpp skips an M-RoPE image in the DFlash drafter (its rows share one
+ position) and a NeoX drafter then rejects the jump, so the first image sent after any
+ text fails the slot. Unreadable headers fail open."""
+ if not (model_path and dflash_path and projector):
+ return False
+ try:
+ from utils.models.gguf_metadata import gguf_has_mrope_sections, mmproj_accepts_image
+ return bool(
+ mmproj_accepts_image(projector)
+ and gguf_has_mrope_sections(model_path) is True
+ and gguf_has_mrope_sections(dflash_path) is False
+ )
+ except Exception:
+ return False
+
+
def _extra_args_requests_dflash(
extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
) -> bool:
@@ -25160,6 +25179,17 @@ class LlamaCppBackend:
return False
return False
+ def _dflash_breaks_on_images() -> bool:
+ if not dflash_draft_path or disable_vision or extra_args_disable_mmproj(extra_args):
+ return False
+ try:
+ projector = self._resolve_launch_mmproj_path(
+ model_path = model_path, mmproj_path = mmproj_path, extra_args = extra_args
+ )
+ except Exception:
+ return False
+ return _dflash_drafter_breaks_on_images(model_path, dflash_draft_path, projector)
+
_unloadable_mtp_draft_path: Optional[str] = None
# Scope HF_HUB_OFFLINE to the download block only when DNS is
# dead; cleanup runs even on exception so a transient hiccup
@@ -25331,11 +25361,30 @@ class LlamaCppBackend:
"keeping tensor split with the MTP drafter "
"(llama.cpp cannot run DFlash2 with --split-mode tensor yet)."
)
+ elif _dflash_breaks_on_images():
+ logger.info(
+ "Auto: DFlash sidecar %s has no M-RoPE sections, so llama.cpp "
+ "cannot carry it past an image on this vision model; keeping "
+ "vision and skipping DFlash. Choose DFlash in the Speculative "
+ "Decoding dropdown to force it, or turn Vision off.",
+ dflash_draft_path,
+ )
else:
_spec_canon = "dflash"
logger.info("Auto: DFlash sidecar available, using draft-dflash.")
except Exception as exc:
logger.debug("DFlash capability probe failed during Auto: %s", exc)
+ elif (
+ _spec_canon == "dflash"
+ and not _extra_args_set_spec_type(extra_args)
+ and _dflash_breaks_on_images()
+ ):
+ logger.warning(
+ "DFlash forced with sidecar %s, which has no M-RoPE sections: llama.cpp "
+ "will fail the first image sent after text in a chat. Use Auto or turn "
+ "Vision off to keep images working.",
+ dflash_draft_path,
+ )
# MTP and DSpark are mutually exclusive, and the drafter sizing and
# lifecycle path below is architecture agnostic, so it carries the
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 92ecffd3aab..9587539c3e4 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -13252,6 +13252,7 @@ def _estimate_gguf_required_gb(
try:
from core.inference.llama_cpp import (
_canonicalize_spec_mode,
+ _dflash_drafter_breaks_on_images,
_extra_args_device,
_extra_args_draft_offloaded_to_cpu,
_extra_args_mtp_draft_path,
@@ -13346,6 +13347,17 @@ def _estimate_gguf_required_gb(
_auto_dflash = False
except Exception:
pass
+ # Mirrors load_model's #13511 rule: Auto skips a NeoX DFlash drafter on an M-RoPE vision load.
+ if _auto_dflash and not disable_vision and not extra_args_disable_mmproj(llama_extra_args):
+ _vd_mmproj = _extra_args_device(llama_extra_args, {"--mmproj", "-mm"}) or getattr(
+ config, "gguf_mmproj_file", None
+ )
+ if _vd_mmproj and Path(_vd_mmproj).is_file():
+ _auto_dflash = not _dflash_drafter_breaks_on_images(
+ getattr(config, "gguf_file", None),
+ getattr(config, "gguf_dflash_file", None),
+ str(_vd_mmproj),
+ )
_dflash_capable = True
if _forced_dflash or _auto_dflash:
try:
diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.py
index b84da076ca7..6e3c73e4007 100644
--- a/studio/backend/tests/test_chat_load_during_training.py
+++ b/studio/backend/tests/test_chat_load_during_training.py
@@ -1772,6 +1772,73 @@ class TestEstimateGgufRequiredGb(unittest.TestCase):
self.assertAlmostEqual(resident_tensor, 7000 / (1024**3), places = 9)
self.assertAlmostEqual(resident_layer, 5000 / (1024**3), places = 9)
+ def test_auto_drops_a_neox_dflash_charge_on_an_mrope_vision_load(self):
+ """#13511: Auto launches no DFlash drafter that an image would break, so none is charged."""
+ import struct
+ import tempfile
+
+ def _header(path, arch, kv, size):
+ entries = [("general.architecture", arch), *kv.items()]
+ out = [struct.pack("<IIQQ", 0x46554747, 3, 0, len(entries))]
+ for key, val in entries:
+ out.append(struct.pack("<Q", len(key)) + key.encode())
+ if isinstance(val, str):
+ out.append(struct.pack("<IQ", 8, len(val)) + val.encode())
+ elif isinstance(val, bool):
+ out.append(struct.pack("<I?", 7, val))
+ else:
+ out.append(
+ struct.pack("<IIQ", 9, 5, len(val)) + struct.pack(f"<{len(val)}i", *val)
+ )
+ blob = b"".join(out)
+ path.write_bytes(blob + b"\0" * (size - len(blob)))
+ return str(path)
+
+ sections = [11, 11, 10, 0]
+ with tempfile.TemporaryDirectory() as d:
+ p = Path(d)
+ target = _header(
+ p / "model.gguf", "qwen35", {"qwen35.rope.dimension_sections": sections}, 2000
+ )
+ neox = _header(p / "dflash-neox.gguf", "dflash", {}, 3000)
+ mrope = _header(
+ p / "dflash-mrope.gguf",
+ "dflash",
+ {"dflash.rope.dimension_sections": sections},
+ 3000,
+ )
+ mmproj = _header(p / "mmproj-F16.gguf", "clip", {"clip.has_vision_encoder": True}, 7000)
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ def _estimate(dflash, **kw):
+ cfg = _gguf_cfg(
+ gguf_file = target,
+ gguf_dflash_file = dflash,
+ gguf_mmproj_file = mmproj,
+ gguf_hf_repo = None,
+ gguf_variant = None,
+ )
+ with (
+ patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
+ patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
+ patch.object(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(lambda cls, binary = None: {"supports_dflash": True}),
+ ),
+ ):
+ return self.route._estimate_gguf_required_gb(cfg, **kw)
+
+ auto_neox = _estimate(neox, speculative_type = "auto")
+ auto_mrope = _estimate(mrope, speculative_type = "auto")
+ auto_neox_vision_off = _estimate(neox, speculative_type = "auto", disable_vision = True)
+ forced_neox = _estimate(neox, speculative_type = "dflash")
+ self.assertAlmostEqual(auto_neox, 9000 / (1024**3), places = 9)
+ self.assertAlmostEqual(auto_mrope, 12000 / (1024**3), places = 9)
+ self.assertAlmostEqual(auto_neox_vision_off, 5000 / (1024**3), places = 9)
+ self.assertAlmostEqual(forced_neox, 12000 / (1024**3), places = 9)
+
def test_extra_args_drafter_is_charged_once_when_it_is_the_local_sidecar(self):
"""--model-draft usually names the very sidecar discovery already found,
and charging it on both paths billed a 1.5 GiB drafter as 3 GiB, so the
diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py
index b1ec5a23eee..cb5a51673ec 100644
--- a/studio/backend/tests/test_mtp_drafter_companion.py
+++ b/studio/backend/tests/test_mtp_drafter_companion.py
@@ -3131,6 +3131,10 @@ def _dflash_fetch_during_auto_load(
mtp_draft_path = None,
mtp_loads = True,
mtp_token = "draft-mtp",
+ model_file = "/cache/snap/model-Q4_K_M.gguf",
+ dflash_file = "/cache/snap/dflash-kquant.gguf",
+ mmproj_file = None,
+ disable_vision = False,
):
"""Whether an Auto load fetches the DFlash sidecar, and what it resolves to.
@@ -3168,9 +3172,8 @@ def _dflash_fetch_during_auto_load(
monkeypatch.setattr(backend, "_get_gpu_memory", lambda _binary = None, **_kw: [(0, 4096, 8192)])
monkeypatch.setattr(backend, "_gguf_path_is_diffusion", lambda *_args: False)
monkeypatch.setattr(backend, "_kill_process", lambda: None)
- monkeypatch.setattr(
- backend, "_download_gguf", lambda **_kwargs: "/cache/snap/model-Q4_K_M.gguf"
- )
+ monkeypatch.setattr(backend, "_download_gguf", lambda **_kwargs: model_file)
+ monkeypatch.setattr(backend, "_resolve_launch_mmproj_path", lambda **_kwargs: mmproj_file)
monkeypatch.setattr(backend, "_download_mtp", lambda **_kwargs: mtp_draft_path)
monkeypatch.setattr(llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _path: mtp_loads)
# Exactly what _download_dspark does for a cached sidecar on a binary that
@@ -3179,7 +3182,7 @@ def _dflash_fetch_during_auto_load(
def _fetch_dflash(**_kwargs):
seen["dflash_fetched"] = True
- return "/cache/snap/dflash-kquant.gguf"
+ return dflash_file
monkeypatch.setattr(backend, "_download_dflash", _fetch_dflash)
seen["log"] = []
@@ -3190,6 +3193,14 @@ def _dflash_fetch_during_auto_load(
return _real_info(msg, *args, **kwargs)
monkeypatch.setattr(llama_cpp_module.logger, "info", _info)
+ seen["warnings"] = []
+ _real_warning = llama_cpp_module.logger.warning
+
+ def _warning(msg, *args, **kwargs):
+ seen["warnings"].append(str(msg))
+ return _real_warning(msg, *args, **kwargs)
+
+ monkeypatch.setattr(llama_cpp_module.logger, "warning", _warning)
def _stop(*_args, **_kwargs):
raise _StopAfterDownloads
@@ -3212,6 +3223,7 @@ def _dflash_fetch_during_auto_load(
extra_args = tuple(extra_args) if extra_args is not None else None,
dflash_draft_path = dflash_draft_path,
mtp_draft_path = mtp_draft_path if gguf_path else None,
+ disable_vision = disable_vision,
)
)
seen["dflash_promoted"] = any("using draft-dflash" in m for m in seen["log"])
@@ -4398,3 +4410,163 @@ def test_mtp_drafter_loads_standalone_needs_only_token_embd(tmp_path):
arch = "qwen4exp",
)
assert _mtp_drafter_loads_standalone(str(drafter))
+
+
+# #13511: a NeoX DFlash drafter cannot follow an M-RoPE image, so Auto keeps vision without it.
+
+
+def _gguf_header(path, arch, kv):
+ import struct
+
+ entries = [("general.architecture", arch), *kv.items()]
+ out = [struct.pack("<IIQQ", 0x46554747, 3, 0, len(entries))]
+ for key, val in entries:
+ out.append(struct.pack("<Q", len(key)) + key.encode())
+ if isinstance(val, str):
+ out.append(struct.pack("<IQ", 8, len(val)) + val.encode())
+ elif isinstance(val, bool):
+ out.append(struct.pack("<I?", 7, val))
+ elif isinstance(val, list):
+ out.append(struct.pack("<IIQ", 9, 5, len(val)) + struct.pack(f"<{len(val)}i", *val))
+ else:
+ out.append(struct.pack("<II", 4, val))
+ path.write_bytes(b"".join(out))
+ return str(path)
+
+
+def vision_files(tmp_path):
+ return {
+ "mrope_model": _gguf_header(
+ tmp_path / "Qwen3.8-27B-Q4_K_M.gguf",
+ "qwen35",
+ {"qwen35.rope.dimension_sections": [11, 11, 10, 0], "qwen35.block_count": 64},
+ ),
+ "neox_model": _gguf_header(
+ tmp_path / "Muse-30B-Q4_K_M.gguf", "llama", {"llama.block_count": 2}
+ ),
+ "neox_dflash": _gguf_header(
+ tmp_path / "dflash-Qwen3.8-27B-BF16.gguf",
+ "dflash",
+ {"dflash.rope.freq_base_u": 1, "tokenizer.ggml.pre": "qwen35"},
+ ),
+ "mrope_dflash": _gguf_header(
+ tmp_path / "dflash-mrope.gguf",
+ "dflash",
+ {"dflash.rope.dimension_sections": [11, 11, 10, 0]},
+ ),
+ "zero_dflash": _gguf_header(
+ tmp_path / "dflash-zero.gguf",
+ "dflash",
+ {"dflash.rope.dimension_sections": [0, 0, 0, 0]},
+ ),
+ "image_mmproj": _gguf_header(
+ tmp_path / "mmproj-F16.gguf", "clip", {"clip.has_vision_encoder": True}
+ ),
+ "audio_mmproj": _gguf_header(
+ tmp_path / "mmproj-audio.gguf",
+ "clip",
+ {"clip.has_vision_encoder": False, "clip.has_audio_encoder": True},
+ ),
+ }
+
+
+def test_mrope_sections_reader(vision_files, tmp_path):
+ from utils.models.gguf_metadata import gguf_has_mrope_sections
+
+ assert gguf_has_mrope_sections(vision_files["mrope_model"]) is True
+ assert gguf_has_mrope_sections(vision_files["mrope_dflash"]) is True
+ assert gguf_has_mrope_sections(vision_files["neox_dflash"]) is False
+ assert gguf_has_mrope_sections(vision_files["zero_dflash"]) is False
+ assert gguf_has_mrope_sections(vision_files["neox_model"]) is False
+ # Another namespace's sections are not this architecture's.
+ other = _gguf_header(
+ tmp_path / "other.gguf", "dflash", {"qwen35.rope.dimension_sections": [1, 1, 1, 0]}
+ )
+ assert gguf_has_mrope_sections(other) is False
+ garbage = tmp_path / "garbage.gguf"
+ garbage.write_bytes(b"not a gguf at all")
+ assert gguf_has_mrope_sections(str(garbage)) is None
+ truncated = tmp_path / "truncated.gguf"
+ truncated.write_bytes(Path(vision_files["mrope_model"]).read_bytes()[:40])
+ assert gguf_has_mrope_sections(str(truncated)) is None
+ assert gguf_has_mrope_sections(str(tmp_path / "missing.gguf")) is None
+
+
+ "model, dflash, mmproj, expected",
+ [
+ ("mrope_model", "neox_dflash", "image_mmproj", True),
+ ("mrope_model", "zero_dflash", "image_mmproj", True),
+ ("mrope_model", "mrope_dflash", "image_mmproj", False),
+ ("neox_model", "neox_dflash", "image_mmproj", False),
+ ("mrope_model", "neox_dflash", "audio_mmproj", False),
+ ("mrope_model", "neox_dflash", None, False),
+ ("mrope_model", "missing", "image_mmproj", False),
+ ],
+)
+def test_dflash_drafter_breaks_on_images(vision_files, tmp_path, model, dflash, mmproj, expected):
+ from core.inference.llama_cpp import _dflash_drafter_breaks_on_images
+ files = {**vision_files, "missing": str(tmp_path / "missing.gguf")}
+ assert (
+ _dflash_drafter_breaks_on_images(
+ files[model], files[dflash], files[mmproj] if mmproj else None
+ )
+ is expected
+ )
+
+
+def test_auto_skips_neox_dflash_on_an_mrope_vision_load(monkeypatch, vision_files):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ model_file = vision_files["mrope_model"],
+ dflash_file = vision_files["neox_dflash"],
+ mmproj_file = vision_files["image_mmproj"],
+ )
+ assert seen["dflash_fetched"] is True
+ assert seen["dflash_promoted"] is False
+ assert any("no M-RoPE sections" in m for m in seen["log"])
+
+
+ "dflash, mmproj, model, disable_vision",
+ [
+ ("mrope_dflash", "image_mmproj", "mrope_model", False),
+ ("neox_dflash", "image_mmproj", "neox_model", False),
+ ("neox_dflash", "audio_mmproj", "mrope_model", False),
+ ("neox_dflash", None, "mrope_model", False),
+ ("neox_dflash", "image_mmproj", "mrope_model", True),
+ ],
+)
+def test_auto_keeps_dflash_when_images_cannot_break_it(
+ monkeypatch, vision_files, dflash, mmproj, model, disable_vision
+):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ model_file = vision_files[model],
+ dflash_file = vision_files[dflash],
+ mmproj_file = vision_files[mmproj] if mmproj else None,
+ disable_vision = disable_vision,
+ )
+ assert seen["dflash_promoted"] is True
+
+
+def test_forced_dflash_on_an_mrope_vision_load_still_launches_and_warns(monkeypatch, vision_files):
+ seen = _dflash_fetch_during_auto_load(
+ monkeypatch,
+ supports_dspark = False,
+ supports_dflash = True,
+ dspark_cached = None,
+ speculative_type = "dflash",
+ model_file = vision_files["mrope_model"],
+ dflash_file = vision_files["neox_dflash"],
+ mmproj_file = vision_files["image_mmproj"],
+ )
+ assert seen["dflash_fetched"] is True
+ assert any("DFlash forced" in m for m in seen["warnings"])
diff --git a/studio/backend/utils/models/gguf_metadata.py b/studio/backend/utils/models/gguf_metadata.py
index d23a935f9e5..929bbdc4add 100644
--- a/studio/backend/utils/models/gguf_metadata.py
+++ b/studio/backend/utils/models/gguf_metadata.py
@@ -68,6 +68,8 @@ _N_EMBD_CACHE: Dict[_CacheKey, Optional[int]] = {}
# Cache the embedded speculative-head count separately for discovery, launch, and sizing.
_NEXTN_CACHE: Dict[_CacheKey, Optional[int]] = {}
+_MROPE_SECTIONS_CACHE: Dict[_CacheKey, Optional[bool]] = {}
+
def _cache_key(path: str) -> Optional[_CacheKey]:
try:
@@ -676,6 +678,82 @@ def _read_gguf_bool(path: str, wanted_key: str) -> Optional[bool]:
return result
+def _parse_gguf_has_mrope_sections(path: str) -> Optional[bool]:
+ arch: Optional[str] = None
+ nonzero: Dict[str, bool] = {}
+ try:
+ with open(path, "rb") as f:
+ head = f.read(24)
+ if len(head) < 24:
+ return None
+ magic, _version, _tcount, kv_count = struct.unpack("<IIQQ", head)
+ if magic != _GGUF_MAGIC:
+ return None
+ for _ in range(kv_count):
+ klen_bytes = f.read(8)
+ if len(klen_bytes) < 8:
+ return None
+ klen = struct.unpack("<Q", klen_bytes)[0]
+ if klen > 1 << 20:
+ return None
+ kbytes = f.read(klen)
+ vt_bytes = f.read(4)
+ if len(kbytes) < klen or len(vt_bytes) < 4:
+ return None
+ key = kbytes.decode("utf-8", "replace")
+ vtype = struct.unpack("<I", vt_bytes)[0]
+ if vtype == 8 and key == "general.architecture":
+ slen_bytes = f.read(8)
+ if len(slen_bytes) < 8:
+ return None
+ slen = struct.unpack("<Q", slen_bytes)[0]
+ if slen > 1 << 22:
+ return None
+ arch = f.read(slen).decode("utf-8", "replace")
+ elif vtype == 9 and key.endswith(".rope.dimension_sections"):
+ ahead = f.read(12)
+ if len(ahead) < 12:
+ return None
+ atype, alen = struct.unpack("<IQ", ahead)
+ # llama.cpp reads these as int32 (get_key_or_arr, at most 4).
+ if atype not in (4, 5) or alen > 16:
+ return None
+ data = f.read(4 * alen)
+ if len(data) < 4 * alen:
+ return None
+ values = struct.unpack(f"<{alen}{'I' if atype == 4 else 'i'}", data)
+ nonzero[key] = any(values)
+ elif not _skip_gguf_value(f, vtype):
+ return None
+ except (OSError, struct.error, UnicodeDecodeError) as e:
+ logger.debug(f"_parse_gguf_has_mrope_sections: cannot read {path}: {e}")
+ return None
+ if arch is None:
+ return None
+ return nonzero.get(f"{arch}.rope.dimension_sections", False)
+
+
+def gguf_has_mrope_sections(path: str) -> Optional[bool]:
+ """Whether the selected architecture declares a nonzero ``rope.dimension_sections``.
+
+ llama.cpp makes a DFlash drafter M-RoPE exactly when this holds (``llama_model_rope_type``), and an M-RoPE target is what pins an image to one position. ``False`` is a readable header without the key (or all zeros); ``None`` is unreadable, so callers can fail open."""
+ fkey = _cache_key(path)
+ if fkey is None:
+ return None
+ with _CACHE_LOCK:
+ if fkey in _MROPE_SECTIONS_CACHE:
+ return _MROPE_SECTIONS_CACHE[fkey]
+ result = _parse_gguf_has_mrope_sections(path)
+ with _CACHE_LOCK:
+ while len(_MROPE_SECTIONS_CACHE) >= _CACHE_MAX_ENTRIES:
+ try:
+ _MROPE_SECTIONS_CACHE.pop(next(iter(_MROPE_SECTIONS_CACHE)))
+ except StopIteration:
+ break
+ _MROPE_SECTIONS_CACHE[fkey] = result
+ return result
+
+
def _parse_gguf_string(path: str, wanted_key: str) -> Optional[str]:
try:
with open(path, "rb") as f: