Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: size a cached DFlash sidecar from disk instead of asking the Hub on every load (#15986)

Daniel Hanc3c04e2881ef
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 12603b59895..c65b4eb4dfb 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -20051,8 +20051,31 @@ class LlamaCppBackend:
# Header-rejected basenames, so the search moves past an impostor to the real sidecar.
rejected: set[str] = set()
+ # Once per call; an empty answer (also a failed model_info) is asked again, as before.
+ remote_sizes: dict[str, int] = {}
+
+ def _remote_sizes() -> dict[str, int]:
+ nonlocal remote_sizes
+ if not remote_sizes:
+ remote_sizes = self._remote_root_gguf_sizes(hf_repo, hf_token)
+ return remote_sizes
+
+ snap = _snapshot_dir_of(near_path) if near_path else None
+
+ def _local_sizes(candidates: list[str]) -> dict[str, int]:
+ sizes: dict[str, int] = {}
+ if snap is None:
+ return sizes
+ for name in candidates:
+ if "/" in name:
+ continue
+ try:
+ sizes[name] = (snap / name).stat().st_size
+ except OSError:
+ continue
+ return sizes
- def _pick_dflash(candidates: list[str]) -> Optional[str]:
+ def _pick_dflash(candidates: list[str], *, local: bool = False) -> Optional[str]:
# Ranked against the weight loaded, not precision alone, so model B does not get model A's drafter; a sidecar naming no family stays eligible.
from utils.models.drafters import (
dflash_repo_preference_key,
@@ -20075,22 +20098,28 @@ class LlamaCppBackend:
]
if not eligible:
return None
- # Same size bound as dflash_plan_files: header reads only after download, a drafter is a few layers of its target, and an unlisted size keeps the candidate.
- sizes = self._remote_root_gguf_sizes(hf_repo, hf_token)
try:
target_bytes = (self._get_gguf_size_bytes(near_path) or 0) if near_path else 0
except OSError:
target_bytes = 0
- files = sorted(
- (
+
+ def _under_bound(sizes: Mapping[str, int]) -> list[str]:
+ # Same size bound as dflash_plan_files: header reads only after download, a drafter is a few layers of its target, and an unlisted size keeps the candidate.
+ return [
name
for name in eligible
# Whole set, not the picked shard: each half of a split weight can sit under the target while the set does not.
if not (
target_bytes and _drafter_set_bytes(sizes, Path(name).name) >= target_bytes
)
- ),
- key = lambda name: dflash_repo_preference_key(name, weight_name, others),
+ ]
+
+ # On-disk sizes settle it only when they keep every candidate, as an unanswered model_info does; else ask the Hub.
+ kept = _under_bound(_local_sizes(candidates)) if local else None
+ if kept is None or len(kept) != len(eligible):
+ kept = _under_bound(_remote_sizes())
+ files = sorted(
+ kept, key = lambda name: dflash_repo_preference_key(name, weight_name, others)
)
return files[0] if files else None
@@ -20110,7 +20139,9 @@ class LlamaCppBackend:
cached: Optional[str] = None
seen_siblings: set[str] = set()
while near_path and not cancel_event.is_set():
- sibling = _companion_snapshot_sibling(near_path, _pick_dflash)
+ sibling = _companion_snapshot_sibling(
+ near_path, functools.partial(_pick_dflash, local = True)
+ )
if sibling is None or sibling in seen_siblings:
break
seen_siblings.add(sibling)
diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py
index b1e4eeec650..69916711889 100644
--- a/studio/backend/tests/test_mtp_drafter_companion.py
+++ b/studio/backend/tests/test_mtp_drafter_companion.py
@@ -4292,3 +4292,175 @@ def test_download_dflash_asks_the_hub_for_sizes_only_with_a_candidate(
picked = _dflash_download_pick(monkeypatch, listing = listing, near_path = str(weight))
assert picked == ("dflash-model.gguf" if sized else None)
assert len(calls) == (1 if sized else 0)
+
+
+def _dflash_snapshot_load(
+ monkeypatch,
+ tmp_path,
+ files,
+ weight,
+ *,
+ hub_sizes = True,
+):
+ """``files``: name -> (architecture, bytes, on disk); the Hub lists them all or answers nothing."""
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ snap = tmp_path / "models--org--repo" / "snapshots" / "abc"
+ snap.mkdir(parents = True)
+ for name, (arch, size, on_disk) in files.items():
+ if on_disk:
+ path = _write_gguf(snap / name, arch)
+ with open(path, "ab") as fh:
+ fh.write(b"\0" * (size - path.stat().st_size))
+ calls: list = []
+
+ def _sizes(repo, token = None):
+ calls.append(repo)
+ return {name: size for name, (_, size, _) in files.items()} if hub_sizes else {}
+
+ monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
+ monkeypatch.setattr(LlamaCppBackend, "_remote_root_gguf_sizes", staticmethod(_sizes))
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(lambda cls, binary = None: {"supports_dflash": True}),
+ )
+
+ def _fake_companion(
+ *,
+ pick,
+ outcome = None,
+ **_,
+ ):
+ target = pick(list(files))
+ return str(snap / target) if target and (snap / target).is_file() else None
+
+ b = LlamaCppBackend()
+ b._download_companion_gguf = _fake_companion
+ got = b._download_dflash(
+ hf_repo = "org/repo", near_path = str(snap / weight), binary = "/fake/llama-server"
+ )
+ return (got and Path(got).relative_to(snap).as_posix()), len(calls)
+
+
+def test_download_dflash_reuses_a_cached_sidecar_without_asking_the_hub_for_sizes(
+ monkeypatch, tmp_path
+):
+ """An unreachable Hub not flagged offline otherwise stalls the load until the client timeout."""
+ files = {
+ "model-Q4_K_M.gguf": ("llama", 100_000, True),
+ "dflash-kquant.gguf": ("dflash", 2_000, True),
+ }
+ got, hub_calls = _dflash_snapshot_load(monkeypatch, tmp_path, files, "model-Q4_K_M.gguf")
+ assert got == "dflash-kquant.gguf"
+ assert hub_calls == 0
+
+
+_D, _L = "dflash", "llama"
+
+
[email protected]("hub_sizes", [True, False], ids = ["hub-answers", "hub-down"])
+ "weight, files, expected, expected_calls",
+ [
+ # Picks and model_info calls as (Hub answers, Hub down).
+ pytest.param(
+ "model-Q4_K_M-00001-of-00002.gguf",
+ {
+ "model-Q4_K_M-00001-of-00002.gguf": (_L, 3_000, True),
+ "model-Q4_K_M-00002-of-00002.gguf": (_L, 3_000, True),
+ "dflash-kquant.gguf": (_D, 5_000, True),
+ },
+ ("dflash-kquant.gguf", "dflash-kquant.gguf"),
+ (0, 0),
+ id = "split-target",
+ ),
+ pytest.param(
+ "model-Q4_K_M.gguf",
+ {
+ "model-Q4_K_M.gguf": (_L, 100_000, True),
+ "dflash-kquant-00001-of-00002.gguf": (_D, 2_000, True),
+ "dflash-kquant-00002-of-00002.gguf": (_D, 2_000, True),
+ },
+ ("dflash-kquant-00001-of-00002.gguf", "dflash-kquant-00001-of-00002.gguf"),
+ (0, 0),
+ id = "local-split",
+ ),
+ pytest.param(
+ "model-Q4_K_M.gguf",
+ {
+ "model-Q4_K_M.gguf": (_L, 100_000, True),
+ "dflash-kquant-00001-of-00002.gguf": (_D, 2_000, True),
+ "dflash-kquant-00002-of-00002.gguf": (_D, 2_000, False),
+ },
+ ("dflash-kquant-00001-of-00002.gguf", "dflash-kquant-00001-of-00002.gguf"),
+ (1, 1),
+ id = "local-split-incomplete",
+ ),
+ pytest.param(
+ "model-B-Q4_K_M.gguf",
+ {
+ "model-A-Q4_K_M.gguf": (_L, 100_000, True),
+ "model-B-Q4_K_M.gguf": (_L, 100_000, True),
+ "dflash-model-A-Q8_0.gguf": (_D, 2_000, True),
+ "dflash-kquant.gguf": (_D, 2_000, True),
+ },
+ ("dflash-kquant.gguf", "dflash-kquant.gguf"),
+ (0, 0),
+ id = "mixed-families",
+ ),
+ pytest.param(
+ "model-Q4_K_M.gguf",
+ {
+ "model-Q4_K_M.gguf": (_L, 4_000, True),
+ "dflash-model-BF16.gguf": (_D, 40_000, True),
+ "dflash-kquant.gguf": (_D, 500, True),
+ },
+ # An unanswered size request keeps every candidate, as before.
+ ("dflash-kquant.gguf", "dflash-model-BF16.gguf"),
+ (1, 1),
+ id = "oversize",
+ ),
+ pytest.param(
+ "model-Q4_K_M.gguf",
+ {
+ "model-Q4_K_M.gguf": (_L, 10_000, True),
+ "dflash-big-00001-of-00002.gguf": (_D, 6_000, True),
+ "dflash-big-00002-of-00002.gguf": (_D, 6_000, True),
+ },
+ (None, "dflash-big-00001-of-00002.gguf"),
+ (1, 1),
+ id = "split-set-over-the-bound",
+ ),
+ pytest.param(
+ "model-Q4_K_M.gguf",
+ {
+ "model-Q4_K_M.gguf": (_L, 100_000, True),
+ "dflash-model-Q4_K_M.gguf": (_L, 2_000, True),
+ "dflash-kquant.gguf": (_D, 2_000, True),
+ },
+ ("dflash-kquant.gguf", "dflash-kquant.gguf"),
+ (0, 0),
+ id = "impostor",
+ ),
+ pytest.param(
+ "model-Q4_K_M.gguf",
+ {
+ "model-Q4_K_M.gguf": (_L, 100_000, True),
+ "dflash-kquant.gguf": (_D, 2_000, False),
+ },
+ (None, None),
+ (1, 1),
+ id = "not-cached",
+ ),
+ ],
+)
+def test_download_dflash_local_sizes_keep_the_hub_sized_picks(
+ monkeypatch, tmp_path, weight, files, expected, expected_calls, hub_sizes
+):
+ """Picks match the Hub-sized scan; only a bound dropping an on-disk file asks, at most once."""
+ got, hub_calls = _dflash_snapshot_load(
+ monkeypatch, tmp_path, files, weight, hub_sizes = hub_sizes
+ )
+ assert got == expected[0 if hub_sizes else 1]
+ assert hub_calls == expected_calls[0 if hub_sizes else 1]