Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: size a cached DFlash sidecar from disk instead of asking the Hub on every load (#15986)
Daniel Hanc3c04e2881ef
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 12603b59895..c65b4eb4dfb 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -20051,8 +20051,31 @@ class LlamaCppBackend:# Header-rejected basenames, so the search moves past an impostor to the real sidecar.rejected: set[str] = set()+ # Once per call; an empty answer (also a failed model_info) is asked again, as before.+ remote_sizes: dict[str, int] = {}++ def _remote_sizes() -> dict[str, int]:+ nonlocal remote_sizes+ if not remote_sizes:+ remote_sizes = self._remote_root_gguf_sizes(hf_repo, hf_token)+ return remote_sizes++ snap = _snapshot_dir_of(near_path) if near_path else None++ def _local_sizes(candidates: list[str]) -> dict[str, int]:+ sizes: dict[str, int] = {}+ if snap is None:+ return sizes+ for name in candidates:+ if "/" in name:+ continue+ try:+ sizes[name] = (snap / name).stat().st_size+ except OSError:+ continue+ return sizes- def _pick_dflash(candidates: list[str]) -> Optional[str]:+ def _pick_dflash(candidates: list[str], *, local: bool = False) -> Optional[str]:# Ranked against the weight loaded, not precision alone, so model B does not get model A's drafter; a sidecar naming no family stays eligible.from utils.models.drafters import (dflash_repo_preference_key,@@ -20075,22 +20098,28 @@ class LlamaCppBackend:]if not eligible:return None- # Same size bound as dflash_plan_files: header reads only after download, a drafter is a few layers of its target, and an unlisted size keeps the candidate.- sizes = self._remote_root_gguf_sizes(hf_repo, hf_token)try:target_bytes = (self._get_gguf_size_bytes(near_path) or 0) if near_path else 0except OSError:target_bytes = 0- files = sorted(- (++ def _under_bound(sizes: Mapping[str, int]) -> list[str]:+ # Same size bound as dflash_plan_files: header reads only after download, a drafter is a few layers of its target, and an unlisted size keeps the candidate.+ return [namefor name in eligible# Whole set, not the picked shard: each half of a split weight can sit under the target while the set does not.if not (target_bytes and _drafter_set_bytes(sizes, Path(name).name) >= target_bytes)- ),- key = lambda name: dflash_repo_preference_key(name, weight_name, others),+ ]++ # On-disk sizes settle it only when they keep every candidate, as an unanswered model_info does; else ask the Hub.+ kept = _under_bound(_local_sizes(candidates)) if local else None+ if kept is None or len(kept) != len(eligible):+ kept = _under_bound(_remote_sizes())+ files = sorted(+ kept, key = lambda name: dflash_repo_preference_key(name, weight_name, others))return files[0] if files else None@@ -20110,7 +20139,9 @@ class LlamaCppBackend:cached: Optional[str] = Noneseen_siblings: set[str] = set()while near_path and not cancel_event.is_set():- sibling = _companion_snapshot_sibling(near_path, _pick_dflash)+ sibling = _companion_snapshot_sibling(+ near_path, functools.partial(_pick_dflash, local = True)+ )if sibling is None or sibling in seen_siblings:breakseen_siblings.add(sibling)diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.pyindex b1e4eeec650..69916711889 100644--- a/studio/backend/tests/test_mtp_drafter_companion.py+++ b/studio/backend/tests/test_mtp_drafter_companion.py@@ -4292,3 +4292,175 @@ def test_download_dflash_asks_the_hub_for_sizes_only_with_a_candidate(picked = _dflash_download_pick(monkeypatch, listing = listing, near_path = str(weight))assert picked == ("dflash-model.gguf" if sized else None)assert len(calls) == (1 if sized else 0)+++def _dflash_snapshot_load(+ monkeypatch,+ tmp_path,+ files,+ weight,+ *,+ hub_sizes = True,+):+ """``files``: name -> (architecture, bytes, on disk); the Hub lists them all or answers nothing."""+ from core.inference.llama_cpp import LlamaCppBackend++ snap = tmp_path / "models--org--repo" / "snapshots" / "abc"+ snap.mkdir(parents = True)+ for name, (arch, size, on_disk) in files.items():+ if on_disk:+ path = _write_gguf(snap / name, arch)+ with open(path, "ab") as fh:+ fh.write(b"\0" * (size - path.stat().st_size))+ calls: list = []++ def _sizes(repo, token = None):+ calls.append(repo)+ return {name: size for name, (_, size, _) in files.items()} if hub_sizes else {}++ monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)+ monkeypatch.setattr(LlamaCppBackend, "_remote_root_gguf_sizes", staticmethod(_sizes))+ monkeypatch.setattr(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(lambda cls, binary = None: {"supports_dflash": True}),+ )++ def _fake_companion(+ *,+ pick,+ outcome = None,+ **_,+ ):+ target = pick(list(files))+ return str(snap / target) if target and (snap / target).is_file() else None++ b = LlamaCppBackend()+ b._download_companion_gguf = _fake_companion+ got = b._download_dflash(+ hf_repo = "org/repo", near_path = str(snap / weight), binary = "/fake/llama-server"+ )+ return (got and Path(got).relative_to(snap).as_posix()), len(calls)+++def test_download_dflash_reuses_a_cached_sidecar_without_asking_the_hub_for_sizes(+ monkeypatch, tmp_path+):+ """An unreachable Hub not flagged offline otherwise stalls the load until the client timeout."""+ files = {+ "model-Q4_K_M.gguf": ("llama", 100_000, True),+ "dflash-kquant.gguf": ("dflash", 2_000, True),+ }+ got, hub_calls = _dflash_snapshot_load(monkeypatch, tmp_path, files, "model-Q4_K_M.gguf")+ assert got == "dflash-kquant.gguf"+ assert hub_calls == 0+++_D, _L = "dflash", "llama"++[email protected]("hub_sizes", [True, False], ids = ["hub-answers", "hub-down"])+ "weight, files, expected, expected_calls",+ [+ # Picks and model_info calls as (Hub answers, Hub down).+ pytest.param(+ "model-Q4_K_M-00001-of-00002.gguf",+ {+ "model-Q4_K_M-00001-of-00002.gguf": (_L, 3_000, True),+ "model-Q4_K_M-00002-of-00002.gguf": (_L, 3_000, True),+ "dflash-kquant.gguf": (_D, 5_000, True),+ },+ ("dflash-kquant.gguf", "dflash-kquant.gguf"),+ (0, 0),+ id = "split-target",+ ),+ pytest.param(+ "model-Q4_K_M.gguf",+ {+ "model-Q4_K_M.gguf": (_L, 100_000, True),+ "dflash-kquant-00001-of-00002.gguf": (_D, 2_000, True),+ "dflash-kquant-00002-of-00002.gguf": (_D, 2_000, True),+ },+ ("dflash-kquant-00001-of-00002.gguf", "dflash-kquant-00001-of-00002.gguf"),+ (0, 0),+ id = "local-split",+ ),+ pytest.param(+ "model-Q4_K_M.gguf",+ {+ "model-Q4_K_M.gguf": (_L, 100_000, True),+ "dflash-kquant-00001-of-00002.gguf": (_D, 2_000, True),+ "dflash-kquant-00002-of-00002.gguf": (_D, 2_000, False),+ },+ ("dflash-kquant-00001-of-00002.gguf", "dflash-kquant-00001-of-00002.gguf"),+ (1, 1),+ id = "local-split-incomplete",+ ),+ pytest.param(+ "model-B-Q4_K_M.gguf",+ {+ "model-A-Q4_K_M.gguf": (_L, 100_000, True),+ "model-B-Q4_K_M.gguf": (_L, 100_000, True),+ "dflash-model-A-Q8_0.gguf": (_D, 2_000, True),+ "dflash-kquant.gguf": (_D, 2_000, True),+ },+ ("dflash-kquant.gguf", "dflash-kquant.gguf"),+ (0, 0),+ id = "mixed-families",+ ),+ pytest.param(+ "model-Q4_K_M.gguf",+ {+ "model-Q4_K_M.gguf": (_L, 4_000, True),+ "dflash-model-BF16.gguf": (_D, 40_000, True),+ "dflash-kquant.gguf": (_D, 500, True),+ },+ # An unanswered size request keeps every candidate, as before.+ ("dflash-kquant.gguf", "dflash-model-BF16.gguf"),+ (1, 1),+ id = "oversize",+ ),+ pytest.param(+ "model-Q4_K_M.gguf",+ {+ "model-Q4_K_M.gguf": (_L, 10_000, True),+ "dflash-big-00001-of-00002.gguf": (_D, 6_000, True),+ "dflash-big-00002-of-00002.gguf": (_D, 6_000, True),+ },+ (None, "dflash-big-00001-of-00002.gguf"),+ (1, 1),+ id = "split-set-over-the-bound",+ ),+ pytest.param(+ "model-Q4_K_M.gguf",+ {+ "model-Q4_K_M.gguf": (_L, 100_000, True),+ "dflash-model-Q4_K_M.gguf": (_L, 2_000, True),+ "dflash-kquant.gguf": (_D, 2_000, True),+ },+ ("dflash-kquant.gguf", "dflash-kquant.gguf"),+ (0, 0),+ id = "impostor",+ ),+ pytest.param(+ "model-Q4_K_M.gguf",+ {+ "model-Q4_K_M.gguf": (_L, 100_000, True),+ "dflash-kquant.gguf": (_D, 2_000, False),+ },+ (None, None),+ (1, 1),+ id = "not-cached",+ ),+ ],+)+def test_download_dflash_local_sizes_keep_the_hub_sized_picks(+ monkeypatch, tmp_path, weight, files, expected, expected_calls, hub_sizes+):+ """Picks match the Hub-sized scan; only a bound dropping an on-disk file asks, at most once."""+ got, hub_calls = _dflash_snapshot_load(+ monkeypatch, tmp_path, files, weight, hub_sizes = hub_sizes+ )+ assert got == expected[0 if hub_sizes else 1]+ assert hub_calls == expected_calls[0 if hub_sizes else 1]