Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read the file digest cache once per reuse estimate (#15914)

Daniel Han97570185a99e
diff --git a/studio/backend/hub/tests/test_snapshot_reuse.py b/studio/backend/hub/tests/test_snapshot_reuse.py
index f88f5eef843..7aca30c3dbc 100644
--- a/studio/backend/hub/tests/test_snapshot_reuse.py
+++ b/studio/backend/hub/tests/test_snapshot_reuse.py
@@ -805,3 +805,51 @@ def test_companion_cleanup_counts_hard_linked_revisions_once(tmp_path):
(repo,) = scan_cache_dir(tmp_path / "hub").repos
assert _repo_blob_bytes(repo) == len(weights)
+
+
+def test_estimate_reads_the_digest_cache_once_and_selects_the_same_copies(tmp_path, monkeypatch):
+ """A plan's estimate loop parses the persisted digest cache once, not once per candidate."""
+ third = "3" * 40
+ current = {f"part-{i}.safetensors": _blob(100 + i, 2048) for i in range(6)}
+ current["config.json"] = b'{"hidden_size": 8}\n'
+ changed = {name: _blob(200 + i, len(data)) for i, (name, data) in enumerate(current.items())}
+ # OLD holds changed bytes for every file; the third snapshot holds the current bytes for half.
+ repo_dir = _copy_layout(tmp_path, changed)
+ _copy_layout(
+ tmp_path,
+ {name: data for i, (name, data) in enumerate(current.items()) if i % 2 == 0},
+ commit = third,
+ )
+ expected = {
+ name: (len(data), _git_sha1(data) if name.endswith(".json") else _sha256(data))
+ for name, data in current.items()
+ }
+ hashed_matches, _ = snapshot_reuse.find_reusable_copies(repo_dir, NEW, expected)
+
+ def reference() -> dict:
+ snapshots = snapshot_reuse._other_snapshots(repo_dir, NEW)
+ out = {}
+ for path, (size, digest) in expected.items():
+ for candidate in snapshot_reuse._candidates(snapshots, path, size):
+ local, _ = snapshot_reuse.cached_file_digest(
+ candidate, snapshot_reuse.digest_kind(digest), compute = False
+ )
+ if local == digest:
+ out[path] = candidate
+ break
+ return out
+
+ real_read = snapshot_reuse._read_digest_cache
+ reads = []
+
+ def counting_read(cache_path):
+ reads.append(cache_path)
+ return real_read(cache_path)
+
+ monkeypatch.setattr(snapshot_reuse, "_read_digest_cache", counting_read)
+ estimate = snapshot_reuse.find_reusable_copies(repo_dir, NEW, expected, allow_hashing = False)
+
+ assert estimate == (hashed_matches, 0)
+ assert set(hashed_matches) == {n for i, n in enumerate(current) if i % 2 == 0}
+ assert len(reads) == 1
+ assert estimate[0] == reference()
diff --git a/studio/backend/hub/utils/snapshot_reuse.py b/studio/backend/hub/utils/snapshot_reuse.py
index 07691ebc896..abc55d1d625 100644
--- a/studio/backend/hub/utils/snapshot_reuse.py
+++ b/studio/backend/hub/utils/snapshot_reuse.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import functools
import glob
import hashlib
import json
@@ -102,6 +103,11 @@ def _read_digest_cache(cache_path: Path) -> dict:
return data if isinstance(data, dict) else {}
+def _load_digest_cache() -> dict:
+ cache_path = _digest_cache_path()
+ return {} if cache_path is None else _read_digest_cache(cache_path)
+
+
def _remember_digests(entries: Mapping[str, str]) -> None:
cache_path = _digest_cache_path()
if cache_path is None or not entries:
@@ -132,6 +138,7 @@ def cached_file_digest(
*,
compute: bool = True,
learned: Optional[dict] = None,
+ persisted: Optional[Callable[[], Mapping[str, str]]] = None,
) -> tuple[Optional[str], int]:
"""(digest, bytes hashed). Stat metadata cannot prove unchanged bytes, so the persisted cache only serves ``compute=False`` estimates."""
try:
@@ -140,10 +147,7 @@ def cached_file_digest(
return None, 0
key = _digest_cache_key(path, kind, st)
if not compute:
- cache_path = _digest_cache_path()
- if cache_path is None:
- return None, 0
- cached = _read_digest_cache(cache_path).get(key)
+ cached = (persisted or _load_digest_cache)().get(key)
return (cached, 0) if isinstance(cached, str) and digest_kind(cached) == kind else (None, 0)
try:
digest = file_digest(path, kind)
@@ -279,13 +283,19 @@ def find_reusable_copies(
if remote.get(path) == expected[path][1]:
matches[path] = candidate
learned: dict[str, str] = {}
+ persisted = functools.cache(_load_digest_cache)
+
for path, found in candidates.items():
if path in matches:
continue
kind = digest_kind(expected[path][1])
for candidate in found:
local, spent = cached_file_digest(
- candidate, kind, compute = allow_hashing, learned = learned
+ candidate,
+ kind,
+ compute = allow_hashing,
+ learned = learned,
+ persisted = persisted,
)
hashed += spent
if local == expected[path][1]: