Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reuse unchanged files from an older snapshot instead of re-downloading them (#11796)

Daniel Han5da3bb2b5cbb
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index e26d1dcbbd0..b8b1e67575f 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -3661,6 +3661,14 @@ class DiffusionBackend:
if not missing:
return
+ # Staged but not counted: the worker links unchanged files from an older snapshot (hub.utils.snapshot_reuse).
+ reusable = self._reusable_from_older_snapshot(
+ repo,
+ [n for n in missing if where.get(n) is None],
+ revision,
+ declared_sizes,
+ hf_token,
+ )
if checkpoint:
missing_checkpoints.add(repo)
for entry in entries:
@@ -3669,7 +3677,9 @@ class DiffusionBackend:
newly_missing = [n for n in missing if n not in entry["files"]]
added = [n for n in scope if n not in entry["files"]]
entry["files"].extend(added)
- entry["bytes"] += int(sum(declared_sizes.get(n, 0) for n in newly_missing))
+ entry["bytes"] += int(
+ sum(declared_sizes.get(n, 0) for n in newly_missing if n not in reusable)
+ )
entry["gguf_filename"] = scoped_gguf[repo]
entry["checkpoint"] = repo in missing_checkpoints
return
@@ -3680,7 +3690,9 @@ class DiffusionBackend:
# are cheap no-op hf_hub_download calls; only the genuinely missing subset contributes to
# bytes/preflight below.
"files": list(scope),
- "bytes": int(sum(declared_sizes.get(name, 0) for name in missing)),
+ "bytes": int(
+ sum(declared_sizes.get(name, 0) for name in missing if name not in reusable)
+ ),
"gguf_filename": scoped_gguf[repo],
"checkpoint": repo in missing_checkpoints,
}
@@ -3801,6 +3813,44 @@ class DiffusionBackend:
return set()
return wanted if _hits(roots[1]) == wanted else set()
+ @staticmethod
+ def _reusable_from_older_snapshot(
+ repo_id: str,
+ names: list[str],
+ revision: Optional[str],
+ declared_sizes: dict[str, int],
+ hf_token: Optional[str],
+ ) -> set[str]:
+ """Missing files an older snapshot holds with the same content; never hashes on the request path."""
+ if not names:
+ return set()
+ try:
+ from hub.utils.snapshot_reuse import (
+ cached_ref_commit,
+ hub_remote_digests,
+ reusable_paths,
+ )
+
+ # Unpinned entries were probed against refs/main; the worker fetches the Hub head.
+ digest_revision = None if revision else "main"
+ revision = revision or cached_ref_commit("model", repo_id, hub_cache_dir())
+ if not revision:
+ return set()
+ return reusable_paths(
+ "model",
+ repo_id,
+ revision,
+ {name: int(declared_sizes.get(name) or 0) for name in names},
+ hub_cache = hub_cache_dir(),
+ # False is the managed-account anonymous sentinel; None would send the installation token.
+ remote_digests = hub_remote_digests(
+ "model", repo_id, hf_token if hf_token is False else (hf_token or None)
+ ),
+ digest_revision = digest_revision,
+ )
+ except Exception: # noqa: BLE001 -- counting the bytes is the conservative answer
+ return set()
+
@staticmethod
def _current_sha(repo_id: str, hf_token: Optional[str]) -> Optional[str]:
"""The current Hub commit for a mirror repo, or None when metadata is unavailable."""
diff --git a/studio/backend/hub/services/datasets/cache_inventory.py b/studio/backend/hub/services/datasets/cache_inventory.py
index d5961015026..aff666b4b97 100644
--- a/studio/backend/hub/services/datasets/cache_inventory.py
+++ b/studio/backend/hub/services/datasets/cache_inventory.py
@@ -483,16 +483,12 @@ def _scan_hf_dataset_caches() -> list[dict]:
# str(...) guards against the library switching repo_type to an Enum.
if str(repo_info.repo_type) != "dataset":
continue
- total_size = int(getattr(repo_info, "size_on_disk", 0) or 0)
- if total_size == 0:
- unique_blobs: dict[str, int] = {}
- for rev in repo_info.revisions:
- rev_id = getattr(rev, "commit_hash", None) or str(id(rev))
- for f in rev.files:
- blob_path = getattr(f, "blob_path", None)
- key = str(blob_path) if blob_path else f"{rev_id}:{f.file_name}"
- unique_blobs[key] = int(f.size_on_disk or 0)
- total_size = sum(unique_blobs.values())
+ from hub.services.models.cache_inventory import repo_unique_size_bytes
+
+ # repo_info.size_on_disk counts a hard-linked revision (snapshot reuse) twice.
+ total_size = repo_unique_size_bytes(repo_info) or int(
+ getattr(repo_info, "size_on_disk", 0) or 0
+ )
key = repo_info.repo_id.lower()
existing = seen_lower.get(key)
cache_dir = Path(repo_info.repo_path)
diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.py
index e6c2022ebb9..bb06bf67c50 100644
--- a/studio/backend/hub/services/models/cache_inventory.py
+++ b/studio/backend/hub/services/models/cache_inventory.py
@@ -206,6 +206,28 @@ def all_hf_cache_scans():
return hf_cache_scan.all_hf_cache_scans()
+def _blob_key(file_obj, fallback: str) -> str:
+ """Key existing files by inode: no-symlink snapshot paths are their own blob_path, and reuse hard links them."""
+ blob_path = getattr(file_obj, "blob_path", None)
+ if not blob_path:
+ return fallback
+ try:
+ st = Path(blob_path).stat()
+ except OSError:
+ return str(blob_path)
+ return f"inode:{st.st_dev}:{st.st_ino}" if st.st_ino else str(blob_path)
+
+
+def repo_unique_size_bytes(repo_info) -> int:
+ """Every file across revisions, each stored copy counted once (see ``_blob_key``)."""
+ unique: dict[str, int] = {}
+ for revision in repo_info.revisions:
+ rev_id = getattr(revision, "commit_hash", None) or str(id(revision))
+ for f in cached_repo_files(revision):
+ unique[_blob_key(f, f"{rev_id}:{f.file_name}")] = int(f.size_on_disk or 0)
+ return sum(unique.values())
+
+
def _repo_gguf_size_bytes(repo_info) -> int:
"""Sum primary GGUF blob sizes across revisions, deduped by blob path (HF hardlinks shared blobs); mmproj is excluded so a vision-adapter-only repo isn't classed as GGUF."""
unique_blobs: dict[str, int] = {}
@@ -215,12 +237,7 @@ def _repo_gguf_size_bytes(repo_info) -> int:
# Snapshot-relative: only the directory marks an MTP/ drafter as a companion.
name = _cached_repo_file_name(f)
if _is_main_gguf_filename(name):
- blob_path = getattr(f, "blob_path", None)
- size = f.size_on_disk or 0
- if blob_path:
- unique_blobs[str(blob_path)] = size
- else:
- unique_blobs[f"{rev_id}:{name}"] = size
+ unique_blobs[_blob_key(f, f"{rev_id}:{name}")] = f.size_on_disk or 0
return sum(unique_blobs.values())
@@ -834,9 +851,8 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:
def _record_blob(
target: dict[str, tuple[int, float]], file_obj, rev_id: str, file_name: str
) -> None:
- blob_path = getattr(file_obj, "blob_path", None)
size = int(file_obj.size_on_disk or 0)
- key = str(blob_path) if blob_path else f"{rev_id}:{file_name}"
+ key = _blob_key(file_obj, f"{rev_id}:{file_name}")
value = (size, _blob_mtime(file_obj))
target[key] = value
all_weight_blobs[key] = value
diff --git a/studio/backend/hub/services/models/companion_cleanup.py b/studio/backend/hub/services/models/companion_cleanup.py
index d6144320670..f5b59257ac3 100644
--- a/studio/backend/hub/services/models/companion_cleanup.py
+++ b/studio/backend/hub/services/models/companion_cleanup.py
@@ -45,9 +45,9 @@ def _repo_blob_bytes(repo_info, *, only = None) -> int:
pass
if only is not None and not only(name):
continue
- blob_path = getattr(f, "blob_path", None)
- size = int(getattr(f, "size_on_disk", 0) or 0)
- unique[str(blob_path) if blob_path else f"{rev_id}:{name}"] = size
+ from hub.services.models.cache_inventory import _blob_key
+
+ unique[_blob_key(f, f"{rev_id}:{name}")] = int(getattr(f, "size_on_disk", 0) or 0)
return sum(unique.values())
diff --git a/studio/backend/hub/tests/test_snapshot_reuse.py b/studio/backend/hub/tests/test_snapshot_reuse.py
new file mode 100644
index 00000000000..8cfe48e03b2
--- /dev/null
+++ b/studio/backend/hub/tests/test_snapshot_reuse.py
@@ -0,0 +1,808 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A new Hub commit must not re-download files it did not change (no-symlink cache)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import textwrap
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.parse import parse_qs, unquote, urlsplit
+
+import pytest
+
+_BACKEND_DIR = Path(__file__).resolve().parents[2]
+if str(_BACKEND_DIR) not in sys.path:
+ sys.path.insert(0, str(_BACKEND_DIR))
+
+from hub.utils import snapshot_reuse
+from hub.utils.download_manifest import ExpectedFile
+
+REPO = "Org/Model"
+OLD = "1" * 40
+NEW = "2" * 40
+
+
+def _sha256(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+def _git_sha1(data: bytes) -> str:
+ return hashlib.sha1(b"blob %d\x00" % len(data) + data).hexdigest()
+
+
+def _blob(seed: int, size: int) -> bytes:
+ out = bytearray()
+ counter = 0
+ while len(out) < size:
+ out += hashlib.sha256(b"%d-%d" % (seed, counter)).digest()
+ counter += 1
+ return bytes(out[:size])
+
+
+def _copy_layout(
+ tmp_path: Path,
+ files: dict[str, bytes],
+ commit: str = OLD,
+) -> Path:
+ repo_dir = tmp_path / "hub" / "models--Org--Model"
+ snap = repo_dir / "snapshots" / commit
+ for rel, data in files.items():
+ (snap / rel).parent.mkdir(parents = True, exist_ok = True)
+ (snap / rel).write_bytes(data)
+ (repo_dir / "blobs").mkdir(parents = True, exist_ok = True)
+ (repo_dir / "refs").mkdir(parents = True, exist_ok = True)
+ (repo_dir / "refs" / "main").write_text(commit)
+ return repo_dir
+
+
+def _reuse(tmp_path, expected, **kwargs):
+ return snapshot_reuse.reuse_unchanged_snapshot_files(
+ "model", REPO, NEW, expected, hub_cache = tmp_path / "hub", **kwargs
+ )
+
+
+def test_unchanged_file_is_linked_and_a_same_size_changed_file_is_not(tmp_path):
+ same = _blob(1, 4096)
+ old_changed, new_changed = _blob(2, 4096), _blob(3, 4096)
+ repo_dir = _copy_layout(
+ tmp_path, {"model.safetensors": same, "vae/vae.safetensors": old_changed}
+ )
+
+ result = _reuse(
+ tmp_path,
+ [
+ ExpectedFile("model.safetensors", len(same), _sha256(same)),
+ ExpectedFile("vae/vae.safetensors", len(new_changed), _sha256(new_changed)),
+ ],
+ )
+
+ assert result.reused == ("model.safetensors",)
+ assert result.reused_bytes == len(same)
+ target = repo_dir / "snapshots" / NEW / "model.safetensors"
+ assert target.read_bytes() == same
+ assert not target.is_symlink()
+ assert (
+ os.stat(target).st_ino == os.stat(repo_dir / "snapshots" / OLD / "model.safetensors").st_ino
+ )
+ assert not (repo_dir / "snapshots" / NEW / "vae" / "vae.safetensors").exists()
+
+
+def test_symlink_layout_is_left_to_huggingface_hub(tmp_path):
+ data = _blob(4, 2048)
+ digest = _sha256(data)
+ repo_dir = tmp_path / "hub" / "models--Org--Model"
+ (repo_dir / "blobs").mkdir(parents = True)
+ (repo_dir / "blobs" / digest).write_bytes(data)
+ (repo_dir / "snapshots" / OLD).mkdir(parents = True)
+ try:
+ (repo_dir / "snapshots" / OLD / "model.safetensors").symlink_to(
+ Path("..", "..", "blobs", digest)
+ )
+ except OSError:
+ pytest.skip("symlinks unavailable")
+
+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), digest)])
+
+ assert result.reused == ()
+ assert not (repo_dir / "snapshots" / NEW).exists()
+
+
+def test_a_symlinked_candidate_whose_blob_is_gone_is_not_used(tmp_path):
+ data = _blob(5, 2048)
+ repo_dir = _copy_layout(tmp_path, {})
+ (repo_dir / "snapshots" / OLD).mkdir(parents = True)
+ try:
+ (repo_dir / "snapshots" / OLD / "model.safetensors").symlink_to(tmp_path / "elsewhere.bin")
+ except OSError:
+ pytest.skip("symlinks unavailable")
+ (tmp_path / "elsewhere.bin").write_bytes(data)
+
+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), _sha256(data))])
+
+ assert result.reused == ()
+
+
+def test_hub_digest_for_the_old_commit_skips_hashing(tmp_path, monkeypatch):
+ data = _blob(6, 8192)
+ digest = _sha256(data)
+ _copy_layout(tmp_path, {"model.safetensors": data})
+ calls = []
+
+ def remote(commit, paths):
+ calls.append((commit, tuple(paths)))
+ return {p: digest for p in paths}
+
+ monkeypatch.setattr(
+ snapshot_reuse,
+ "file_digest",
+ lambda *a, **k: pytest.fail("hashed although the Hub vouched for the old commit"),
+ )
+ result = _reuse(
+ tmp_path, [ExpectedFile("model.safetensors", len(data), digest)], remote_digests = remote
+ )
+
+ assert result.reused == ("model.safetensors",)
+ assert result.hashed_bytes == 0
+ assert calls == [(OLD, ("model.safetensors",))]
+
+
+def test_hub_digest_mismatch_falls_back_to_hashing_and_still_refuses_changed_bytes(tmp_path):
+ old, new = _blob(7, 1024), _blob(8, 1024)
+ _copy_layout(tmp_path, {"model.safetensors": old})
+
+ result = _reuse(
+ tmp_path,
+ [ExpectedFile("model.safetensors", len(new), _sha256(new))],
+ remote_digests = lambda commit, paths: {p: _sha256(old) for p in paths},
+ )
+
+ assert result.reused == ()
+ assert result.hashed_bytes == len(old)
+
+
+def test_unreachable_old_commit_is_verified_by_hashing(tmp_path):
+ data = _blob(9, 4096)
+ _copy_layout(tmp_path, {"model.safetensors": data})
+
+ def remote(commit, paths):
+ raise OSError("404 revision not found")
+
+ result = _reuse(
+ tmp_path,
+ [ExpectedFile("model.safetensors", len(data), _sha256(data))],
+ remote_digests = remote,
+ )
+
+ assert result.reused == ("model.safetensors",)
+ assert result.hashed_bytes == len(data)
+
+
+def _linked_snapshots(tmp_path: Path, data: bytes, n: int) -> list[str]:
+ commits = [f"{i + 3:x}" * 40 for i in range(n)]
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data}, commit = commits[0])
+ for commit in commits[1:]:
+ (repo_dir / "snapshots" / commit).mkdir(parents = True)
+ os.link(
+ repo_dir / "snapshots" / commits[0] / "model.safetensors",
+ repo_dir / "snapshots" / commit / "model.safetensors",
+ )
+ return commits
+
+
+def test_one_file_linked_into_many_snapshots_is_hashed_once(tmp_path):
+ old, new = _blob(20, 8192), _blob(21, 8192)
+ _linked_snapshots(tmp_path, old, 5)
+
+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(new), _sha256(new))])
+
+ assert result.reused == ()
+ assert result.hashed_bytes == len(old)
+
+
+def test_hub_digests_are_asked_newest_first_and_only_until_matched(tmp_path):
+ data = _blob(22, 8192)
+ commits = []
+ for i in range(4):
+ commit = f"{i + 3:x}" * 40
+ commits.append(commit)
+ _copy_layout(tmp_path, {"model.safetensors": data}, commit = commit)
+ calls = []
+
+ def remote(commit, paths):
+ calls.append(commit)
+ return {p: _sha256(data) for p in paths}
+
+ found = snapshot_reuse.reusable_paths(
+ "model",
+ REPO,
+ NEW,
+ {"model.safetensors": len(data)},
+ hub_cache = tmp_path / "hub",
+ remote_digests = remote,
+ )
+
+ assert found == {"model.safetensors"}
+ assert len(calls) == 2
+ assert calls[0] == NEW and calls[1] in commits
+
+
+def test_a_copy_cut_short_by_a_cancel_is_removed_by_the_next_reuse(tmp_path):
+ data = _blob(23, 8192)
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})
+ target = repo_dir / "snapshots" / NEW
+ target.mkdir(parents = True)
+ (target / ".model.safetensors.reuse-99999-deadbeef").write_bytes(data[:100])
+
+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), _sha256(data))])
+
+ assert result.reused == ("model.safetensors",)
+ assert sorted(p.name for p in target.iterdir()) == ["model.safetensors"]
+
+
+def test_a_retry_does_not_preflight_files_an_earlier_attempt_already_placed(tmp_path, monkeypatch):
+ # Preflight must not count a file an earlier cancelled attempt already placed.
+ from huggingface_hub import constants
+
+ from hub.workers import hf_download
+
+ data, missing = _blob(24, 8192), _blob(25, 4096)
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})
+ target = repo_dir / "snapshots" / NEW
+ target.mkdir(parents = True)
+ os.link(repo_dir / "snapshots" / OLD / "model.safetensors", target / "model.safetensors")
+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path / "hub"))
+ monkeypatch.setattr(hf_download, "_protected_blob_hashes", lambda: frozenset())
+ expected = [
+ ExpectedFile("model.safetensors", len(data), _sha256(data)),
+ ExpectedFile("vae/vae.safetensors", len(missing), _sha256(missing)),
+ ]
+
+ left = hf_download._reuse_unchanged_files("model", REPO, NEW, expected, None)
+
+ assert [f.path for f in left] == ["vae/vae.safetensors"]
+ assert hf_download._reuse_unchanged_files("model", REPO, None, expected, None) == expected
+
+
+def test_size_mismatch_is_rejected_without_reading_the_file(tmp_path, monkeypatch):
+ _copy_layout(tmp_path, {"model.safetensors": _blob(10, 1000)})
+ monkeypatch.setattr(
+ snapshot_reuse, "file_digest", lambda *a, **k: pytest.fail("hashed a wrong-size file")
+ )
+
+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", 1001, "a" * 64)])
+
+ assert result.reused == ()
+
+
+def test_cached_digest_feeds_the_plan_but_the_worker_rehashes(tmp_path, monkeypatch):
+ data = _blob(11, 4096)
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})
+ expected = [ExpectedFile("model.safetensors", len(data), _sha256(data))]
+
+ first = snapshot_reuse.find_reusable_copies(
+ repo_dir, NEW, {"model.safetensors": (len(data), _sha256(data))}
+ )
+ assert first[1] == len(data)
+ estimate = snapshot_reuse.find_reusable_copies(
+ repo_dir, NEW, {"model.safetensors": (len(data), _sha256(data))}, allow_hashing = False
+ )
+ assert estimate == ({"model.safetensors": first[0]["model.safetensors"]}, 0)
+
+ result = _reuse(tmp_path, expected)
+
+ assert result.reused == ("model.safetensors",)
+ assert result.hashed_bytes == len(data)
+
+
+def test_a_stale_cached_digest_does_not_carry_changed_bytes_forward(tmp_path):
+ """Same size, same mtime (restored, or FAT/exFAT 2 s granularity), different bytes."""
+ good, bad = _blob(12, 4096), _blob(13, 4096)
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": bad})
+ candidate = repo_dir / "snapshots" / OLD / "model.safetensors"
+ key = snapshot_reuse._digest_cache_key(candidate, "sha256", os.stat(candidate))
+ snapshot_reuse._remember_digests({key: _sha256(good)})
+
+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(good), _sha256(good))])
+
+ assert result.reused == ()
+ assert not (repo_dir / "snapshots" / NEW / "model.safetensors").exists()
+
+
+def test_small_non_lfs_file_is_verified_by_git_blob_id(tmp_path):
+ config = b'{"hidden_size": 8}\n'
+ _copy_layout(tmp_path, {"config.json": config})
+
+ result = _reuse(tmp_path, [ExpectedFile("config.json", len(config), _git_sha1(config))])
+
+ assert result.reused == ("config.json",)
+
+
+def test_copy_when_hard_links_are_unavailable(tmp_path, monkeypatch):
+ data = _blob(12, 4096)
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})
+
+ def no_link(*a, **k):
+ raise OSError("hard links not supported")
+
+ monkeypatch.setattr(snapshot_reuse.os, "link", no_link)
+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), _sha256(data))])
+
+ assert result.copied == 1 and result.linked == 0
+ target = repo_dir / "snapshots" / NEW / "model.safetensors"
+ assert target.read_bytes() == data
+ assert os.stat(target).st_nlink == 1
+
+
+def test_existing_target_and_existing_blob_are_left_alone(tmp_path):
+ data = _blob(13, 2048)
+ other = _blob(14, 2048)
+ repo_dir = _copy_layout(tmp_path, {"a.bin": data, "b.bin": other})
+ (repo_dir / "snapshots" / NEW).mkdir(parents = True)
+ (repo_dir / "snapshots" / NEW / "a.bin").write_bytes(b"already here")
+ (repo_dir / "blobs" / _sha256(other)).write_bytes(other)
+
+ result = _reuse(
+ tmp_path,
+ [
+ ExpectedFile("a.bin", len(data), _sha256(data)),
+ ExpectedFile("b.bin", len(other), _sha256(other)),
+ ],
+ )
+
+ assert result.reused == ()
+ assert (repo_dir / "snapshots" / NEW / "a.bin").read_bytes() == b"already here"
+ assert not (repo_dir / "snapshots" / NEW / "b.bin").exists()
+
+
+def test_superseded_partial_is_removed_unless_protected(tmp_path):
+ data = _blob(15, 4096)
+ digest = _sha256(data)
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})
+ partial = repo_dir / "blobs" / f"{digest}.incomplete"
+ partial.write_bytes(data[:100])
+
+ protected = _reuse(
+ tmp_path,
+ [ExpectedFile("model.safetensors", len(data), digest)],
+ protected_blob_hashes = frozenset({digest}),
+ )
+ assert protected.reused == ()
+ assert not (repo_dir / "snapshots" / NEW / "model.safetensors").exists()
+ assert partial.exists()
+
+ _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), digest)])
+ assert (repo_dir / "snapshots" / NEW / "model.safetensors").exists()
+ assert not partial.exists()
+
+
+def test_unsafe_paths_and_unknown_digests_are_ignored(tmp_path):
+ data = _blob(16, 512)
+ _copy_layout(tmp_path, {"model.safetensors": data})
+
+ result = _reuse(
+ tmp_path,
+ [
+ ExpectedFile("../escape.bin", len(data), _sha256(data)),
+ ExpectedFile("model.safetensors", len(data), None),
+ ExpectedFile("model.safetensors", len(data), "not-a-digest"),
+ ],
+ )
+
+ assert result.reused == ()
+
+
+def test_reusable_paths_is_read_only_and_never_hashes(tmp_path, monkeypatch):
+ same, changed = _blob(17, 2048), _blob(18, 2048)
+ repo_dir = _copy_layout(tmp_path, {"te.safetensors": same, "vae.safetensors": changed})
+ monkeypatch.setattr(snapshot_reuse, "file_digest", lambda *a, **k: pytest.fail("plan hashed"))
+ digests = {
+ OLD: {"te.safetensors": _sha256(same), "vae.safetensors": _sha256(changed)},
+ NEW: {"te.safetensors": _sha256(same), "vae.safetensors": _sha256(b"new" + changed[3:])},
+ }
+
+ found = snapshot_reuse.reusable_paths(
+ "model",
+ REPO,
+ NEW,
+ {"te.safetensors": len(same), "vae.safetensors": len(changed), "absent.bin": 5},
+ hub_cache = tmp_path / "hub",
+ remote_digests = lambda commit, paths: {p: digests[commit][p] for p in paths},
+ )
+
+ assert found == {"te.safetensors"}
+ assert not (repo_dir / "snapshots" / NEW).exists()
+
+
+def test_reusable_paths_makes_no_request_without_a_local_candidate(tmp_path):
+ _copy_layout(tmp_path, {"other.bin": b"x"})
+
+ found = snapshot_reuse.reusable_paths(
+ "model",
+ REPO,
+ NEW,
+ {"te.safetensors": 10},
+ hub_cache = tmp_path / "hub",
+ remote_digests = lambda commit, paths: pytest.fail("network without a candidate"),
+ )
+
+ assert found == set()
+
+
+_UNCHANGED = _blob(100, 3 * 1024 * 1024)
+_VAE = _blob(101, 256 * 1024)
+_CHANGED_OLD = _blob(102, 512 * 1024)
+_CHANGED_NEW = _blob(103, 512 * 1024)
+_REVISIONS = {
+ OLD: {
+ "README.md": b"# v1\n",
+ "text_encoder.safetensors": _UNCHANGED,
+ "vae/vae.safetensors": _VAE,
+ "changed.safetensors": _CHANGED_OLD,
+ },
+ NEW: {
+ "README.md": b"# v2, a card edit\n",
+ "text_encoder.safetensors": _UNCHANGED,
+ "vae/vae.safetensors": _VAE,
+ "changed.safetensors": _CHANGED_NEW,
+ },
+}
+_LFS_MIN = 1024
+
+
+class _FakeHub:
+ def __init__(self):
+ self.head = OLD
+ self.downloaded: dict[tuple[str, str], int] = {}
+ self.lock = threading.Lock()
+ hub = self
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def _json(self, payload):
+ body = json.dumps(payload).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def _commit(self, rev):
+ rev = unquote(rev)
+ return hub.head if rev == "main" else rev if rev in _REVISIONS else None
+
+ def _resolve(self, head_only):
+ parts = urlsplit(self.path).path.split("/")
+ commit = self._commit(parts[4])
+ rel = unquote("/".join(parts[5:]))
+ data = _REVISIONS.get(commit, {}).get(rel)
+ if data is None:
+ self.send_response(404)
+ self.send_header("X-Error-Code", "EntryNotFound")
+ self.end_headers()
+ return
+ lfs = len(data) >= _LFS_MIN
+ self.send_response(200)
+ self.send_header("X-Repo-Commit", commit)
+ self.send_header("ETag", f'"{_git_sha1(data)}"')
+ if lfs:
+ self.send_header("X-Linked-Etag", f'"{_sha256(data)}"')
+ self.send_header("X-Linked-Size", str(len(data)))
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ if not head_only:
+ self.wfile.write(data)
+ with hub.lock:
+ key = (commit, rel)
+ hub.downloaded[key] = hub.downloaded.get(key, 0) + len(data)
+
+ def do_HEAD(self):
+ self._resolve(head_only = True)
+
+ def do_GET(self):
+ path = urlsplit(self.path).path
+ if path.startswith(f"/api/models/{REPO}"):
+ rest = path[len(f"/api/models/{REPO}") :]
+ if rest in ("", "/"):
+ return self._json(_model_info(hub.head))
+ if rest.startswith("/revision/"):
+ commit = self._commit(rest[len("/revision/") :])
+ return self._json(_model_info(commit))
+ if rest.startswith("/tree/"):
+ commit = self._commit(rest[len("/tree/") :].split("/")[0])
+ return self._json(
+ [_tree_entry(p, d) for p, d in _REVISIONS[commit].items()]
+ )
+ if "/resolve/" in path:
+ return self._resolve(head_only = False)
+ self.send_response(404)
+ self.end_headers()
+
+ def do_POST(self):
+ path = urlsplit(self.path).path
+ prefix = f"/api/models/{REPO}/paths-info/"
+ if not path.startswith(prefix):
+ self.send_response(404)
+ self.end_headers()
+ return
+ commit = self._commit(path[len(prefix) :])
+ length = int(self.headers.get("Content-Length") or 0)
+ form = parse_qs(self.rfile.read(length).decode())
+ files = _REVISIONS.get(commit, {})
+ self._json([_tree_entry(p, files[p]) for p in form.get("paths", []) if p in files])
+
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.url = f"http://127.0.0.1:{self.server.server_address[1]}"
+ threading.Thread(target = self.server.serve_forever, daemon = True).start()
+
+ def close(self):
+ self.server.shutdown()
+
+
+def _tree_entry(path, data):
+ entry = {"type": "file", "path": path, "size": len(data), "oid": _git_sha1(data)}
+ if len(data) >= _LFS_MIN:
+ entry["lfs"] = {"oid": _sha256(data), "size": len(data), "pointerSize": 134}
+ return entry
+
+
+def _model_info(commit):
+ siblings = []
+ for path, data in _REVISIONS[commit].items():
+ sibling = {"rfilename": path, "size": len(data), "blobId": _git_sha1(data)}
+ if len(data) >= _LFS_MIN:
+ sibling["lfs"] = {"sha256": _sha256(data), "size": len(data), "pointerSize": 134}
+ siblings.append(sibling)
+ return {"id": REPO, "modelId": REPO, "sha": commit, "siblings": siblings, "private": False}
+
+
+_WORKER_SCRIPT = textwrap.dedent(
+ """
+ import sys
+ backend, mode, symlinks = sys.argv[1], sys.argv[2], sys.argv[3] == "1"
+ sys.path.insert(0, backend)
+ from huggingface_hub import file_download
+ if not symlinks:
+ # What a Windows machine without Developer Mode answers: blobs are moved into snapshots.
+ file_download.are_symlinks_supported = lambda cache_dir = None: False
+ from hub.workers import hf_download
+ if mode == "scoped":
+ hf_download._download_scoped_snapshot(
+ "Org/Model", "@diffusion",
+ ["text_encoder.safetensors", "vae/vae.safetensors", "changed.safetensors"],
+ None, "http",
+ )
+ else:
+ hf_download._download_snapshot("Org/Model", None, "http")
+ """
+)
+
+
+def fake_hub():
+ hub = _FakeHub()
+ yield hub
+ hub.close()
+
+
+def _run_worker(tmp_path, hub, mode, symlinks):
+ env = {
+ k: v
+ for k, v in os.environ.items()
+ if not k.startswith(("HF_", "HUGGING_FACE_", "UNSLOTH_PROTECTED"))
+ }
+ env.update(
+ HF_ENDPOINT = hub.url,
+ HF_HOME = str(tmp_path / "hf_home"),
+ HF_HUB_CACHE = str(tmp_path / "hub"),
+ HF_HUB_DISABLE_XET = "1",
+ HF_HUB_ENABLE_HF_TRANSFER = "0",
+ HF_HUB_DISABLE_TELEMETRY = "1",
+ HF_HUB_DISABLE_SYMLINKS_WARNING = "1",
+ HF_TOKEN = "",
+ UNSLOTH_STUDIO_HOME = os.environ.get("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio")),
+ NO_PROXY = "127.0.0.1,localhost",
+ no_proxy = "127.0.0.1,localhost",
+ )
+ proc = subprocess.run(
+ [sys.executable, "-c", _WORKER_SCRIPT, str(_BACKEND_DIR), mode, "1" if symlinks else "0"],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 300,
+ )
+ assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-3000:]
+ return proc
+
+
+def _symlinks_available(tmp_path) -> bool:
+ try:
+ (tmp_path / "probe_link").symlink_to(tmp_path)
+ return True
+ except OSError:
+ return False
+
+
[email protected]("mode", ["scoped", "snapshot"])
[email protected]("symlinks", [False, True])
+def test_new_revision_downloads_only_what_changed(tmp_path, fake_hub, mode, symlinks):
+ if symlinks and not _symlinks_available(tmp_path):
+ pytest.skip("symlinks unavailable")
+ _run_worker(tmp_path, fake_hub, mode, symlinks)
+ first = dict(fake_hub.downloaded)
+ assert first[(OLD, "text_encoder.safetensors")] == len(_UNCHANGED)
+
+ fake_hub.head = NEW
+ fake_hub.downloaded.clear()
+ proc = _run_worker(tmp_path, fake_hub, mode, symlinks)
+
+ second = fake_hub.downloaded
+ assert (NEW, "text_encoder.safetensors") not in second, proc.stderr[-2000:]
+ assert (NEW, "vae/vae.safetensors") not in second
+ assert second[(NEW, "changed.safetensors")] == len(_CHANGED_NEW)
+
+ snap = tmp_path / "hub" / "models--Org--Model" / "snapshots" / NEW
+ assert (snap / "text_encoder.safetensors").read_bytes() == _UNCHANGED
+ assert (snap / "vae" / "vae.safetensors").read_bytes() == _VAE
+ assert (snap / "changed.safetensors").read_bytes() == _CHANGED_NEW
+ old_snap = snap.parent / OLD
+ assert (old_snap / "changed.safetensors").read_bytes() == _CHANGED_OLD
+ assert (tmp_path / "hub" / "models--Org--Model" / "refs" / "main").read_text() == NEW
+ if symlinks:
+ assert (snap / "text_encoder.safetensors").is_symlink()
+ assert "Reused" not in proc.stderr
+ else:
+ assert not (snap / "text_encoder.safetensors").is_symlink()
+ assert "Reused" in proc.stderr
+
+
+def test_a_corrupted_same_size_copy_is_downloaded_again_not_carried_forward(tmp_path, fake_hub):
+ """The Hub vouching for the OLD commit's digest says nothing about the bytes on disk now."""
+ _run_worker(tmp_path, fake_hub, "scoped", symlinks = False)
+ old_encoder = (
+ tmp_path / "hub" / "models--Org--Model" / "snapshots" / OLD / "text_encoder.safetensors"
+ )
+ damaged = bytearray(old_encoder.read_bytes())
+ damaged[1000:1010] = b"\x00" * 10
+ old_encoder.write_bytes(bytes(damaged))
+
+ fake_hub.head = NEW
+ fake_hub.downloaded.clear()
+ _run_worker(tmp_path, fake_hub, "scoped", symlinks = False)
+
+ assert fake_hub.downloaded[(NEW, "text_encoder.safetensors")] == len(_UNCHANGED)
+ assert (NEW, "vae/vae.safetensors") not in fake_hub.downloaded
+ snap = old_encoder.parent.parent / NEW
+ assert (snap / "text_encoder.safetensors").read_bytes() == _UNCHANGED
+
+
+def test_a_reuse_pass_persists_its_digests_in_one_write(tmp_path, monkeypatch):
+ files = {f"shard-{i}.bin": _blob(40 + i, 2048) for i in range(5)}
+ repo_dir = _copy_layout(tmp_path, files)
+ writes = []
+ monkeypatch.setattr(
+ snapshot_reuse, "_remember_digests", lambda entries: writes.append(dict(entries))
+ )
+
+ matches, _ = snapshot_reuse.find_reusable_copies(
+ repo_dir, NEW, {p: (len(d), _sha256(d)) for p, d in files.items()}
+ )
+
+ assert set(matches) == set(files)
+ assert len(writes) == 1 and len(writes[0]) == len(files)
+
+
+def test_a_blob_a_live_peer_is_downloading_gets_no_pointer(tmp_path):
+ from filelock import FileLock
+
+ data = _blob(50, 4096)
+ digest = _sha256(data)
+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})
+ lock = tmp_path / "hub" / ".locks" / repo_dir.name / f"{digest}.lock"
+ lock.parent.mkdir(parents = True)
+ expected = [ExpectedFile("model.safetensors", len(data), digest)]
+
+ with FileLock(str(lock)):
+ held = _reuse(tmp_path, expected)
+ assert held.reused == ()
+ assert not (repo_dir / "snapshots" / NEW / "model.safetensors").exists()
+
+ assert _reuse(tmp_path, expected).reused == ("model.safetensors",)
+
+
+def test_hard_linked_revisions_are_counted_once_in_cache_usage(tmp_path):
+ from huggingface_hub import scan_cache_dir
+
+ from hub.services.models.cache_inventory import _repo_gguf_size_bytes, repo_unique_size_bytes
+
+ gguf = _blob(60, 8192)
+ repo_dir = _copy_layout(tmp_path, {"model-Q4_K_M.gguf": gguf, "README.md": b"v1"})
+ new = repo_dir / "snapshots" / NEW
+ new.mkdir(parents = True)
+ os.link(repo_dir / "snapshots" / OLD / "model-Q4_K_M.gguf", new / "model-Q4_K_M.gguf")
+ (new / "README.md").write_bytes(b"v2")
+
+ (repo,) = scan_cache_dir(tmp_path / "hub").repos
+ assert (
+ len(
+ {
+ str(f.blob_path)
+ for r in repo.revisions
+ for f in r.files
+ if f.file_name.endswith(".gguf")
+ }
+ )
+ == 2
+ )
+ assert _repo_gguf_size_bytes(repo) == len(gguf)
+ assert repo_unique_size_bytes(repo) == len(gguf) + len(b"v1") + len(b"v2")
+
+ (new / "model-Q4_K_M.gguf").unlink()
+ (new / "model-Q4_K_M.gguf").write_bytes(gguf) # a real second copy still counts twice
+ (repo,) = scan_cache_dir(tmp_path / "hub").repos
+ assert _repo_gguf_size_bytes(repo) == 2 * len(gguf)
+ assert repo_unique_size_bytes(repo) == 2 * len(gguf) + len(b"v1") + len(b"v2")
+
+
+def test_a_plan_asks_the_hub_about_at_most_a_few_old_commits(tmp_path):
+ repo_dir = tmp_path / "hub" / "models--Org--Model"
+ for i in range(6):
+ snap = repo_dir / "snapshots" / (f"{i:x}" * 40)[:40]
+ snap.mkdir(parents = True)
+ (snap / "w.safetensors").write_bytes(_blob(70 + i, 1024)) # same size, new bytes each time
+ os.utime(snap, (1_000 + i, 1_000 + i))
+ asked = []
+
+ matches, _ = snapshot_reuse.find_reusable_copies(
+ repo_dir,
+ NEW,
+ {"w.safetensors": (1024, "f" * 64)},
+ remote_digests = lambda commit, paths: asked.append(commit) or {p: "e" * 64 for p in paths},
+ allow_hashing = False,
+ )
+
+ assert matches == {}
+ assert len(asked) == snapshot_reuse._REMOTE_DIGEST_COMMITS
+
+
+def test_a_partial_is_only_removed_while_holding_the_blob_lock(tmp_path):
+ from filelock import FileLock
+
+ repo_dir = _copy_layout(tmp_path, {"x.bin": b"x"})
+ digest = "d" * 64
+ partial = repo_dir / "blobs" / f"{digest}.incomplete"
+ partial.write_bytes(b"partial")
+ lock = tmp_path / "hub" / ".locks" / repo_dir.name / f"{digest}.lock"
+ lock.parent.mkdir(parents = True)
+
+ with FileLock(str(lock)): # a peer's download in progress
+ snapshot_reuse._drop_superseded_partial(repo_dir, digest, frozenset())
+ assert partial.exists()
+ snapshot_reuse._drop_superseded_partial(repo_dir, digest, frozenset())
+ assert not partial.exists()
+
+
+def test_companion_cleanup_counts_hard_linked_revisions_once(tmp_path):
+ from huggingface_hub import scan_cache_dir
+
+ from hub.services.models.companion_cleanup import _repo_blob_bytes
+
+ weights = _blob(80, 8192)
+ repo_dir = _copy_layout(tmp_path, {"vae/vae.safetensors": weights})
+ (repo_dir / "snapshots" / NEW / "vae").mkdir(parents = True)
+ os.link(
+ repo_dir / "snapshots" / OLD / "vae" / "vae.safetensors",
+ repo_dir / "snapshots" / NEW / "vae" / "vae.safetensors",
+ )
+
+ (repo,) = scan_cache_dir(tmp_path / "hub").repos
+ assert _repo_blob_bytes(repo) == len(weights)
diff --git a/studio/backend/hub/utils/snapshot_reuse.py b/studio/backend/hub/utils/snapshot_reuse.py
new file mode 100644
index 00000000000..07691ebc896
--- /dev/null
+++ b/studio/backend/hub/utils/snapshot_reuse.py
@@ -0,0 +1,517 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Reuse unchanged files from older snapshots on a no-symlink cache, where ``file_download._create_symlink`` moves each blob into its snapshot and leaves ``blobs/`` empty."""
+
+from __future__ import annotations
+
+import glob
+import hashlib
+import json
+import os
+import re
+import shutil
+import stat as stat_module
+import threading
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Iterable, Mapping, Optional, Sequence
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_GIT_SHA1 = re.compile(r"^[0-9a-f]{40}$")
+_HASH_CHUNK = 8 * 1024 * 1024
+_DIGEST_CACHE_NAME = "file_digests.json"
+_DIGEST_CACHE_LIMIT = 2048
+_PATHS_INFO_BATCH = 100
+# Old snapshots a plan asks the Hub about; the rest stay counted (the plan is on the request path).
+_REMOTE_DIGEST_COMMITS = 3
+_digest_cache_lock = threading.Lock()
+
+RemoteDigests = Callable[[str, Sequence[str]], Mapping[str, str]]
+
+
+@dataclass(frozen = True)
+class ReuseResult:
+ reused: tuple[str, ...] = ()
+ reused_bytes: int = 0
+ linked: int = 0
+ copied: int = 0
+ hashed_bytes: int = 0
+
+
+def digest_kind(digest: Optional[str]) -> Optional[str]:
+ if not isinstance(digest, str):
+ return None
+ if _SHA256.match(digest):
+ return "sha256"
+ if _GIT_SHA1.match(digest):
+ return "git-sha1"
+ return None
+
+
+def file_digest(path: Path, kind: str) -> str:
+ if kind == "sha256":
+ h = hashlib.sha256()
+ elif kind == "git-sha1":
+ h = hashlib.sha1()
+ h.update(b"blob %d\x00" % os.stat(path).st_size)
+ else:
+ raise ValueError(f"unsupported digest kind: {kind}")
+ with open(path, "rb") as f:
+ while True:
+ chunk = f.read(_HASH_CHUNK)
+ if not chunk:
+ break
+ h.update(chunk)
+ return h.hexdigest()
+
+
+def _digest_cache_path() -> Optional[Path]:
+ try:
+ from hub.utils.state_dir import state_root
+ root = state_root(create = True)
+ except Exception: # noqa: BLE001 - the cache is an optimisation
+ return None
+ return None if root is None else root / _DIGEST_CACHE_NAME
+
+
+def _digest_cache_key(path: Path, kind: str, st: os.stat_result) -> str:
+ return "|".join(
+ (
+ kind,
+ os.path.normcase(os.path.abspath(str(path))),
+ str(st.st_size),
+ str(st.st_mtime_ns),
+ str(getattr(st, "st_ino", 0)),
+ "" if os.name == "nt" else str(st.st_ctime_ns),
+ )
+ )
+
+
+def _read_digest_cache(cache_path: Path) -> dict:
+ try:
+ with open(cache_path, "r", encoding = "utf-8") as f:
+ data = json.load(f)
+ except (OSError, ValueError):
+ return {}
+ return data if isinstance(data, dict) else {}
+
+
+def _remember_digests(entries: Mapping[str, str]) -> None:
+ cache_path = _digest_cache_path()
+ if cache_path is None or not entries:
+ return
+ with _digest_cache_lock:
+ data = _read_digest_cache(cache_path)
+ for key, digest in entries.items():
+ data.pop(key, None)
+ data[key] = digest
+ while len(data) > _DIGEST_CACHE_LIMIT:
+ data.pop(next(iter(data)))
+ tmp = cache_path.with_name(f".{cache_path.name}.tmp-{uuid.uuid4().hex[:8]}")
+ try:
+ with open(tmp, "w", encoding = "utf-8") as f:
+ json.dump(data, f)
+ os.replace(tmp, cache_path)
+ except OSError as exc:
+ logger.debug("could not persist file digest cache: %s", exc)
+ try:
+ tmp.unlink()
+ except OSError:
+ pass
+
+
+def cached_file_digest(
+ path: Path,
+ kind: str,
+ *,
+ compute: bool = True,
+ learned: Optional[dict] = None,
+) -> tuple[Optional[str], int]:
+ """(digest, bytes hashed). Stat metadata cannot prove unchanged bytes, so the persisted cache only serves ``compute=False`` estimates."""
+ try:
+ st = os.stat(path)
+ except OSError:
+ return None, 0
+ key = _digest_cache_key(path, kind, st)
+ if not compute:
+ cache_path = _digest_cache_path()
+ if cache_path is None:
+ return None, 0
+ cached = _read_digest_cache(cache_path).get(key)
+ return (cached, 0) if isinstance(cached, str) and digest_kind(cached) == kind else (None, 0)
+ try:
+ digest = file_digest(path, kind)
+ after = os.stat(path)
+ except OSError:
+ return None, 0
+ if (after.st_size, after.st_mtime_ns) != (st.st_size, st.st_mtime_ns):
+ return None, st.st_size
+ if learned is None:
+ _remember_digests({key: digest})
+ else:
+ learned[key] = digest # the caller persists a whole pass in one write
+ return digest, st.st_size
+
+
+def repo_cache_dir(
+ repo_type: str,
+ repo_id: str,
+ hub_cache: Optional[str | Path] = None,
+) -> Path:
+ from huggingface_hub import constants
+ from huggingface_hub.file_download import repo_folder_name
+
+ root = Path(hub_cache) if hub_cache else Path(constants.HF_HUB_CACHE)
+ return root / repo_folder_name(repo_id = repo_id, repo_type = repo_type)
+
+
+def cached_ref_commit(
+ repo_type: str,
+ repo_id: str,
+ hub_cache: Optional[str | Path] = None,
+ ref: str = "main",
+) -> Optional[str]:
+ try:
+ from hub.utils.download_manifest import normalized_commit_hash
+ text = (repo_cache_dir(repo_type, repo_id, hub_cache) / "refs" / ref).read_text(
+ encoding = "utf-8"
+ )
+ return normalized_commit_hash(text.strip())
+ except Exception: # noqa: BLE001 - no readable ref means nothing to compare against
+ return None
+
+
+def _is_regular_file(path: Path) -> bool:
+ try:
+ return stat_module.S_ISREG(os.lstat(path).st_mode)
+ except OSError:
+ return False
+
+
+def _other_snapshots(repo_dir: Path, commit_hash: str) -> list[Path]:
+ try:
+ snapshots = [
+ p for p in (repo_dir / "snapshots").iterdir() if p.is_dir() and p.name != commit_hash
+ ]
+ except OSError:
+ return []
+ from hub.utils.hf_cache_state import snapshot_selection_key
+ return sorted(snapshots, key = snapshot_selection_key, reverse = True)
+
+
+def _candidates(snapshots: Iterable[Path], rel_path: str, size: int) -> list[Path]:
+ found = []
+ seen: set[tuple[int, int]] = set()
+ for snap in snapshots:
+ candidate = snap / rel_path
+ # A symlink means the blob layout, which huggingface_hub already reuses.
+ if not _is_regular_file(candidate):
+ continue
+ try:
+ st = os.stat(candidate)
+ except OSError:
+ continue
+ if st.st_size != size:
+ continue
+ inode = (st.st_dev, st.st_ino)
+ if st.st_ino and inode in seen:
+ continue
+ seen.add(inode)
+ found.append(candidate)
+ return found
+
+
+def _needs_reuse(repo_dir: Path, target: Path, digest: str) -> bool:
+ if os.path.lexists(target):
+ return False
+ if os.path.exists(repo_dir / "blobs" / digest):
+ return False
+ return True
+
+
+def find_reusable_copies(
+ repo_dir: Path,
+ commit_hash: str,
+ expected: Mapping[str, tuple[int, str]],
+ *,
+ remote_digests: Optional[RemoteDigests] = None,
+ allow_hashing: bool = True,
+) -> tuple[dict[str, Path], int]:
+ snapshots = _other_snapshots(repo_dir, commit_hash)
+ if not snapshots:
+ return {}, 0
+ candidates = {
+ path: _candidates(snapshots, path, size)
+ for path, (size, digest) in expected.items()
+ if size > 0 and digest_kind(digest) is not None
+ }
+
+ matches: dict[str, Path] = {}
+ hashed = 0
+ # Plan estimate only: the worker passes no remote_digests and hashes instead.
+ if remote_digests is not None:
+ asked = 0
+ for snap in snapshots:
+ if asked >= _REMOTE_DIGEST_COMMITS:
+ break
+ here = {
+ path: candidate
+ for path, found in candidates.items()
+ if path not in matches
+ for candidate in found
+ if candidate == snap / path
+ }
+ if not here:
+ continue
+ asked += 1
+ try:
+ remote = remote_digests(snap.name, sorted(here)) or {}
+ except Exception as exc: # noqa: BLE001 - squashed history, offline, 404: hash instead
+ logger.debug("remote digests unavailable for %s: %s", snap.name, exc)
+ continue
+ for path, candidate in here.items():
+ if remote.get(path) == expected[path][1]:
+ matches[path] = candidate
+ learned: dict[str, str] = {}
+ for path, found in candidates.items():
+ if path in matches:
+ continue
+ kind = digest_kind(expected[path][1])
+ for candidate in found:
+ local, spent = cached_file_digest(
+ candidate, kind, compute = allow_hashing, learned = learned
+ )
+ hashed += spent
+ if local == expected[path][1]:
+ matches[path] = candidate
+ break
+ _remember_digests(learned)
+ return matches, hashed
+
+
+def _place(src: Path, dst: Path, size: int) -> Optional[str]:
+ dst.parent.mkdir(parents = True, exist_ok = True)
+ for stale in dst.parent.glob(f".{glob.escape(dst.name)}.reuse-*"):
+ try:
+ stale.unlink()
+ except OSError:
+ pass
+ tmp = dst.with_name(f".{dst.name}.reuse-{os.getpid()}-{uuid.uuid4().hex[:8]}")
+ how = None
+ try:
+ try:
+ os.link(src, tmp)
+ how = "link"
+ except OSError:
+ if shutil.disk_usage(dst.parent).free < size:
+ return None
+ shutil.copyfile(src, tmp)
+ how = "copy"
+ if os.path.lexists(dst):
+ return None
+ os.replace(tmp, dst)
+ return how
+ except OSError as exc:
+ logger.info("could not reuse %s for %s: %s", src, dst, exc)
+ return None
+ finally:
+ try:
+ if os.path.lexists(tmp):
+ os.unlink(tmp)
+ except OSError:
+ pass
+
+
+def _drop_superseded_partial(repo_dir: Path, digest: str, protected: frozenset[str]) -> None:
+ if digest in protected:
+ return
+ partial = repo_dir / "blobs" / f"{digest}.incomplete"
+ if not partial.is_file():
+ return
+ lock_path = repo_dir.parent / ".locks" / repo_dir.name / f"{digest}.lock"
+ try:
+ from filelock import FileLock, Timeout
+ lock_path.parent.mkdir(parents = True, exist_ok = True)
+ # Held, not probed: huggingface_hub writes partials only under this lock.
+ with FileLock(str(lock_path), timeout = 0):
+ partial.unlink(missing_ok = True)
+ except Timeout:
+ return
+ except Exception as exc: # noqa: BLE001 - no flock here (SoftFileLock territory): leave it
+ logger.debug("could not remove superseded partial %s: %s", partial, exc)
+
+
+def reuse_unchanged_snapshot_files(
+ repo_type: str,
+ repo_id: str,
+ commit_hash: Optional[str],
+ expected_files: Sequence,
+ *,
+ hub_cache: Optional[str | Path] = None,
+ remote_digests: Optional[RemoteDigests] = None,
+ allow_hashing: bool = True,
+ protected_blob_hashes: frozenset[str] = frozenset(),
+) -> ReuseResult:
+ """Never raises: anything it cannot prove is left for the normal download."""
+ try:
+ from hub.utils.download_manifest import expected_path_is_safe, normalized_commit_hash
+
+ commit = normalized_commit_hash(commit_hash)
+ if not commit:
+ return ReuseResult()
+ repo_dir = repo_cache_dir(repo_type, repo_id, hub_cache)
+ if not (repo_dir / "snapshots").is_dir():
+ return ReuseResult()
+ target_root = repo_dir / "snapshots" / commit
+ pending: dict[str, tuple[int, str]] = {}
+ for item in expected_files:
+ path = getattr(item, "path", None)
+ size = int(getattr(item, "size", 0) or 0)
+ digest = getattr(item, "sha256", None)
+ if not expected_path_is_safe(path) or size <= 0 or digest_kind(digest) is None:
+ continue
+ # huggingface_hub links a finished blob only when the pointer is absent: none under a peer.
+ if digest in protected_blob_hashes:
+ continue
+ if _needs_reuse(repo_dir, target_root / path, digest):
+ pending[path] = (size, digest)
+ if not pending:
+ return ReuseResult()
+ matches, hashed = find_reusable_copies(
+ repo_dir,
+ commit,
+ pending,
+ remote_digests = remote_digests,
+ allow_hashing = allow_hashing,
+ )
+ reused, linked, copied, reused_bytes = [], 0, 0, 0
+ from hub.utils.hf_cache_state import blob_download_lock_held
+
+ for path, src in matches.items():
+ size, digest = pending[path]
+ # A peer may have started after launch.
+ if blob_download_lock_held(repo_dir, digest):
+ continue
+ how = _place(src, target_root / path, size)
+ if how is None:
+ continue
+ reused.append(path)
+ reused_bytes += size
+ linked += how == "link"
+ copied += how == "copy"
+ _drop_superseded_partial(repo_dir, digest, protected_blob_hashes)
+ return ReuseResult(tuple(reused), reused_bytes, linked, copied, hashed)
+ except Exception as exc: # noqa: BLE001 - reuse is an optimisation, never a failure
+ logger.warning("snapshot reuse skipped for %s: %s", repo_id, exc)
+ return ReuseResult()
+
+
+def paths_in_snapshot(
+ repo_type: str,
+ repo_id: str,
+ commit_hash: Optional[str],
+ paths: Iterable[str],
+ *,
+ hub_cache: Optional[str | Path] = None,
+) -> set[str]:
+ try:
+ from hub.utils.download_manifest import expected_path_is_safe, normalized_commit_hash
+
+ commit = normalized_commit_hash(commit_hash)
+ if not commit:
+ return set()
+ root = repo_cache_dir(repo_type, repo_id, hub_cache) / "snapshots" / commit
+ return {p for p in paths if expected_path_is_safe(p) and os.path.exists(root / p)}
+ except Exception: # noqa: BLE001 - counting a present file again only makes the preflight stricter
+ return set()
+
+
+def hub_remote_digests(repo_type: str, repo_id: str, token) -> RemoteDigests:
+ def lookup(commit: str, paths: Sequence[str]) -> Mapping[str, str]:
+ from huggingface_hub import HfApi
+
+ api = HfApi(token = token)
+ paths = list(paths)
+ infos = []
+ for start in range(0, len(paths), _PATHS_INFO_BATCH):
+ infos.extend(
+ api.get_paths_info(
+ repo_id,
+ paths[start : start + _PATHS_INFO_BATCH],
+ revision = commit,
+ repo_type = repo_type,
+ )
+ )
+ out: dict[str, str] = {}
+ for info in infos:
+ lfs = getattr(info, "lfs", None)
+ digest = getattr(lfs, "sha256", None) if lfs is not None else None
+ if isinstance(lfs, dict):
+ digest = lfs.get("sha256")
+ digest = digest or getattr(info, "blob_id", None)
+ path = getattr(info, "path", None)
+ if isinstance(path, str) and isinstance(digest, str):
+ out[path] = digest
+ return out
+
+ return lookup
+
+
+def reusable_paths(
+ repo_type: str,
+ repo_id: str,
+ commit_hash: Optional[str],
+ sizes: Mapping[str, int],
+ *,
+ hub_cache: Optional[str | Path] = None,
+ remote_digests: Optional[RemoteDigests] = None,
+ allow_hashing: bool = False,
+ digest_revision: Optional[str] = None,
+) -> set[str]:
+ """Read-only plan estimate: no network unless a same-size local copy exists."""
+ try:
+ from hub.utils.download_manifest import expected_path_is_safe, normalized_commit_hash
+
+ commit = normalized_commit_hash(commit_hash)
+ if not commit or not sizes:
+ return set()
+ repo_dir = repo_cache_dir(repo_type, repo_id, hub_cache)
+ snapshots = _other_snapshots(repo_dir, commit)
+ if not snapshots:
+ return set()
+ target_root = repo_dir / "snapshots" / commit
+ local = [
+ path
+ for path, size in sizes.items()
+ if expected_path_is_safe(path)
+ and int(size or 0) > 0
+ and not os.path.lexists(target_root / path)
+ and _candidates(snapshots, path, int(size))
+ ]
+ if not local or remote_digests is None:
+ return set()
+ # digest_revision names what the worker will fetch when commit_hash is only the local ref.
+ target = remote_digests(digest_revision or commit, sorted(local)) or {}
+ expected = {
+ path: (int(sizes[path]), target[path])
+ for path in local
+ if digest_kind(target.get(path)) is not None
+ and not os.path.exists(repo_dir / "blobs" / target[path])
+ }
+ matches, _ = find_reusable_copies(
+ repo_dir,
+ commit,
+ expected,
+ remote_digests = remote_digests,
+ allow_hashing = allow_hashing,
+ )
+ return set(matches)
+ except Exception as exc: # noqa: BLE001 - a plan must never fail over an optimisation
+ logger.debug("reusable_paths skipped for %s: %s", repo_id, exc)
+ return set()
diff --git a/studio/backend/hub/workers/hf_download.py b/studio/backend/hub/workers/hf_download.py
index 9a2529886d7..e4fb595691d 100644
--- a/studio/backend/hub/workers/hf_download.py
+++ b/studio/backend/hub/workers/hf_download.py
@@ -179,9 +179,13 @@ def _retry_metadata_fetch(repo_id: str, fetch, *, label: str):
raise RuntimeError(f"{label} unavailable for {repo_id}")
+_RESOLVED_COMMITS: dict[str, str] = {}
+
+
def _model_info_with_retry(repo_id: str, hf_token: str | None):
from huggingface_hub import model_info as hf_model_info
- return _retry_metadata_fetch(
+
+ info = _retry_metadata_fetch(
repo_id,
lambda timeout: hf_model_info(
repo_id,
@@ -191,6 +195,39 @@ def _model_info_with_retry(repo_id: str, hf_token: str | None):
),
label = "Metadata",
)
+ commit = getattr(info, "sha", None)
+ if isinstance(commit, str) and commit:
+ _RESOLVED_COMMITS[repo_id] = commit
+ return info
+
+
+def _reuse_unchanged_files(
+ repo_type: RepoType, repo_id: str, commit_hash, expected_files: list, hf_token: str | None
+) -> list:
+ """Link files unchanged since an older snapshot into this commit; return the files still to download."""
+ from hub.utils.snapshot_reuse import paths_in_snapshot, reuse_unchanged_snapshot_files
+
+ if not expected_files or not commit_hash:
+ return list(expected_files)
+ result = reuse_unchanged_snapshot_files(
+ repo_type,
+ repo_id,
+ commit_hash,
+ expected_files,
+ # Always hash locally: a Hub digest proves what the old commit served, not what is on disk now.
+ protected_blob_hashes = _protected_blob_hashes(),
+ )
+ if result.reused:
+ print(
+ f"Reused {len(result.reused)} unchanged file(s) ({result.reused_bytes / 1e9:.2f} GB) "
+ f"from an older snapshot of {repo_id} instead of downloading them again.",
+ file = sys.stderr,
+ )
+ # Files an earlier attempt placed are skipped by snapshot_download and have no blob for the preflight to discount.
+ present = paths_in_snapshot(
+ repo_type, repo_id, commit_hash, [getattr(f, "path", None) for f in expected_files]
+ )
+ return [f for f in expected_files if getattr(f, "path", None) not in present]
def _dataset_info_with_retry(repo_id: str, hf_token: str | None):
@@ -541,7 +578,10 @@ def _download_snapshot(repo_id: str, hf_token: str | None, mode: str) -> None:
f"before starting {mode} download.",
file = sys.stderr,
)
- _preflight_disk_space("model", repo_id, expected_files)
+ to_download = _reuse_unchanged_files(
+ "model", repo_id, getattr(info, "sha", None), expected_files, hf_token
+ )
+ _preflight_disk_space("model", repo_id, to_download)
snapshot_path = snapshot_download(
repo_id = repo_id,
token = _hf_token_arg(hf_token),
@@ -676,7 +716,14 @@ def _download_gguf_variant(repo_id: str, variant: str, hf_token: str | None, mod
f"before starting {mode} download.",
file = sys.stderr,
)
- _preflight_disk_space("model", repo_id, expected_files)
+ to_download = (
+ expected_files
+ if metadata_unavailable
+ else _reuse_unchanged_files(
+ "model", repo_id, _RESOLVED_COMMITS.get(repo_id), expected_files, hf_token
+ )
+ )
+ _preflight_disk_space("model", repo_id, to_download)
snapshot_path = snapshot_download(
repo_id = repo_id,
token = _hf_token_arg(hf_token),
@@ -768,7 +815,10 @@ def _download_scoped_snapshot(
f"before starting {mode} download.",
file = sys.stderr,
)
- _preflight_disk_space("model", repo_id, expected_files)
+ to_download = _reuse_unchanged_files(
+ "model", repo_id, getattr(info, "sha", None), expected_files, hf_token
+ )
+ _preflight_disk_space("model", repo_id, to_download)
snapshot_path = snapshot_download(
repo_id = repo_id,
token = _hf_token_arg(hf_token),
@@ -834,7 +884,8 @@ def _download_dataset(repo_id: str, hf_token: str | None, mode: str) -> None:
f"before starting {mode} download.",
file = sys.stderr,
)
- _preflight_disk_space("dataset", repo_id, expected_files)
+ to_download = _reuse_unchanged_files("dataset", repo_id, commit_hash, expected_files, hf_token)
+ _preflight_disk_space("dataset", repo_id, to_download)
download_kwargs = {
"repo_id": repo_id,
"token": _hf_token_arg(hf_token),
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index c4ffda8abe1..a1c37c9bde6 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -4844,12 +4844,8 @@ def _repo_gguf_size_bytes(repo_info) -> int:
# Snapshot-relative: only the directory tells an MTP/ drafter from a primary quant.
name = _cached_repo_file_name(f)
if _is_main_gguf_filename(name):
- blob_path = getattr(f, "blob_path", None)
- size = f.size_on_disk or 0
- if blob_path:
- unique_blobs[str(blob_path)] = size
- else:
- unique_blobs[f"{rev_id}:{name}"] = size
+ from hub.services.models.cache_inventory import _blob_key
+ unique_blobs[_blob_key(f, f"{rev_id}:{name}")] = f.size_on_disk or 0
return sum(unique_blobs.values())
@@ -5243,9 +5239,9 @@ def cached_model_rows(cache_scans = None) -> list[dict]:
and _snapshot_can_serve_a_load(selected)
):
continue
- total_size = sum(
- (f.size_on_disk or 0) for rev in repo_info.revisions for f in rev.files
- )
+ from hub.services.models.cache_inventory import repo_unique_size_bytes
+
+ total_size = repo_unique_size_bytes(repo_info)
if total_size == 0:
continue
weight_files = [
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index d863f07ff8d..3d454279dec 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -7670,6 +7670,130 @@ def test_download_plan_omits_a_cached_gguf_but_keeps_missing_companions(monkeypa
assert plan["checkpoint_bytes"] == 7 * GB
+def test_download_plan_stages_but_does_not_count_a_file_an_older_snapshot_holds(monkeypatch):
+ """README-only commit on a no-symlink cache: the GGUF is still staged but counts no bytes."""
+ _fake_flux_hub(monkeypatch)
+ _no_cache(monkeypatch)
+ asked = []
+
+ def reusable(repo_id, names, revision, declared_sizes, hf_token):
+ asked.append((repo_id, tuple(names)))
+ return {"flux1-dev-Q4_K_M.gguf"} if repo_id == "unsloth/FLUX.1-dev-GGUF" else set()
+
+ monkeypatch.setattr(DiffusionBackend, "_reusable_from_older_snapshot", staticmethod(reusable))
+
+ plan = _flux_download_plan()
+
+ checkpoint, base = plan["entries"]
+ assert checkpoint["repo_id"] == "unsloth/FLUX.1-dev-GGUF"
+ assert checkpoint["files"] == ["flux1-dev-Q4_K_M.gguf"]
+ assert checkpoint["bytes"] == 0
+ assert checkpoint["checkpoint"] is True
+ assert base["bytes"] > 0
+ assert plan["total_bytes"] == base["bytes"]
+ assert plan["checkpoint_bytes"] == 7 * GB
+ assert ("unsloth/FLUX.1-dev-GGUF", ("flux1-dev-Q4_K_M.gguf",)) in asked
+
+
+def test_reusable_from_older_snapshot_reads_the_live_cache_without_hashing(monkeypatch, tmp_path):
+ from hub.utils import snapshot_reuse
+
+ old, new = "1" * 40, "2" * 40
+ same, changed = b"s" * 4096, b"c" * 4096
+ snap = tmp_path / "models--unsloth--Qwen-Image-2.1-FP8" / "snapshots" / old
+ (snap / "vae").mkdir(parents = True)
+ (snap / "text_encoder.safetensors").write_bytes(same)
+ (snap / "vae" / "vae.safetensors").write_bytes(changed)
+ digests = {
+ old: {"text_encoder.safetensors": "a" * 64, "vae/vae.safetensors": "b" * 64},
+ new: {"text_encoder.safetensors": "a" * 64, "vae/vae.safetensors": "c" * 64},
+ }
+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: str(tmp_path))
+ monkeypatch.setattr(
+ snapshot_reuse,
+ "hub_remote_digests",
+ lambda repo_type, repo_id, token: lambda commit, paths: {
+ p: digests[commit][p] for p in paths
+ },
+ )
+ monkeypatch.setattr(snapshot_reuse, "file_digest", lambda *a, **k: pytest.fail("plan hashed"))
+
+ found = DiffusionBackend._reusable_from_older_snapshot(
+ "unsloth/Qwen-Image-2.1-FP8",
+ ["text_encoder.safetensors", "vae/vae.safetensors"],
+ new,
+ {"text_encoder.safetensors": len(same), "vae/vae.safetensors": len(changed)},
+ None,
+ )
+
+ assert found == {"text_encoder.safetensors"}
+ assert not (snap.parent / new).exists()
+ assert (
+ DiffusionBackend._reusable_from_older_snapshot(
+ "unsloth/Qwen-Image-2.1-FP8", ["text_encoder.safetensors"], None, {}, None
+ )
+ == set()
+ )
+
+
+def test_reusable_from_older_snapshot_targets_the_main_ref_when_unpinned(monkeypatch, tmp_path):
+ from hub.utils import snapshot_reuse
+
+ old, new = "1" * 40, "2" * 40
+ encoder = b"s" * 4096
+ repo = tmp_path / "models--unsloth--Qwen-Image-2.1-FP8"
+ (repo / "snapshots" / old).mkdir(parents = True)
+ (repo / "snapshots" / new / "vae").mkdir(parents = True)
+ (repo / "snapshots" / old / "te.safetensors").write_bytes(encoder)
+ (repo / "refs").mkdir()
+ (repo / "refs" / "main").write_text(new)
+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: str(tmp_path))
+ head = {"main": "a" * 64}
+ asked = []
+
+ def digests(repo_type, repo_id, token):
+ def lookup(commit, paths):
+ asked.append(commit)
+ return {p: head.get(commit, "a" * 64) for p in paths}
+
+ return lookup
+
+ monkeypatch.setattr(snapshot_reuse, "hub_remote_digests", digests)
+
+ def reusable():
+ return DiffusionBackend._reusable_from_older_snapshot(
+ "unsloth/Qwen-Image-2.1-FP8",
+ ["te.safetensors"],
+ None,
+ {"te.safetensors": len(encoder)},
+ None,
+ )
+
+ assert reusable() == {"te.safetensors"}
+ assert asked[0] == "main" # the worker fetches the Hub head, not the cached ref
+ head["main"] = "b" * 64
+ assert reusable() == set()
+
+
+def test_reusable_from_older_snapshot_keeps_the_anonymous_token_sentinel(monkeypatch, tmp_path):
+ from hub.utils import snapshot_reuse
+
+ tokens = []
+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: str(tmp_path))
+ monkeypatch.setattr(
+ snapshot_reuse,
+ "hub_remote_digests",
+ lambda repo_type, repo_id, token: tokens.append(token) or (lambda commit, paths: {}),
+ )
+
+ for token in (False, "", None, "hf_x"):
+ DiffusionBackend._reusable_from_older_snapshot(
+ "unsloth/Qwen-Image-2.1-FP8", ["te.safetensors"], "2" * 40, {"te.safetensors": 1}, token
+ )
+
+ assert tokens == [False, None, None, "hf_x"]
+
+
def test_download_plan_is_empty_when_every_required_file_is_cached(monkeypatch):
_fake_flux_hub(monkeypatch)
_all_cached(monkeypatch)