Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reuse unchanged files from an older snapshot instead of re-downloading them (#11796)
Daniel Han5da3bb2b5cbb
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex e26d1dcbbd0..b8b1e67575f 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -3661,6 +3661,14 @@ class DiffusionBackend:if not missing:return+ # Staged but not counted: the worker links unchanged files from an older snapshot (hub.utils.snapshot_reuse).+ reusable = self._reusable_from_older_snapshot(+ repo,+ [n for n in missing if where.get(n) is None],+ revision,+ declared_sizes,+ hf_token,+ )if checkpoint:missing_checkpoints.add(repo)for entry in entries:@@ -3669,7 +3677,9 @@ class DiffusionBackend:newly_missing = [n for n in missing if n not in entry["files"]]added = [n for n in scope if n not in entry["files"]]entry["files"].extend(added)- entry["bytes"] += int(sum(declared_sizes.get(n, 0) for n in newly_missing))+ entry["bytes"] += int(+ sum(declared_sizes.get(n, 0) for n in newly_missing if n not in reusable)+ )entry["gguf_filename"] = scoped_gguf[repo]entry["checkpoint"] = repo in missing_checkpointsreturn@@ -3680,7 +3690,9 @@ class DiffusionBackend:# are cheap no-op hf_hub_download calls; only the genuinely missing subset contributes to# bytes/preflight below."files": list(scope),- "bytes": int(sum(declared_sizes.get(name, 0) for name in missing)),+ "bytes": int(+ sum(declared_sizes.get(name, 0) for name in missing if name not in reusable)+ ),"gguf_filename": scoped_gguf[repo],"checkpoint": repo in missing_checkpoints,}@@ -3801,6 +3813,44 @@ class DiffusionBackend:return set()return wanted if _hits(roots[1]) == wanted else set()+ @staticmethod+ def _reusable_from_older_snapshot(+ repo_id: str,+ names: list[str],+ revision: Optional[str],+ declared_sizes: dict[str, int],+ hf_token: Optional[str],+ ) -> set[str]:+ """Missing files an older snapshot holds with the same content; never hashes on the request path."""+ if not names:+ return set()+ try:+ from hub.utils.snapshot_reuse import (+ cached_ref_commit,+ hub_remote_digests,+ reusable_paths,+ )++ # Unpinned entries were probed against refs/main; the worker fetches the Hub head.+ digest_revision = None if revision else "main"+ revision = revision or cached_ref_commit("model", repo_id, hub_cache_dir())+ if not revision:+ return set()+ return reusable_paths(+ "model",+ repo_id,+ revision,+ {name: int(declared_sizes.get(name) or 0) for name in names},+ hub_cache = hub_cache_dir(),+ # False is the managed-account anonymous sentinel; None would send the installation token.+ remote_digests = hub_remote_digests(+ "model", repo_id, hf_token if hf_token is False else (hf_token or None)+ ),+ digest_revision = digest_revision,+ )+ except Exception: # noqa: BLE001 -- counting the bytes is the conservative answer+ return set()+@staticmethoddef _current_sha(repo_id: str, hf_token: Optional[str]) -> Optional[str]:"""The current Hub commit for a mirror repo, or None when metadata is unavailable."""diff --git a/studio/backend/hub/services/datasets/cache_inventory.py b/studio/backend/hub/services/datasets/cache_inventory.pyindex d5961015026..aff666b4b97 100644--- a/studio/backend/hub/services/datasets/cache_inventory.py+++ b/studio/backend/hub/services/datasets/cache_inventory.py@@ -483,16 +483,12 @@ def _scan_hf_dataset_caches() -> list[dict]:# str(...) guards against the library switching repo_type to an Enum.if str(repo_info.repo_type) != "dataset":continue- total_size = int(getattr(repo_info, "size_on_disk", 0) or 0)- if total_size == 0:- unique_blobs: dict[str, int] = {}- for rev in repo_info.revisions:- rev_id = getattr(rev, "commit_hash", None) or str(id(rev))- for f in rev.files:- blob_path = getattr(f, "blob_path", None)- key = str(blob_path) if blob_path else f"{rev_id}:{f.file_name}"- unique_blobs[key] = int(f.size_on_disk or 0)- total_size = sum(unique_blobs.values())+ from hub.services.models.cache_inventory import repo_unique_size_bytes++ # repo_info.size_on_disk counts a hard-linked revision (snapshot reuse) twice.+ total_size = repo_unique_size_bytes(repo_info) or int(+ getattr(repo_info, "size_on_disk", 0) or 0+ )key = repo_info.repo_id.lower()existing = seen_lower.get(key)cache_dir = Path(repo_info.repo_path)diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.pyindex e6c2022ebb9..bb06bf67c50 100644--- a/studio/backend/hub/services/models/cache_inventory.py+++ b/studio/backend/hub/services/models/cache_inventory.py@@ -206,6 +206,28 @@ def all_hf_cache_scans():return hf_cache_scan.all_hf_cache_scans()+def _blob_key(file_obj, fallback: str) -> str:+ """Key existing files by inode: no-symlink snapshot paths are their own blob_path, and reuse hard links them."""+ blob_path = getattr(file_obj, "blob_path", None)+ if not blob_path:+ return fallback+ try:+ st = Path(blob_path).stat()+ except OSError:+ return str(blob_path)+ return f"inode:{st.st_dev}:{st.st_ino}" if st.st_ino else str(blob_path)+++def repo_unique_size_bytes(repo_info) -> int:+ """Every file across revisions, each stored copy counted once (see ``_blob_key``)."""+ unique: dict[str, int] = {}+ for revision in repo_info.revisions:+ rev_id = getattr(revision, "commit_hash", None) or str(id(revision))+ for f in cached_repo_files(revision):+ unique[_blob_key(f, f"{rev_id}:{f.file_name}")] = int(f.size_on_disk or 0)+ return sum(unique.values())++def _repo_gguf_size_bytes(repo_info) -> int:"""Sum primary GGUF blob sizes across revisions, deduped by blob path (HF hardlinks shared blobs); mmproj is excluded so a vision-adapter-only repo isn't classed as GGUF."""unique_blobs: dict[str, int] = {}@@ -215,12 +237,7 @@ def _repo_gguf_size_bytes(repo_info) -> int:# Snapshot-relative: only the directory marks an MTP/ drafter as a companion.name = _cached_repo_file_name(f)if _is_main_gguf_filename(name):- blob_path = getattr(f, "blob_path", None)- size = f.size_on_disk or 0- if blob_path:- unique_blobs[str(blob_path)] = size- else:- unique_blobs[f"{rev_id}:{name}"] = size+ unique_blobs[_blob_key(f, f"{rev_id}:{name}")] = f.size_on_disk or 0return sum(unique_blobs.values())@@ -834,9 +851,8 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:def _record_blob(target: dict[str, tuple[int, float]], file_obj, rev_id: str, file_name: str) -> None:- blob_path = getattr(file_obj, "blob_path", None)size = int(file_obj.size_on_disk or 0)- key = str(blob_path) if blob_path else f"{rev_id}:{file_name}"+ key = _blob_key(file_obj, f"{rev_id}:{file_name}")value = (size, _blob_mtime(file_obj))target[key] = valueall_weight_blobs[key] = valuediff --git a/studio/backend/hub/services/models/companion_cleanup.py b/studio/backend/hub/services/models/companion_cleanup.pyindex d6144320670..f5b59257ac3 100644--- a/studio/backend/hub/services/models/companion_cleanup.py+++ b/studio/backend/hub/services/models/companion_cleanup.py@@ -45,9 +45,9 @@ def _repo_blob_bytes(repo_info, *, only = None) -> int:passif only is not None and not only(name):continue- blob_path = getattr(f, "blob_path", None)- size = int(getattr(f, "size_on_disk", 0) or 0)- unique[str(blob_path) if blob_path else f"{rev_id}:{name}"] = size+ from hub.services.models.cache_inventory import _blob_key++ unique[_blob_key(f, f"{rev_id}:{name}")] = int(getattr(f, "size_on_disk", 0) or 0)return sum(unique.values())diff --git a/studio/backend/hub/tests/test_snapshot_reuse.py b/studio/backend/hub/tests/test_snapshot_reuse.pynew file mode 100644index 00000000000..8cfe48e03b2--- /dev/null+++ b/studio/backend/hub/tests/test_snapshot_reuse.py@@ -0,0 +1,808 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A new Hub commit must not re-download files it did not change (no-symlink cache)."""++from __future__ import annotations++import hashlib+import json+import os+import subprocess+import sys+import textwrap+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path+from urllib.parse import parse_qs, unquote, urlsplit++import pytest++_BACKEND_DIR = Path(__file__).resolve().parents[2]+if str(_BACKEND_DIR) not in sys.path:+ sys.path.insert(0, str(_BACKEND_DIR))++from hub.utils import snapshot_reuse+from hub.utils.download_manifest import ExpectedFile++REPO = "Org/Model"+OLD = "1" * 40+NEW = "2" * 40+++def _sha256(data: bytes) -> str:+ return hashlib.sha256(data).hexdigest()+++def _git_sha1(data: bytes) -> str:+ return hashlib.sha1(b"blob %d\x00" % len(data) + data).hexdigest()+++def _blob(seed: int, size: int) -> bytes:+ out = bytearray()+ counter = 0+ while len(out) < size:+ out += hashlib.sha256(b"%d-%d" % (seed, counter)).digest()+ counter += 1+ return bytes(out[:size])+++def _copy_layout(+ tmp_path: Path,+ files: dict[str, bytes],+ commit: str = OLD,+) -> Path:+ repo_dir = tmp_path / "hub" / "models--Org--Model"+ snap = repo_dir / "snapshots" / commit+ for rel, data in files.items():+ (snap / rel).parent.mkdir(parents = True, exist_ok = True)+ (snap / rel).write_bytes(data)+ (repo_dir / "blobs").mkdir(parents = True, exist_ok = True)+ (repo_dir / "refs").mkdir(parents = True, exist_ok = True)+ (repo_dir / "refs" / "main").write_text(commit)+ return repo_dir+++def _reuse(tmp_path, expected, **kwargs):+ return snapshot_reuse.reuse_unchanged_snapshot_files(+ "model", REPO, NEW, expected, hub_cache = tmp_path / "hub", **kwargs+ )+++def test_unchanged_file_is_linked_and_a_same_size_changed_file_is_not(tmp_path):+ same = _blob(1, 4096)+ old_changed, new_changed = _blob(2, 4096), _blob(3, 4096)+ repo_dir = _copy_layout(+ tmp_path, {"model.safetensors": same, "vae/vae.safetensors": old_changed}+ )++ result = _reuse(+ tmp_path,+ [+ ExpectedFile("model.safetensors", len(same), _sha256(same)),+ ExpectedFile("vae/vae.safetensors", len(new_changed), _sha256(new_changed)),+ ],+ )++ assert result.reused == ("model.safetensors",)+ assert result.reused_bytes == len(same)+ target = repo_dir / "snapshots" / NEW / "model.safetensors"+ assert target.read_bytes() == same+ assert not target.is_symlink()+ assert (+ os.stat(target).st_ino == os.stat(repo_dir / "snapshots" / OLD / "model.safetensors").st_ino+ )+ assert not (repo_dir / "snapshots" / NEW / "vae" / "vae.safetensors").exists()+++def test_symlink_layout_is_left_to_huggingface_hub(tmp_path):+ data = _blob(4, 2048)+ digest = _sha256(data)+ repo_dir = tmp_path / "hub" / "models--Org--Model"+ (repo_dir / "blobs").mkdir(parents = True)+ (repo_dir / "blobs" / digest).write_bytes(data)+ (repo_dir / "snapshots" / OLD).mkdir(parents = True)+ try:+ (repo_dir / "snapshots" / OLD / "model.safetensors").symlink_to(+ Path("..", "..", "blobs", digest)+ )+ except OSError:+ pytest.skip("symlinks unavailable")++ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), digest)])++ assert result.reused == ()+ assert not (repo_dir / "snapshots" / NEW).exists()+++def test_a_symlinked_candidate_whose_blob_is_gone_is_not_used(tmp_path):+ data = _blob(5, 2048)+ repo_dir = _copy_layout(tmp_path, {})+ (repo_dir / "snapshots" / OLD).mkdir(parents = True)+ try:+ (repo_dir / "snapshots" / OLD / "model.safetensors").symlink_to(tmp_path / "elsewhere.bin")+ except OSError:+ pytest.skip("symlinks unavailable")+ (tmp_path / "elsewhere.bin").write_bytes(data)++ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), _sha256(data))])++ assert result.reused == ()+++def test_hub_digest_for_the_old_commit_skips_hashing(tmp_path, monkeypatch):+ data = _blob(6, 8192)+ digest = _sha256(data)+ _copy_layout(tmp_path, {"model.safetensors": data})+ calls = []++ def remote(commit, paths):+ calls.append((commit, tuple(paths)))+ return {p: digest for p in paths}++ monkeypatch.setattr(+ snapshot_reuse,+ "file_digest",+ lambda *a, **k: pytest.fail("hashed although the Hub vouched for the old commit"),+ )+ result = _reuse(+ tmp_path, [ExpectedFile("model.safetensors", len(data), digest)], remote_digests = remote+ )++ assert result.reused == ("model.safetensors",)+ assert result.hashed_bytes == 0+ assert calls == [(OLD, ("model.safetensors",))]+++def test_hub_digest_mismatch_falls_back_to_hashing_and_still_refuses_changed_bytes(tmp_path):+ old, new = _blob(7, 1024), _blob(8, 1024)+ _copy_layout(tmp_path, {"model.safetensors": old})++ result = _reuse(+ tmp_path,+ [ExpectedFile("model.safetensors", len(new), _sha256(new))],+ remote_digests = lambda commit, paths: {p: _sha256(old) for p in paths},+ )++ assert result.reused == ()+ assert result.hashed_bytes == len(old)+++def test_unreachable_old_commit_is_verified_by_hashing(tmp_path):+ data = _blob(9, 4096)+ _copy_layout(tmp_path, {"model.safetensors": data})++ def remote(commit, paths):+ raise OSError("404 revision not found")++ result = _reuse(+ tmp_path,+ [ExpectedFile("model.safetensors", len(data), _sha256(data))],+ remote_digests = remote,+ )++ assert result.reused == ("model.safetensors",)+ assert result.hashed_bytes == len(data)+++def _linked_snapshots(tmp_path: Path, data: bytes, n: int) -> list[str]:+ commits = [f"{i + 3:x}" * 40 for i in range(n)]+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data}, commit = commits[0])+ for commit in commits[1:]:+ (repo_dir / "snapshots" / commit).mkdir(parents = True)+ os.link(+ repo_dir / "snapshots" / commits[0] / "model.safetensors",+ repo_dir / "snapshots" / commit / "model.safetensors",+ )+ return commits+++def test_one_file_linked_into_many_snapshots_is_hashed_once(tmp_path):+ old, new = _blob(20, 8192), _blob(21, 8192)+ _linked_snapshots(tmp_path, old, 5)++ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(new), _sha256(new))])++ assert result.reused == ()+ assert result.hashed_bytes == len(old)+++def test_hub_digests_are_asked_newest_first_and_only_until_matched(tmp_path):+ data = _blob(22, 8192)+ commits = []+ for i in range(4):+ commit = f"{i + 3:x}" * 40+ commits.append(commit)+ _copy_layout(tmp_path, {"model.safetensors": data}, commit = commit)+ calls = []++ def remote(commit, paths):+ calls.append(commit)+ return {p: _sha256(data) for p in paths}++ found = snapshot_reuse.reusable_paths(+ "model",+ REPO,+ NEW,+ {"model.safetensors": len(data)},+ hub_cache = tmp_path / "hub",+ remote_digests = remote,+ )++ assert found == {"model.safetensors"}+ assert len(calls) == 2+ assert calls[0] == NEW and calls[1] in commits+++def test_a_copy_cut_short_by_a_cancel_is_removed_by_the_next_reuse(tmp_path):+ data = _blob(23, 8192)+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})+ target = repo_dir / "snapshots" / NEW+ target.mkdir(parents = True)+ (target / ".model.safetensors.reuse-99999-deadbeef").write_bytes(data[:100])++ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), _sha256(data))])++ assert result.reused == ("model.safetensors",)+ assert sorted(p.name for p in target.iterdir()) == ["model.safetensors"]+++def test_a_retry_does_not_preflight_files_an_earlier_attempt_already_placed(tmp_path, monkeypatch):+ # Preflight must not count a file an earlier cancelled attempt already placed.+ from huggingface_hub import constants++ from hub.workers import hf_download++ data, missing = _blob(24, 8192), _blob(25, 4096)+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})+ target = repo_dir / "snapshots" / NEW+ target.mkdir(parents = True)+ os.link(repo_dir / "snapshots" / OLD / "model.safetensors", target / "model.safetensors")+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path / "hub"))+ monkeypatch.setattr(hf_download, "_protected_blob_hashes", lambda: frozenset())+ expected = [+ ExpectedFile("model.safetensors", len(data), _sha256(data)),+ ExpectedFile("vae/vae.safetensors", len(missing), _sha256(missing)),+ ]++ left = hf_download._reuse_unchanged_files("model", REPO, NEW, expected, None)++ assert [f.path for f in left] == ["vae/vae.safetensors"]+ assert hf_download._reuse_unchanged_files("model", REPO, None, expected, None) == expected+++def test_size_mismatch_is_rejected_without_reading_the_file(tmp_path, monkeypatch):+ _copy_layout(tmp_path, {"model.safetensors": _blob(10, 1000)})+ monkeypatch.setattr(+ snapshot_reuse, "file_digest", lambda *a, **k: pytest.fail("hashed a wrong-size file")+ )++ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", 1001, "a" * 64)])++ assert result.reused == ()+++def test_cached_digest_feeds_the_plan_but_the_worker_rehashes(tmp_path, monkeypatch):+ data = _blob(11, 4096)+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})+ expected = [ExpectedFile("model.safetensors", len(data), _sha256(data))]++ first = snapshot_reuse.find_reusable_copies(+ repo_dir, NEW, {"model.safetensors": (len(data), _sha256(data))}+ )+ assert first[1] == len(data)+ estimate = snapshot_reuse.find_reusable_copies(+ repo_dir, NEW, {"model.safetensors": (len(data), _sha256(data))}, allow_hashing = False+ )+ assert estimate == ({"model.safetensors": first[0]["model.safetensors"]}, 0)++ result = _reuse(tmp_path, expected)++ assert result.reused == ("model.safetensors",)+ assert result.hashed_bytes == len(data)+++def test_a_stale_cached_digest_does_not_carry_changed_bytes_forward(tmp_path):+ """Same size, same mtime (restored, or FAT/exFAT 2 s granularity), different bytes."""+ good, bad = _blob(12, 4096), _blob(13, 4096)+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": bad})+ candidate = repo_dir / "snapshots" / OLD / "model.safetensors"+ key = snapshot_reuse._digest_cache_key(candidate, "sha256", os.stat(candidate))+ snapshot_reuse._remember_digests({key: _sha256(good)})++ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(good), _sha256(good))])++ assert result.reused == ()+ assert not (repo_dir / "snapshots" / NEW / "model.safetensors").exists()+++def test_small_non_lfs_file_is_verified_by_git_blob_id(tmp_path):+ config = b'{"hidden_size": 8}\n'+ _copy_layout(tmp_path, {"config.json": config})++ result = _reuse(tmp_path, [ExpectedFile("config.json", len(config), _git_sha1(config))])++ assert result.reused == ("config.json",)+++def test_copy_when_hard_links_are_unavailable(tmp_path, monkeypatch):+ data = _blob(12, 4096)+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})++ def no_link(*a, **k):+ raise OSError("hard links not supported")++ monkeypatch.setattr(snapshot_reuse.os, "link", no_link)+ result = _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), _sha256(data))])++ assert result.copied == 1 and result.linked == 0+ target = repo_dir / "snapshots" / NEW / "model.safetensors"+ assert target.read_bytes() == data+ assert os.stat(target).st_nlink == 1+++def test_existing_target_and_existing_blob_are_left_alone(tmp_path):+ data = _blob(13, 2048)+ other = _blob(14, 2048)+ repo_dir = _copy_layout(tmp_path, {"a.bin": data, "b.bin": other})+ (repo_dir / "snapshots" / NEW).mkdir(parents = True)+ (repo_dir / "snapshots" / NEW / "a.bin").write_bytes(b"already here")+ (repo_dir / "blobs" / _sha256(other)).write_bytes(other)++ result = _reuse(+ tmp_path,+ [+ ExpectedFile("a.bin", len(data), _sha256(data)),+ ExpectedFile("b.bin", len(other), _sha256(other)),+ ],+ )++ assert result.reused == ()+ assert (repo_dir / "snapshots" / NEW / "a.bin").read_bytes() == b"already here"+ assert not (repo_dir / "snapshots" / NEW / "b.bin").exists()+++def test_superseded_partial_is_removed_unless_protected(tmp_path):+ data = _blob(15, 4096)+ digest = _sha256(data)+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})+ partial = repo_dir / "blobs" / f"{digest}.incomplete"+ partial.write_bytes(data[:100])++ protected = _reuse(+ tmp_path,+ [ExpectedFile("model.safetensors", len(data), digest)],+ protected_blob_hashes = frozenset({digest}),+ )+ assert protected.reused == ()+ assert not (repo_dir / "snapshots" / NEW / "model.safetensors").exists()+ assert partial.exists()++ _reuse(tmp_path, [ExpectedFile("model.safetensors", len(data), digest)])+ assert (repo_dir / "snapshots" / NEW / "model.safetensors").exists()+ assert not partial.exists()+++def test_unsafe_paths_and_unknown_digests_are_ignored(tmp_path):+ data = _blob(16, 512)+ _copy_layout(tmp_path, {"model.safetensors": data})++ result = _reuse(+ tmp_path,+ [+ ExpectedFile("../escape.bin", len(data), _sha256(data)),+ ExpectedFile("model.safetensors", len(data), None),+ ExpectedFile("model.safetensors", len(data), "not-a-digest"),+ ],+ )++ assert result.reused == ()+++def test_reusable_paths_is_read_only_and_never_hashes(tmp_path, monkeypatch):+ same, changed = _blob(17, 2048), _blob(18, 2048)+ repo_dir = _copy_layout(tmp_path, {"te.safetensors": same, "vae.safetensors": changed})+ monkeypatch.setattr(snapshot_reuse, "file_digest", lambda *a, **k: pytest.fail("plan hashed"))+ digests = {+ OLD: {"te.safetensors": _sha256(same), "vae.safetensors": _sha256(changed)},+ NEW: {"te.safetensors": _sha256(same), "vae.safetensors": _sha256(b"new" + changed[3:])},+ }++ found = snapshot_reuse.reusable_paths(+ "model",+ REPO,+ NEW,+ {"te.safetensors": len(same), "vae.safetensors": len(changed), "absent.bin": 5},+ hub_cache = tmp_path / "hub",+ remote_digests = lambda commit, paths: {p: digests[commit][p] for p in paths},+ )++ assert found == {"te.safetensors"}+ assert not (repo_dir / "snapshots" / NEW).exists()+++def test_reusable_paths_makes_no_request_without_a_local_candidate(tmp_path):+ _copy_layout(tmp_path, {"other.bin": b"x"})++ found = snapshot_reuse.reusable_paths(+ "model",+ REPO,+ NEW,+ {"te.safetensors": 10},+ hub_cache = tmp_path / "hub",+ remote_digests = lambda commit, paths: pytest.fail("network without a candidate"),+ )++ assert found == set()+++_UNCHANGED = _blob(100, 3 * 1024 * 1024)+_VAE = _blob(101, 256 * 1024)+_CHANGED_OLD = _blob(102, 512 * 1024)+_CHANGED_NEW = _blob(103, 512 * 1024)+_REVISIONS = {+ OLD: {+ "README.md": b"# v1\n",+ "text_encoder.safetensors": _UNCHANGED,+ "vae/vae.safetensors": _VAE,+ "changed.safetensors": _CHANGED_OLD,+ },+ NEW: {+ "README.md": b"# v2, a card edit\n",+ "text_encoder.safetensors": _UNCHANGED,+ "vae/vae.safetensors": _VAE,+ "changed.safetensors": _CHANGED_NEW,+ },+}+_LFS_MIN = 1024+++class _FakeHub:+ def __init__(self):+ self.head = OLD+ self.downloaded: dict[tuple[str, str], int] = {}+ self.lock = threading.Lock()+ hub = self++ class Handler(BaseHTTPRequestHandler):+ def log_message(self, *args):+ pass++ def _json(self, payload):+ body = json.dumps(payload).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def _commit(self, rev):+ rev = unquote(rev)+ return hub.head if rev == "main" else rev if rev in _REVISIONS else None++ def _resolve(self, head_only):+ parts = urlsplit(self.path).path.split("/")+ commit = self._commit(parts[4])+ rel = unquote("/".join(parts[5:]))+ data = _REVISIONS.get(commit, {}).get(rel)+ if data is None:+ self.send_response(404)+ self.send_header("X-Error-Code", "EntryNotFound")+ self.end_headers()+ return+ lfs = len(data) >= _LFS_MIN+ self.send_response(200)+ self.send_header("X-Repo-Commit", commit)+ self.send_header("ETag", f'"{_git_sha1(data)}"')+ if lfs:+ self.send_header("X-Linked-Etag", f'"{_sha256(data)}"')+ self.send_header("X-Linked-Size", str(len(data)))+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ if not head_only:+ self.wfile.write(data)+ with hub.lock:+ key = (commit, rel)+ hub.downloaded[key] = hub.downloaded.get(key, 0) + len(data)++ def do_HEAD(self):+ self._resolve(head_only = True)++ def do_GET(self):+ path = urlsplit(self.path).path+ if path.startswith(f"/api/models/{REPO}"):+ rest = path[len(f"/api/models/{REPO}") :]+ if rest in ("", "/"):+ return self._json(_model_info(hub.head))+ if rest.startswith("/revision/"):+ commit = self._commit(rest[len("/revision/") :])+ return self._json(_model_info(commit))+ if rest.startswith("/tree/"):+ commit = self._commit(rest[len("/tree/") :].split("/")[0])+ return self._json(+ [_tree_entry(p, d) for p, d in _REVISIONS[commit].items()]+ )+ if "/resolve/" in path:+ return self._resolve(head_only = False)+ self.send_response(404)+ self.end_headers()++ def do_POST(self):+ path = urlsplit(self.path).path+ prefix = f"/api/models/{REPO}/paths-info/"+ if not path.startswith(prefix):+ self.send_response(404)+ self.end_headers()+ return+ commit = self._commit(path[len(prefix) :])+ length = int(self.headers.get("Content-Length") or 0)+ form = parse_qs(self.rfile.read(length).decode())+ files = _REVISIONS.get(commit, {})+ self._json([_tree_entry(p, files[p]) for p in form.get("paths", []) if p in files])++ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ self.url = f"http://127.0.0.1:{self.server.server_address[1]}"+ threading.Thread(target = self.server.serve_forever, daemon = True).start()++ def close(self):+ self.server.shutdown()+++def _tree_entry(path, data):+ entry = {"type": "file", "path": path, "size": len(data), "oid": _git_sha1(data)}+ if len(data) >= _LFS_MIN:+ entry["lfs"] = {"oid": _sha256(data), "size": len(data), "pointerSize": 134}+ return entry+++def _model_info(commit):+ siblings = []+ for path, data in _REVISIONS[commit].items():+ sibling = {"rfilename": path, "size": len(data), "blobId": _git_sha1(data)}+ if len(data) >= _LFS_MIN:+ sibling["lfs"] = {"sha256": _sha256(data), "size": len(data), "pointerSize": 134}+ siblings.append(sibling)+ return {"id": REPO, "modelId": REPO, "sha": commit, "siblings": siblings, "private": False}+++_WORKER_SCRIPT = textwrap.dedent(+ """+ import sys+ backend, mode, symlinks = sys.argv[1], sys.argv[2], sys.argv[3] == "1"+ sys.path.insert(0, backend)+ from huggingface_hub import file_download+ if not symlinks:+ # What a Windows machine without Developer Mode answers: blobs are moved into snapshots.+ file_download.are_symlinks_supported = lambda cache_dir = None: False+ from hub.workers import hf_download+ if mode == "scoped":+ hf_download._download_scoped_snapshot(+ "Org/Model", "@diffusion",+ ["text_encoder.safetensors", "vae/vae.safetensors", "changed.safetensors"],+ None, "http",+ )+ else:+ hf_download._download_snapshot("Org/Model", None, "http")+ """+)+++def fake_hub():+ hub = _FakeHub()+ yield hub+ hub.close()+++def _run_worker(tmp_path, hub, mode, symlinks):+ env = {+ k: v+ for k, v in os.environ.items()+ if not k.startswith(("HF_", "HUGGING_FACE_", "UNSLOTH_PROTECTED"))+ }+ env.update(+ HF_ENDPOINT = hub.url,+ HF_HOME = str(tmp_path / "hf_home"),+ HF_HUB_CACHE = str(tmp_path / "hub"),+ HF_HUB_DISABLE_XET = "1",+ HF_HUB_ENABLE_HF_TRANSFER = "0",+ HF_HUB_DISABLE_TELEMETRY = "1",+ HF_HUB_DISABLE_SYMLINKS_WARNING = "1",+ HF_TOKEN = "",+ UNSLOTH_STUDIO_HOME = os.environ.get("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio")),+ NO_PROXY = "127.0.0.1,localhost",+ no_proxy = "127.0.0.1,localhost",+ )+ proc = subprocess.run(+ [sys.executable, "-c", _WORKER_SCRIPT, str(_BACKEND_DIR), mode, "1" if symlinks else "0"],+ env = env,+ capture_output = True,+ text = True,+ timeout = 300,+ )+ assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-3000:]+ return proc+++def _symlinks_available(tmp_path) -> bool:+ try:+ (tmp_path / "probe_link").symlink_to(tmp_path)+ return True+ except OSError:+ return False++[email protected]("mode", ["scoped", "snapshot"])[email protected]("symlinks", [False, True])+def test_new_revision_downloads_only_what_changed(tmp_path, fake_hub, mode, symlinks):+ if symlinks and not _symlinks_available(tmp_path):+ pytest.skip("symlinks unavailable")+ _run_worker(tmp_path, fake_hub, mode, symlinks)+ first = dict(fake_hub.downloaded)+ assert first[(OLD, "text_encoder.safetensors")] == len(_UNCHANGED)++ fake_hub.head = NEW+ fake_hub.downloaded.clear()+ proc = _run_worker(tmp_path, fake_hub, mode, symlinks)++ second = fake_hub.downloaded+ assert (NEW, "text_encoder.safetensors") not in second, proc.stderr[-2000:]+ assert (NEW, "vae/vae.safetensors") not in second+ assert second[(NEW, "changed.safetensors")] == len(_CHANGED_NEW)++ snap = tmp_path / "hub" / "models--Org--Model" / "snapshots" / NEW+ assert (snap / "text_encoder.safetensors").read_bytes() == _UNCHANGED+ assert (snap / "vae" / "vae.safetensors").read_bytes() == _VAE+ assert (snap / "changed.safetensors").read_bytes() == _CHANGED_NEW+ old_snap = snap.parent / OLD+ assert (old_snap / "changed.safetensors").read_bytes() == _CHANGED_OLD+ assert (tmp_path / "hub" / "models--Org--Model" / "refs" / "main").read_text() == NEW+ if symlinks:+ assert (snap / "text_encoder.safetensors").is_symlink()+ assert "Reused" not in proc.stderr+ else:+ assert not (snap / "text_encoder.safetensors").is_symlink()+ assert "Reused" in proc.stderr+++def test_a_corrupted_same_size_copy_is_downloaded_again_not_carried_forward(tmp_path, fake_hub):+ """The Hub vouching for the OLD commit's digest says nothing about the bytes on disk now."""+ _run_worker(tmp_path, fake_hub, "scoped", symlinks = False)+ old_encoder = (+ tmp_path / "hub" / "models--Org--Model" / "snapshots" / OLD / "text_encoder.safetensors"+ )+ damaged = bytearray(old_encoder.read_bytes())+ damaged[1000:1010] = b"\x00" * 10+ old_encoder.write_bytes(bytes(damaged))++ fake_hub.head = NEW+ fake_hub.downloaded.clear()+ _run_worker(tmp_path, fake_hub, "scoped", symlinks = False)++ assert fake_hub.downloaded[(NEW, "text_encoder.safetensors")] == len(_UNCHANGED)+ assert (NEW, "vae/vae.safetensors") not in fake_hub.downloaded+ snap = old_encoder.parent.parent / NEW+ assert (snap / "text_encoder.safetensors").read_bytes() == _UNCHANGED+++def test_a_reuse_pass_persists_its_digests_in_one_write(tmp_path, monkeypatch):+ files = {f"shard-{i}.bin": _blob(40 + i, 2048) for i in range(5)}+ repo_dir = _copy_layout(tmp_path, files)+ writes = []+ monkeypatch.setattr(+ snapshot_reuse, "_remember_digests", lambda entries: writes.append(dict(entries))+ )++ matches, _ = snapshot_reuse.find_reusable_copies(+ repo_dir, NEW, {p: (len(d), _sha256(d)) for p, d in files.items()}+ )++ assert set(matches) == set(files)+ assert len(writes) == 1 and len(writes[0]) == len(files)+++def test_a_blob_a_live_peer_is_downloading_gets_no_pointer(tmp_path):+ from filelock import FileLock++ data = _blob(50, 4096)+ digest = _sha256(data)+ repo_dir = _copy_layout(tmp_path, {"model.safetensors": data})+ lock = tmp_path / "hub" / ".locks" / repo_dir.name / f"{digest}.lock"+ lock.parent.mkdir(parents = True)+ expected = [ExpectedFile("model.safetensors", len(data), digest)]++ with FileLock(str(lock)):+ held = _reuse(tmp_path, expected)+ assert held.reused == ()+ assert not (repo_dir / "snapshots" / NEW / "model.safetensors").exists()++ assert _reuse(tmp_path, expected).reused == ("model.safetensors",)+++def test_hard_linked_revisions_are_counted_once_in_cache_usage(tmp_path):+ from huggingface_hub import scan_cache_dir++ from hub.services.models.cache_inventory import _repo_gguf_size_bytes, repo_unique_size_bytes++ gguf = _blob(60, 8192)+ repo_dir = _copy_layout(tmp_path, {"model-Q4_K_M.gguf": gguf, "README.md": b"v1"})+ new = repo_dir / "snapshots" / NEW+ new.mkdir(parents = True)+ os.link(repo_dir / "snapshots" / OLD / "model-Q4_K_M.gguf", new / "model-Q4_K_M.gguf")+ (new / "README.md").write_bytes(b"v2")++ (repo,) = scan_cache_dir(tmp_path / "hub").repos+ assert (+ len(+ {+ str(f.blob_path)+ for r in repo.revisions+ for f in r.files+ if f.file_name.endswith(".gguf")+ }+ )+ == 2+ )+ assert _repo_gguf_size_bytes(repo) == len(gguf)+ assert repo_unique_size_bytes(repo) == len(gguf) + len(b"v1") + len(b"v2")++ (new / "model-Q4_K_M.gguf").unlink()+ (new / "model-Q4_K_M.gguf").write_bytes(gguf) # a real second copy still counts twice+ (repo,) = scan_cache_dir(tmp_path / "hub").repos+ assert _repo_gguf_size_bytes(repo) == 2 * len(gguf)+ assert repo_unique_size_bytes(repo) == 2 * len(gguf) + len(b"v1") + len(b"v2")+++def test_a_plan_asks_the_hub_about_at_most_a_few_old_commits(tmp_path):+ repo_dir = tmp_path / "hub" / "models--Org--Model"+ for i in range(6):+ snap = repo_dir / "snapshots" / (f"{i:x}" * 40)[:40]+ snap.mkdir(parents = True)+ (snap / "w.safetensors").write_bytes(_blob(70 + i, 1024)) # same size, new bytes each time+ os.utime(snap, (1_000 + i, 1_000 + i))+ asked = []++ matches, _ = snapshot_reuse.find_reusable_copies(+ repo_dir,+ NEW,+ {"w.safetensors": (1024, "f" * 64)},+ remote_digests = lambda commit, paths: asked.append(commit) or {p: "e" * 64 for p in paths},+ allow_hashing = False,+ )++ assert matches == {}+ assert len(asked) == snapshot_reuse._REMOTE_DIGEST_COMMITS+++def test_a_partial_is_only_removed_while_holding_the_blob_lock(tmp_path):+ from filelock import FileLock++ repo_dir = _copy_layout(tmp_path, {"x.bin": b"x"})+ digest = "d" * 64+ partial = repo_dir / "blobs" / f"{digest}.incomplete"+ partial.write_bytes(b"partial")+ lock = tmp_path / "hub" / ".locks" / repo_dir.name / f"{digest}.lock"+ lock.parent.mkdir(parents = True)++ with FileLock(str(lock)): # a peer's download in progress+ snapshot_reuse._drop_superseded_partial(repo_dir, digest, frozenset())+ assert partial.exists()+ snapshot_reuse._drop_superseded_partial(repo_dir, digest, frozenset())+ assert not partial.exists()+++def test_companion_cleanup_counts_hard_linked_revisions_once(tmp_path):+ from huggingface_hub import scan_cache_dir++ from hub.services.models.companion_cleanup import _repo_blob_bytes++ weights = _blob(80, 8192)+ repo_dir = _copy_layout(tmp_path, {"vae/vae.safetensors": weights})+ (repo_dir / "snapshots" / NEW / "vae").mkdir(parents = True)+ os.link(+ repo_dir / "snapshots" / OLD / "vae" / "vae.safetensors",+ repo_dir / "snapshots" / NEW / "vae" / "vae.safetensors",+ )++ (repo,) = scan_cache_dir(tmp_path / "hub").repos+ assert _repo_blob_bytes(repo) == len(weights)diff --git a/studio/backend/hub/utils/snapshot_reuse.py b/studio/backend/hub/utils/snapshot_reuse.pynew file mode 100644index 00000000000..07691ebc896--- /dev/null+++ b/studio/backend/hub/utils/snapshot_reuse.py@@ -0,0 +1,517 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Reuse unchanged files from older snapshots on a no-symlink cache, where ``file_download._create_symlink`` moves each blob into its snapshot and leaves ``blobs/`` empty."""++from __future__ import annotations++import glob+import hashlib+import json+import os+import re+import shutil+import stat as stat_module+import threading+import uuid+from dataclasses import dataclass+from pathlib import Path+from typing import Callable, Iterable, Mapping, Optional, Sequence++from loggers import get_logger++logger = get_logger(__name__)++_SHA256 = re.compile(r"^[0-9a-f]{64}$")+_GIT_SHA1 = re.compile(r"^[0-9a-f]{40}$")+_HASH_CHUNK = 8 * 1024 * 1024+_DIGEST_CACHE_NAME = "file_digests.json"+_DIGEST_CACHE_LIMIT = 2048+_PATHS_INFO_BATCH = 100+# Old snapshots a plan asks the Hub about; the rest stay counted (the plan is on the request path).+_REMOTE_DIGEST_COMMITS = 3+_digest_cache_lock = threading.Lock()++RemoteDigests = Callable[[str, Sequence[str]], Mapping[str, str]]+++@dataclass(frozen = True)+class ReuseResult:+ reused: tuple[str, ...] = ()+ reused_bytes: int = 0+ linked: int = 0+ copied: int = 0+ hashed_bytes: int = 0+++def digest_kind(digest: Optional[str]) -> Optional[str]:+ if not isinstance(digest, str):+ return None+ if _SHA256.match(digest):+ return "sha256"+ if _GIT_SHA1.match(digest):+ return "git-sha1"+ return None+++def file_digest(path: Path, kind: str) -> str:+ if kind == "sha256":+ h = hashlib.sha256()+ elif kind == "git-sha1":+ h = hashlib.sha1()+ h.update(b"blob %d\x00" % os.stat(path).st_size)+ else:+ raise ValueError(f"unsupported digest kind: {kind}")+ with open(path, "rb") as f:+ while True:+ chunk = f.read(_HASH_CHUNK)+ if not chunk:+ break+ h.update(chunk)+ return h.hexdigest()+++def _digest_cache_path() -> Optional[Path]:+ try:+ from hub.utils.state_dir import state_root+ root = state_root(create = True)+ except Exception: # noqa: BLE001 - the cache is an optimisation+ return None+ return None if root is None else root / _DIGEST_CACHE_NAME+++def _digest_cache_key(path: Path, kind: str, st: os.stat_result) -> str:+ return "|".join(+ (+ kind,+ os.path.normcase(os.path.abspath(str(path))),+ str(st.st_size),+ str(st.st_mtime_ns),+ str(getattr(st, "st_ino", 0)),+ "" if os.name == "nt" else str(st.st_ctime_ns),+ )+ )+++def _read_digest_cache(cache_path: Path) -> dict:+ try:+ with open(cache_path, "r", encoding = "utf-8") as f:+ data = json.load(f)+ except (OSError, ValueError):+ return {}+ return data if isinstance(data, dict) else {}+++def _remember_digests(entries: Mapping[str, str]) -> None:+ cache_path = _digest_cache_path()+ if cache_path is None or not entries:+ return+ with _digest_cache_lock:+ data = _read_digest_cache(cache_path)+ for key, digest in entries.items():+ data.pop(key, None)+ data[key] = digest+ while len(data) > _DIGEST_CACHE_LIMIT:+ data.pop(next(iter(data)))+ tmp = cache_path.with_name(f".{cache_path.name}.tmp-{uuid.uuid4().hex[:8]}")+ try:+ with open(tmp, "w", encoding = "utf-8") as f:+ json.dump(data, f)+ os.replace(tmp, cache_path)+ except OSError as exc:+ logger.debug("could not persist file digest cache: %s", exc)+ try:+ tmp.unlink()+ except OSError:+ pass+++def cached_file_digest(+ path: Path,+ kind: str,+ *,+ compute: bool = True,+ learned: Optional[dict] = None,+) -> tuple[Optional[str], int]:+ """(digest, bytes hashed). Stat metadata cannot prove unchanged bytes, so the persisted cache only serves ``compute=False`` estimates."""+ try:+ st = os.stat(path)+ except OSError:+ return None, 0+ key = _digest_cache_key(path, kind, st)+ if not compute:+ cache_path = _digest_cache_path()+ if cache_path is None:+ return None, 0+ cached = _read_digest_cache(cache_path).get(key)+ return (cached, 0) if isinstance(cached, str) and digest_kind(cached) == kind else (None, 0)+ try:+ digest = file_digest(path, kind)+ after = os.stat(path)+ except OSError:+ return None, 0+ if (after.st_size, after.st_mtime_ns) != (st.st_size, st.st_mtime_ns):+ return None, st.st_size+ if learned is None:+ _remember_digests({key: digest})+ else:+ learned[key] = digest # the caller persists a whole pass in one write+ return digest, st.st_size+++def repo_cache_dir(+ repo_type: str,+ repo_id: str,+ hub_cache: Optional[str | Path] = None,+) -> Path:+ from huggingface_hub import constants+ from huggingface_hub.file_download import repo_folder_name++ root = Path(hub_cache) if hub_cache else Path(constants.HF_HUB_CACHE)+ return root / repo_folder_name(repo_id = repo_id, repo_type = repo_type)+++def cached_ref_commit(+ repo_type: str,+ repo_id: str,+ hub_cache: Optional[str | Path] = None,+ ref: str = "main",+) -> Optional[str]:+ try:+ from hub.utils.download_manifest import normalized_commit_hash+ text = (repo_cache_dir(repo_type, repo_id, hub_cache) / "refs" / ref).read_text(+ encoding = "utf-8"+ )+ return normalized_commit_hash(text.strip())+ except Exception: # noqa: BLE001 - no readable ref means nothing to compare against+ return None+++def _is_regular_file(path: Path) -> bool:+ try:+ return stat_module.S_ISREG(os.lstat(path).st_mode)+ except OSError:+ return False+++def _other_snapshots(repo_dir: Path, commit_hash: str) -> list[Path]:+ try:+ snapshots = [+ p for p in (repo_dir / "snapshots").iterdir() if p.is_dir() and p.name != commit_hash+ ]+ except OSError:+ return []+ from hub.utils.hf_cache_state import snapshot_selection_key+ return sorted(snapshots, key = snapshot_selection_key, reverse = True)+++def _candidates(snapshots: Iterable[Path], rel_path: str, size: int) -> list[Path]:+ found = []+ seen: set[tuple[int, int]] = set()+ for snap in snapshots:+ candidate = snap / rel_path+ # A symlink means the blob layout, which huggingface_hub already reuses.+ if not _is_regular_file(candidate):+ continue+ try:+ st = os.stat(candidate)+ except OSError:+ continue+ if st.st_size != size:+ continue+ inode = (st.st_dev, st.st_ino)+ if st.st_ino and inode in seen:+ continue+ seen.add(inode)+ found.append(candidate)+ return found+++def _needs_reuse(repo_dir: Path, target: Path, digest: str) -> bool:+ if os.path.lexists(target):+ return False+ if os.path.exists(repo_dir / "blobs" / digest):+ return False+ return True+++def find_reusable_copies(+ repo_dir: Path,+ commit_hash: str,+ expected: Mapping[str, tuple[int, str]],+ *,+ remote_digests: Optional[RemoteDigests] = None,+ allow_hashing: bool = True,+) -> tuple[dict[str, Path], int]:+ snapshots = _other_snapshots(repo_dir, commit_hash)+ if not snapshots:+ return {}, 0+ candidates = {+ path: _candidates(snapshots, path, size)+ for path, (size, digest) in expected.items()+ if size > 0 and digest_kind(digest) is not None+ }++ matches: dict[str, Path] = {}+ hashed = 0+ # Plan estimate only: the worker passes no remote_digests and hashes instead.+ if remote_digests is not None:+ asked = 0+ for snap in snapshots:+ if asked >= _REMOTE_DIGEST_COMMITS:+ break+ here = {+ path: candidate+ for path, found in candidates.items()+ if path not in matches+ for candidate in found+ if candidate == snap / path+ }+ if not here:+ continue+ asked += 1+ try:+ remote = remote_digests(snap.name, sorted(here)) or {}+ except Exception as exc: # noqa: BLE001 - squashed history, offline, 404: hash instead+ logger.debug("remote digests unavailable for %s: %s", snap.name, exc)+ continue+ for path, candidate in here.items():+ if remote.get(path) == expected[path][1]:+ matches[path] = candidate+ learned: dict[str, str] = {}+ for path, found in candidates.items():+ if path in matches:+ continue+ kind = digest_kind(expected[path][1])+ for candidate in found:+ local, spent = cached_file_digest(+ candidate, kind, compute = allow_hashing, learned = learned+ )+ hashed += spent+ if local == expected[path][1]:+ matches[path] = candidate+ break+ _remember_digests(learned)+ return matches, hashed+++def _place(src: Path, dst: Path, size: int) -> Optional[str]:+ dst.parent.mkdir(parents = True, exist_ok = True)+ for stale in dst.parent.glob(f".{glob.escape(dst.name)}.reuse-*"):+ try:+ stale.unlink()+ except OSError:+ pass+ tmp = dst.with_name(f".{dst.name}.reuse-{os.getpid()}-{uuid.uuid4().hex[:8]}")+ how = None+ try:+ try:+ os.link(src, tmp)+ how = "link"+ except OSError:+ if shutil.disk_usage(dst.parent).free < size:+ return None+ shutil.copyfile(src, tmp)+ how = "copy"+ if os.path.lexists(dst):+ return None+ os.replace(tmp, dst)+ return how+ except OSError as exc:+ logger.info("could not reuse %s for %s: %s", src, dst, exc)+ return None+ finally:+ try:+ if os.path.lexists(tmp):+ os.unlink(tmp)+ except OSError:+ pass+++def _drop_superseded_partial(repo_dir: Path, digest: str, protected: frozenset[str]) -> None:+ if digest in protected:+ return+ partial = repo_dir / "blobs" / f"{digest}.incomplete"+ if not partial.is_file():+ return+ lock_path = repo_dir.parent / ".locks" / repo_dir.name / f"{digest}.lock"+ try:+ from filelock import FileLock, Timeout+ lock_path.parent.mkdir(parents = True, exist_ok = True)+ # Held, not probed: huggingface_hub writes partials only under this lock.+ with FileLock(str(lock_path), timeout = 0):+ partial.unlink(missing_ok = True)+ except Timeout:+ return+ except Exception as exc: # noqa: BLE001 - no flock here (SoftFileLock territory): leave it+ logger.debug("could not remove superseded partial %s: %s", partial, exc)+++def reuse_unchanged_snapshot_files(+ repo_type: str,+ repo_id: str,+ commit_hash: Optional[str],+ expected_files: Sequence,+ *,+ hub_cache: Optional[str | Path] = None,+ remote_digests: Optional[RemoteDigests] = None,+ allow_hashing: bool = True,+ protected_blob_hashes: frozenset[str] = frozenset(),+) -> ReuseResult:+ """Never raises: anything it cannot prove is left for the normal download."""+ try:+ from hub.utils.download_manifest import expected_path_is_safe, normalized_commit_hash++ commit = normalized_commit_hash(commit_hash)+ if not commit:+ return ReuseResult()+ repo_dir = repo_cache_dir(repo_type, repo_id, hub_cache)+ if not (repo_dir / "snapshots").is_dir():+ return ReuseResult()+ target_root = repo_dir / "snapshots" / commit+ pending: dict[str, tuple[int, str]] = {}+ for item in expected_files:+ path = getattr(item, "path", None)+ size = int(getattr(item, "size", 0) or 0)+ digest = getattr(item, "sha256", None)+ if not expected_path_is_safe(path) or size <= 0 or digest_kind(digest) is None:+ continue+ # huggingface_hub links a finished blob only when the pointer is absent: none under a peer.+ if digest in protected_blob_hashes:+ continue+ if _needs_reuse(repo_dir, target_root / path, digest):+ pending[path] = (size, digest)+ if not pending:+ return ReuseResult()+ matches, hashed = find_reusable_copies(+ repo_dir,+ commit,+ pending,+ remote_digests = remote_digests,+ allow_hashing = allow_hashing,+ )+ reused, linked, copied, reused_bytes = [], 0, 0, 0+ from hub.utils.hf_cache_state import blob_download_lock_held++ for path, src in matches.items():+ size, digest = pending[path]+ # A peer may have started after launch.+ if blob_download_lock_held(repo_dir, digest):+ continue+ how = _place(src, target_root / path, size)+ if how is None:+ continue+ reused.append(path)+ reused_bytes += size+ linked += how == "link"+ copied += how == "copy"+ _drop_superseded_partial(repo_dir, digest, protected_blob_hashes)+ return ReuseResult(tuple(reused), reused_bytes, linked, copied, hashed)+ except Exception as exc: # noqa: BLE001 - reuse is an optimisation, never a failure+ logger.warning("snapshot reuse skipped for %s: %s", repo_id, exc)+ return ReuseResult()+++def paths_in_snapshot(+ repo_type: str,+ repo_id: str,+ commit_hash: Optional[str],+ paths: Iterable[str],+ *,+ hub_cache: Optional[str | Path] = None,+) -> set[str]:+ try:+ from hub.utils.download_manifest import expected_path_is_safe, normalized_commit_hash++ commit = normalized_commit_hash(commit_hash)+ if not commit:+ return set()+ root = repo_cache_dir(repo_type, repo_id, hub_cache) / "snapshots" / commit+ return {p for p in paths if expected_path_is_safe(p) and os.path.exists(root / p)}+ except Exception: # noqa: BLE001 - counting a present file again only makes the preflight stricter+ return set()+++def hub_remote_digests(repo_type: str, repo_id: str, token) -> RemoteDigests:+ def lookup(commit: str, paths: Sequence[str]) -> Mapping[str, str]:+ from huggingface_hub import HfApi++ api = HfApi(token = token)+ paths = list(paths)+ infos = []+ for start in range(0, len(paths), _PATHS_INFO_BATCH):+ infos.extend(+ api.get_paths_info(+ repo_id,+ paths[start : start + _PATHS_INFO_BATCH],+ revision = commit,+ repo_type = repo_type,+ )+ )+ out: dict[str, str] = {}+ for info in infos:+ lfs = getattr(info, "lfs", None)+ digest = getattr(lfs, "sha256", None) if lfs is not None else None+ if isinstance(lfs, dict):+ digest = lfs.get("sha256")+ digest = digest or getattr(info, "blob_id", None)+ path = getattr(info, "path", None)+ if isinstance(path, str) and isinstance(digest, str):+ out[path] = digest+ return out++ return lookup+++def reusable_paths(+ repo_type: str,+ repo_id: str,+ commit_hash: Optional[str],+ sizes: Mapping[str, int],+ *,+ hub_cache: Optional[str | Path] = None,+ remote_digests: Optional[RemoteDigests] = None,+ allow_hashing: bool = False,+ digest_revision: Optional[str] = None,+) -> set[str]:+ """Read-only plan estimate: no network unless a same-size local copy exists."""+ try:+ from hub.utils.download_manifest import expected_path_is_safe, normalized_commit_hash++ commit = normalized_commit_hash(commit_hash)+ if not commit or not sizes:+ return set()+ repo_dir = repo_cache_dir(repo_type, repo_id, hub_cache)+ snapshots = _other_snapshots(repo_dir, commit)+ if not snapshots:+ return set()+ target_root = repo_dir / "snapshots" / commit+ local = [+ path+ for path, size in sizes.items()+ if expected_path_is_safe(path)+ and int(size or 0) > 0+ and not os.path.lexists(target_root / path)+ and _candidates(snapshots, path, int(size))+ ]+ if not local or remote_digests is None:+ return set()+ # digest_revision names what the worker will fetch when commit_hash is only the local ref.+ target = remote_digests(digest_revision or commit, sorted(local)) or {}+ expected = {+ path: (int(sizes[path]), target[path])+ for path in local+ if digest_kind(target.get(path)) is not None+ and not os.path.exists(repo_dir / "blobs" / target[path])+ }+ matches, _ = find_reusable_copies(+ repo_dir,+ commit,+ expected,+ remote_digests = remote_digests,+ allow_hashing = allow_hashing,+ )+ return set(matches)+ except Exception as exc: # noqa: BLE001 - a plan must never fail over an optimisation+ logger.debug("reusable_paths skipped for %s: %s", repo_id, exc)+ return set()diff --git a/studio/backend/hub/workers/hf_download.py b/studio/backend/hub/workers/hf_download.pyindex 9a2529886d7..e4fb595691d 100644--- a/studio/backend/hub/workers/hf_download.py+++ b/studio/backend/hub/workers/hf_download.py@@ -179,9 +179,13 @@ def _retry_metadata_fetch(repo_id: str, fetch, *, label: str):raise RuntimeError(f"{label} unavailable for {repo_id}")+_RESOLVED_COMMITS: dict[str, str] = {}++def _model_info_with_retry(repo_id: str, hf_token: str | None):from huggingface_hub import model_info as hf_model_info- return _retry_metadata_fetch(++ info = _retry_metadata_fetch(repo_id,lambda timeout: hf_model_info(repo_id,@@ -191,6 +195,39 @@ def _model_info_with_retry(repo_id: str, hf_token: str | None):),label = "Metadata",)+ commit = getattr(info, "sha", None)+ if isinstance(commit, str) and commit:+ _RESOLVED_COMMITS[repo_id] = commit+ return info+++def _reuse_unchanged_files(+ repo_type: RepoType, repo_id: str, commit_hash, expected_files: list, hf_token: str | None+) -> list:+ """Link files unchanged since an older snapshot into this commit; return the files still to download."""+ from hub.utils.snapshot_reuse import paths_in_snapshot, reuse_unchanged_snapshot_files++ if not expected_files or not commit_hash:+ return list(expected_files)+ result = reuse_unchanged_snapshot_files(+ repo_type,+ repo_id,+ commit_hash,+ expected_files,+ # Always hash locally: a Hub digest proves what the old commit served, not what is on disk now.+ protected_blob_hashes = _protected_blob_hashes(),+ )+ if result.reused:+ print(+ f"Reused {len(result.reused)} unchanged file(s) ({result.reused_bytes / 1e9:.2f} GB) "+ f"from an older snapshot of {repo_id} instead of downloading them again.",+ file = sys.stderr,+ )+ # Files an earlier attempt placed are skipped by snapshot_download and have no blob for the preflight to discount.+ present = paths_in_snapshot(+ repo_type, repo_id, commit_hash, [getattr(f, "path", None) for f in expected_files]+ )+ return [f for f in expected_files if getattr(f, "path", None) not in present]def _dataset_info_with_retry(repo_id: str, hf_token: str | None):@@ -541,7 +578,10 @@ def _download_snapshot(repo_id: str, hf_token: str | None, mode: str) -> None:f"before starting {mode} download.",file = sys.stderr,)- _preflight_disk_space("model", repo_id, expected_files)+ to_download = _reuse_unchanged_files(+ "model", repo_id, getattr(info, "sha", None), expected_files, hf_token+ )+ _preflight_disk_space("model", repo_id, to_download)snapshot_path = snapshot_download(repo_id = repo_id,token = _hf_token_arg(hf_token),@@ -676,7 +716,14 @@ def _download_gguf_variant(repo_id: str, variant: str, hf_token: str | None, modf"before starting {mode} download.",file = sys.stderr,)- _preflight_disk_space("model", repo_id, expected_files)+ to_download = (+ expected_files+ if metadata_unavailable+ else _reuse_unchanged_files(+ "model", repo_id, _RESOLVED_COMMITS.get(repo_id), expected_files, hf_token+ )+ )+ _preflight_disk_space("model", repo_id, to_download)snapshot_path = snapshot_download(repo_id = repo_id,token = _hf_token_arg(hf_token),@@ -768,7 +815,10 @@ def _download_scoped_snapshot(f"before starting {mode} download.",file = sys.stderr,)- _preflight_disk_space("model", repo_id, expected_files)+ to_download = _reuse_unchanged_files(+ "model", repo_id, getattr(info, "sha", None), expected_files, hf_token+ )+ _preflight_disk_space("model", repo_id, to_download)snapshot_path = snapshot_download(repo_id = repo_id,token = _hf_token_arg(hf_token),@@ -834,7 +884,8 @@ def _download_dataset(repo_id: str, hf_token: str | None, mode: str) -> None:f"before starting {mode} download.",file = sys.stderr,)- _preflight_disk_space("dataset", repo_id, expected_files)+ to_download = _reuse_unchanged_files("dataset", repo_id, commit_hash, expected_files, hf_token)+ _preflight_disk_space("dataset", repo_id, to_download)download_kwargs = {"repo_id": repo_id,"token": _hf_token_arg(hf_token),diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex c4ffda8abe1..a1c37c9bde6 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -4844,12 +4844,8 @@ def _repo_gguf_size_bytes(repo_info) -> int:# Snapshot-relative: only the directory tells an MTP/ drafter from a primary quant.name = _cached_repo_file_name(f)if _is_main_gguf_filename(name):- blob_path = getattr(f, "blob_path", None)- size = f.size_on_disk or 0- if blob_path:- unique_blobs[str(blob_path)] = size- else:- unique_blobs[f"{rev_id}:{name}"] = size+ from hub.services.models.cache_inventory import _blob_key+ unique_blobs[_blob_key(f, f"{rev_id}:{name}")] = f.size_on_disk or 0return sum(unique_blobs.values())@@ -5243,9 +5239,9 @@ def cached_model_rows(cache_scans = None) -> list[dict]:and _snapshot_can_serve_a_load(selected)):continue- total_size = sum(- (f.size_on_disk or 0) for rev in repo_info.revisions for f in rev.files- )+ from hub.services.models.cache_inventory import repo_unique_size_bytes++ total_size = repo_unique_size_bytes(repo_info)if total_size == 0:continueweight_files = [diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.pyindex d863f07ff8d..3d454279dec 100644--- a/studio/backend/tests/test_diffusion_backend.py+++ b/studio/backend/tests/test_diffusion_backend.py@@ -7670,6 +7670,130 @@ def test_download_plan_omits_a_cached_gguf_but_keeps_missing_companions(monkeypaassert plan["checkpoint_bytes"] == 7 * GB+def test_download_plan_stages_but_does_not_count_a_file_an_older_snapshot_holds(monkeypatch):+ """README-only commit on a no-symlink cache: the GGUF is still staged but counts no bytes."""+ _fake_flux_hub(monkeypatch)+ _no_cache(monkeypatch)+ asked = []++ def reusable(repo_id, names, revision, declared_sizes, hf_token):+ asked.append((repo_id, tuple(names)))+ return {"flux1-dev-Q4_K_M.gguf"} if repo_id == "unsloth/FLUX.1-dev-GGUF" else set()++ monkeypatch.setattr(DiffusionBackend, "_reusable_from_older_snapshot", staticmethod(reusable))++ plan = _flux_download_plan()++ checkpoint, base = plan["entries"]+ assert checkpoint["repo_id"] == "unsloth/FLUX.1-dev-GGUF"+ assert checkpoint["files"] == ["flux1-dev-Q4_K_M.gguf"]+ assert checkpoint["bytes"] == 0+ assert checkpoint["checkpoint"] is True+ assert base["bytes"] > 0+ assert plan["total_bytes"] == base["bytes"]+ assert plan["checkpoint_bytes"] == 7 * GB+ assert ("unsloth/FLUX.1-dev-GGUF", ("flux1-dev-Q4_K_M.gguf",)) in asked+++def test_reusable_from_older_snapshot_reads_the_live_cache_without_hashing(monkeypatch, tmp_path):+ from hub.utils import snapshot_reuse++ old, new = "1" * 40, "2" * 40+ same, changed = b"s" * 4096, b"c" * 4096+ snap = tmp_path / "models--unsloth--Qwen-Image-2.1-FP8" / "snapshots" / old+ (snap / "vae").mkdir(parents = True)+ (snap / "text_encoder.safetensors").write_bytes(same)+ (snap / "vae" / "vae.safetensors").write_bytes(changed)+ digests = {+ old: {"text_encoder.safetensors": "a" * 64, "vae/vae.safetensors": "b" * 64},+ new: {"text_encoder.safetensors": "a" * 64, "vae/vae.safetensors": "c" * 64},+ }+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: str(tmp_path))+ monkeypatch.setattr(+ snapshot_reuse,+ "hub_remote_digests",+ lambda repo_type, repo_id, token: lambda commit, paths: {+ p: digests[commit][p] for p in paths+ },+ )+ monkeypatch.setattr(snapshot_reuse, "file_digest", lambda *a, **k: pytest.fail("plan hashed"))++ found = DiffusionBackend._reusable_from_older_snapshot(+ "unsloth/Qwen-Image-2.1-FP8",+ ["text_encoder.safetensors", "vae/vae.safetensors"],+ new,+ {"text_encoder.safetensors": len(same), "vae/vae.safetensors": len(changed)},+ None,+ )++ assert found == {"text_encoder.safetensors"}+ assert not (snap.parent / new).exists()+ assert (+ DiffusionBackend._reusable_from_older_snapshot(+ "unsloth/Qwen-Image-2.1-FP8", ["text_encoder.safetensors"], None, {}, None+ )+ == set()+ )+++def test_reusable_from_older_snapshot_targets_the_main_ref_when_unpinned(monkeypatch, tmp_path):+ from hub.utils import snapshot_reuse++ old, new = "1" * 40, "2" * 40+ encoder = b"s" * 4096+ repo = tmp_path / "models--unsloth--Qwen-Image-2.1-FP8"+ (repo / "snapshots" / old).mkdir(parents = True)+ (repo / "snapshots" / new / "vae").mkdir(parents = True)+ (repo / "snapshots" / old / "te.safetensors").write_bytes(encoder)+ (repo / "refs").mkdir()+ (repo / "refs" / "main").write_text(new)+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: str(tmp_path))+ head = {"main": "a" * 64}+ asked = []++ def digests(repo_type, repo_id, token):+ def lookup(commit, paths):+ asked.append(commit)+ return {p: head.get(commit, "a" * 64) for p in paths}++ return lookup++ monkeypatch.setattr(snapshot_reuse, "hub_remote_digests", digests)++ def reusable():+ return DiffusionBackend._reusable_from_older_snapshot(+ "unsloth/Qwen-Image-2.1-FP8",+ ["te.safetensors"],+ None,+ {"te.safetensors": len(encoder)},+ None,+ )++ assert reusable() == {"te.safetensors"}+ assert asked[0] == "main" # the worker fetches the Hub head, not the cached ref+ head["main"] = "b" * 64+ assert reusable() == set()+++def test_reusable_from_older_snapshot_keeps_the_anonymous_token_sentinel(monkeypatch, tmp_path):+ from hub.utils import snapshot_reuse++ tokens = []+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: str(tmp_path))+ monkeypatch.setattr(+ snapshot_reuse,+ "hub_remote_digests",+ lambda repo_type, repo_id, token: tokens.append(token) or (lambda commit, paths: {}),+ )++ for token in (False, "", None, "hf_x"):+ DiffusionBackend._reusable_from_older_snapshot(+ "unsloth/Qwen-Image-2.1-FP8", ["te.safetensors"], "2" * 40, {"te.safetensors": 1}, token+ )++ assert tokens == [False, None, None, "hf_x"]++def test_download_plan_is_empty_when_every_required_file_is_cached(monkeypatch):_fake_flux_hub(monkeypatch)_all_cached(monkeypatch)