Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load image / video / text-encoder weights from .safetensors first, reuse a cached .pt / .bin when its sha256 matches the Hub (#17061)

Daniel Han2fb404b85ca0
diff --git a/studio/backend/core/inference/cached_weight_sha.py b/studio/backend/core/inference/cached_weight_sha.py
new file mode 100644
index 00000000000..bffc0167602
--- /dev/null
+++ b/studio/backend/core/inference/cached_weight_sha.py
@@ -0,0 +1,198 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Verify a cached ``.pt`` / ``.bin`` is byte-identical to the Hub's copy (LFS sha256) so a load can reuse it.
+
+Digests are memoised per (real path, size, mtime) in memory and on disk. Any unanswerable case (offline, no LFS
+record, error) is "not verified", so the caller keeps safetensors first. Never raises."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import threading
+import time
+from pathlib import Path
+from typing import Iterable, Optional
+
+ENV_DISABLE = "UNSLOTH_REUSE_CACHED_PICKLE_WEIGHTS"
+_CHUNK = 8 * 1024 * 1024
+# Hub misses retry after _MISS_TTL_S, branch answers after _MUTABLE_TTL_S; commit-id answers never expire.
+_MISS_TTL_S = 60.0
+_MUTABLE_TTL_S = 300.0
+_FILE_NAME = "weight_sha256.json"
+_MAX_ENTRIES = 512
+
+_lock = threading.Lock()
+_hub: dict = {}
+_local: dict = {}
+_persisted: Optional[dict] = None
+
+
+def reuse_enabled() -> bool:
+ return (os.environ.get(ENV_DISABLE) or "").strip().lower() not in ("0", "false", "no", "off")
+
+
+def _offline() -> bool:
+ try:
+ from huggingface_hub import constants
+ return bool(constants.HF_HUB_OFFLINE)
+ except Exception: # noqa: BLE001 - no hub library: nothing to compare against
+ return True
+
+
+def hub_sha256s(
+ repo_id: Optional[str],
+ filenames: Iterable[str],
+ *,
+ revision: Optional[str] = None,
+ token: Optional[str] = None,
+) -> dict:
+ """``{filename: sha256}`` for the LFS files of ``repo_id`` among ``filenames``, one Hub call for the uncached
+ ones. A file the Hub stores without an LFS record, or any failure, is simply absent from the answer."""
+ names = [str(n) for n in dict.fromkeys(filenames) if n]
+ if not repo_id or not names or _offline():
+ return {}
+ now = time.monotonic()
+ pinned = (
+ bool(revision)
+ and len(str(revision)) == 40
+ and all(c in "0123456789abcdef" for c in str(revision))
+ )
+ out: dict = {}
+ wanted: list = []
+ with _lock:
+ for name in names:
+ hit = _hub.get((repo_id, revision, name))
+ if (
+ hit is None
+ or (hit[0] is None and now - hit[1] > _MISS_TTL_S)
+ or (not pinned and now - hit[1] > _MUTABLE_TTL_S)
+ ):
+ wanted.append(name)
+ elif hit[0]:
+ out[name] = hit[0]
+ if not wanted:
+ return out
+ answered: dict = {}
+ try:
+ from huggingface_hub import HfApi
+ for entry in HfApi().get_paths_info(
+ repo_id, wanted, revision = revision, token = token or None
+ ):
+ lfs = getattr(entry, "lfs", None)
+ sha = lfs.get("sha256") if isinstance(lfs, dict) else getattr(lfs, "sha256", None)
+ path = getattr(entry, "path", None)
+ if path and isinstance(sha, str) and len(sha) == 64:
+ answered[path] = sha.lower()
+ except Exception: # noqa: BLE001 - offline, gated, rate limited: unverified, retried later
+ with _lock:
+ for name in wanted:
+ _hub[(repo_id, revision, name)] = (None, now)
+ return out
+ with _lock:
+ for name in wanted:
+ sha = answered.get(name)
+ # A listed non-LFS file is answered for good (""), a missing one retried after the TTL.
+ _hub[(repo_id, revision, name)] = (sha or None, now)
+ if sha:
+ out[name] = sha
+ return out
+
+
+def _store_path() -> Optional[Path]:
+ try:
+ from utils.paths.storage_roots import cache_root
+ return cache_root() / _FILE_NAME
+ except Exception: # noqa: BLE001 - outside the Studio backend: memory only
+ return None
+
+
+def _load_persisted() -> dict:
+ global _persisted
+ if _persisted is None:
+ _persisted = {}
+ path = _store_path()
+ try:
+ if path is not None and path.is_file():
+ data = json.loads(path.read_text(encoding = "utf-8"))
+ if isinstance(data, dict):
+ _persisted = {
+ k: v for k, v in data.items() if isinstance(v, list) and len(v) == 3
+ }
+ except Exception: # noqa: BLE001 - a bad record file is an empty one
+ _persisted = {}
+ return _persisted
+
+
+def _save_persisted(data: dict) -> None:
+ path = _store_path()
+ if path is None:
+ return
+ try:
+ path.parent.mkdir(parents = True, exist_ok = True)
+ keep = dict(list(data.items())[-_MAX_ENTRIES:])
+ tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
+ tmp.write_text(json.dumps(keep), encoding = "utf-8")
+ os.replace(tmp, path)
+ except Exception: # noqa: BLE001 - the digest is still remembered in memory
+ pass
+
+
+def file_sha256(path: str) -> Optional[str]:
+ """The sha256 of ``path``'s bytes (symlinks followed), from the record when the size and mtime still match."""
+ try:
+ real = os.path.realpath(path)
+ st = os.stat(real)
+ except OSError:
+ return None
+ key = (real, int(st.st_size), int(st.st_mtime_ns))
+ with _lock:
+ hit = _local.get(key)
+ if hit is not None:
+ return hit
+ rec = _load_persisted().get(real)
+ if rec and int(rec[0]) == key[1] and int(rec[1]) == key[2]:
+ _local[key] = str(rec[2])
+ return str(rec[2])
+ digest = hashlib.sha256()
+ try:
+ with open(real, "rb") as handle:
+ for chunk in iter(lambda: handle.read(_CHUNK), b""):
+ digest.update(chunk)
+ except OSError:
+ return None
+ sha = digest.hexdigest()
+ with _lock:
+ _local[key] = sha
+ data = _load_persisted()
+ data.pop(real, None)
+ data[real] = [key[1], key[2], sha]
+ _save_persisted(data)
+ return sha
+
+
+def cached_copy_matches_hub(
+ repo_id: Optional[str],
+ filename: Optional[str],
+ path: Optional[str],
+ *,
+ revision: Optional[str] = None,
+ token: Optional[str] = None,
+ expected: Optional[str] = None,
+) -> bool:
+ """True only when the Hub records an sha256 for ``repo_id/filename`` and ``path`` hashes to it."""
+ if not reuse_enabled() or not repo_id or not filename or not path:
+ return False
+ try:
+ sha = expected or hub_sha256s(repo_id, [filename], revision = revision, token = token).get(
+ filename
+ )
+ if not sha:
+ return False
+ # Always the bytes, never the cache blob's name: newer huggingface_hub caches name blobs by a Xet / etag
+ # hash that is not the file's sha256.
+ return file_sha256(path) == sha
+ except Exception: # noqa: BLE001 - unverified
+ return False
diff --git a/studio/backend/core/inference/diffusers_safetensors.py b/studio/backend/core/inference/diffusers_safetensors.py
new file mode 100644
index 00000000000..bda97d66672
--- /dev/null
+++ b/studio/backend/core/inference/diffusers_safetensors.py
@@ -0,0 +1,359 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""When a diffusers load may pass ``use_safetensors``, and which repo files a pipeline download skips.
+
+``True`` turns the silent pickle fallback into an error, so it is passed only when every weight-bearing component of
+the folder ships ``.safetensors``; an uninspectable folder keeps the library default. ``False`` is passed only to
+reuse cached ``.bin`` weights whose sha256 matches the Hub (or offline, where the default would refuse the load).
+No torch, never raises."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any, Optional
+
+_PICKLE_WEIGHT_SUFFIXES = (".pt", ".pth", ".ckpt", ".bin", ".pkl", ".pickle")
+
+
+def is_pickle_weight(name: Optional[Any]) -> bool:
+ return str(name or "").strip().lower().endswith(_PICKLE_WEIGHT_SUFFIXES)
+
+
+_MANIFESTS = ("model_index.json", "modular_model_index.json")
+_WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pt", ".pth", ".ckpt", ".pkl", ".pickle")
+# Pickle containers no diffusers / transformers component load reads: always skipped in a pipeline download.
+_NEVER_READ_PICKLES = (".pt", ".pth", ".ckpt", ".pkl", ".pickle")
+
+
+# The weight names a component load resolves (diffusers ``diffusion_pytorch_model``, transformers ``model``), single or
+# sharded. dtype-variant twins (``model.fp16.safetensors``) never match, and any other ``.safetensors``
+# (``adapter_model.safetensors``, an aux file) is not the component's weights.
+_MODEL_SAFETENSORS = re.compile(
+ r"^(diffusion_pytorch_model|model)(-\d+-of-\d+)?\.safetensors(\.index\.json)?$"
+)
+
+
+def _is_model_safetensors(name: str) -> bool:
+ return bool(_MODEL_SAFETENSORS.match(name.rsplit("/", 1)[-1]))
+
+
+# The pickle twins of those names (``diffusion_pytorch_model.bin``, ``pytorch_model-00001-of-00002.bin``).
+_MODEL_BIN = re.compile(r"^(diffusion_pytorch_model|pytorch_model)(-\d+-of-\d+)?\.bin$")
+
+
+def _is_model_bin(name: str) -> bool:
+ return bool(_MODEL_BIN.match(name.rsplit("/", 1)[-1]))
+
+
+def _snapshot_identity(root: Path) -> Optional[tuple]:
+ """``(repo_id, commit)`` of a Hub cache snapshot folder (``.../models--a--b/snapshots/<commit>``), else None."""
+ try:
+ parts = Path(os.path.realpath(str(root))).parts
+ if len(parts) < 3 or parts[-2] != "snapshots" or not parts[-3].startswith("models--"):
+ return None
+ return parts[-3][len("models--") :].replace("--", "/"), parts[-1]
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _hub_offline() -> bool:
+ try:
+ from huggingface_hub import constants
+ return bool(constants.HF_HUB_OFFLINE)
+ except Exception: # noqa: BLE001
+ return True
+
+
+def verified_bin_components(
+ root: Any,
+ components: Iterable[str],
+ *,
+ token: Optional[str] = None,
+ offline: bool = False,
+ revision: Optional[str] = None,
+) -> set:
+ """The ``components`` of the cached snapshot ``root`` whose weights are on disk only as ``.bin`` (no
+ ``.safetensors`` weight cached) and every such ``.bin`` hashes to the sha256 the Hub records for it at the
+ revision the load will read (``revision``, or the Hub's current ``main`` when the load is unpinned, never the
+ cached snapshot's possibly stale commit). Empty for a folder that is not a Hub cache snapshot or on any doubt. ``offline`` (nothing can
+ be downloaded, so the alternative to the cached ``.bin`` is a failed load) accepts the cached ``.bin`` unhashed."""
+ try:
+ from .cached_weight_sha import cached_copy_matches_hub, hub_sha256s, reuse_enabled
+
+ if not reuse_enabled():
+ return set()
+ ident = _snapshot_identity(Path(str(root)))
+ if ident is None:
+ return set()
+ repo_id, _commit = ident
+ offline = offline or _hub_offline()
+ wanted: dict = {}
+ for component in components:
+ folder = Path(str(root)) / component
+ if not folder.is_dir() or component_weight_state(folder) is not False:
+ continue
+ bins = [p.name for p in folder.iterdir() if p.is_file() and _is_model_bin(p.name)]
+ if bins:
+ wanted[component] = [f"{component}/{b}" for b in bins]
+ if not wanted:
+ return set()
+ if offline:
+ return set(wanted)
+ shas = hub_sha256s(
+ repo_id, [n for names in wanted.values() for n in names], revision = revision, token = token
+ )
+ return {
+ component
+ for component, names in wanted.items()
+ if all(
+ n in shas
+ and cached_copy_matches_hub(
+ repo_id, n, str(Path(str(root)) / n), revision = revision, expected = shas[n]
+ )
+ for n in names
+ )
+ }
+ except Exception: # noqa: BLE001 - unverified: safetensors stays first
+ return set()
+
+
+def component_weight_state(folder: Any) -> Optional[bool]:
+ """True: ``folder`` holds the component's own non-variant ``.safetensors`` weight (or its index); False: it holds
+ weights and none of them is that; None: no weight file at all (or unreadable)."""
+ try:
+ names = [p.name for p in Path(str(folder)).iterdir() if p.is_file()]
+ except OSError:
+ return None
+ weights = [
+ n for n in names if n.lower().endswith(_WEIGHT_SUFFIXES) or n.endswith(".index.json")
+ ]
+ if not weights:
+ return None
+ # A lone shard without its index is not loadable (an interrupted download): only a whole file or an index counts.
+ if any(
+ _is_model_safetensors(n) and not re.search(r"-\d+-of-\d+\.safetensors$", n) for n in weights
+ ):
+ return True
+ return False if any(not n.endswith(".index.json") for n in weights) else None
+
+
+def _manifest_components(root: Path) -> Optional[list]:
+ for manifest in _MANIFESTS:
+ path = root / manifest
+ if not path.is_file():
+ continue
+ try:
+ data = json.loads(path.read_text(encoding = "utf-8"))
+ except Exception: # noqa: BLE001 - unreadable manifest: unknown
+ return None
+ if not isinstance(data, dict):
+ return None
+ out = []
+ for key, value in data.items():
+ if key.startswith("_"):
+ continue
+ # classic manifests: [library, class]; modular ones: [library, class, {spec}] or nested specs
+ if isinstance(value, (list, tuple)) and value and value[0] is not None:
+ out.append((key, value[1] if len(value) > 1 else None))
+ return out
+ return None
+
+
+# Manifest classes that never hold weights (a scheduler / tokenizer folder is config only).
+_WEIGHTLESS_CLASS_SUFFIXES = (
+ "Scheduler",
+ "Tokenizer",
+ "TokenizerFast",
+ "Processor",
+ "FeatureExtractor",
+)
+
+
+def _weightless(class_name: Any) -> bool:
+ return isinstance(class_name, str) and class_name.endswith(_WEIGHTLESS_CLASS_SUFFIXES)
+
+
+def local_pipeline_safetensors_complete(root: Any, passed: Iterable[str] = ()) -> bool:
+ """True when ``root`` is a local diffusers pipeline folder whose every manifest component that holds weights
+ (and is not supplied pre-built in ``passed``) has a ``.safetensors`` weight. False when unknown (a weight-bearing
+ component missing or without weights, e.g. a partly cached snapshot) or any component is pickle-only."""
+ try:
+ base = Path(str(root)).expanduser()
+ if not base.is_dir():
+ return False
+ except (OSError, ValueError):
+ return False
+ components = _manifest_components(base)
+ if components is None:
+ return False
+ skip = set(passed or ())
+ seen = False
+ for name, class_name in components:
+ if name in skip:
+ continue
+ folder = base / name
+ state = component_weight_state(folder) if folder.is_dir() else None
+ if state is False:
+ return False
+ if state is None:
+ if _weightless(class_name):
+ continue
+ return False
+ seen = True
+ return seen
+
+
+def _cached_snapshot_dir(
+ repo_id: str,
+ probe: str,
+ cache_dir: Optional[str],
+ revision: Optional[str] = None,
+) -> Optional[Path]:
+ """The cached snapshot folder of ``revision`` (default: the cached ``main``) holding ``repo_id/probe``, or None.
+ Offline: reads the cache only."""
+ try:
+ from huggingface_hub import try_to_load_from_cache
+ except Exception: # noqa: BLE001
+ return None
+ for root in dict.fromkeys((cache_dir, None)):
+ try:
+ hit = try_to_load_from_cache(
+ repo_id, probe, cache_dir = root, **({"revision": revision} if revision else {})
+ )
+ except Exception: # noqa: BLE001
+ continue
+ if isinstance(hit, str) and os.path.isfile(hit):
+ return Path(hit).parents[len(probe.split("/")) - 1]
+ return None
+
+
+def _takes_use_safetensors(cls: Any, base_name: str) -> bool:
+ """Only diffusers' own ``DiffusionPipeline`` / ``ModelMixin`` subclasses are known to accept the kwarg."""
+ if cls is None:
+ return True
+ try:
+ import diffusers
+ base = getattr(diffusers, base_name, None)
+ return isinstance(cls, type) and base is not None and issubclass(cls, base)
+ except Exception: # noqa: BLE001
+ return False
+
+
+def pipeline_safetensors_kwargs(
+ source: Any,
+ *,
+ passed: Iterable[str] = (),
+ cache_dir: Optional[str] = None,
+ pipeline_cls: Any = None,
+) -> dict:
+ """``{"use_safetensors": True}`` for a pipeline ``from_pretrained(source)`` whose folder (``source`` itself, or the
+ cached snapshot of a Hub id) ships safetensors for every weight-bearing component; ``{}`` otherwise (and for a
+ ``pipeline_cls`` that is not a diffusers ``DiffusionPipeline``)."""
+ try:
+ if not _takes_use_safetensors(pipeline_cls, "DiffusionPipeline"):
+ return {}
+ text = str(source or "").strip()
+ if not text:
+ return {}
+ if os.path.isdir(os.path.expanduser(text)):
+ root: Optional[Path] = Path(text).expanduser()
+ # A snapshot folder is read as is: its own commit is the revision the load reads.
+ revision = (_snapshot_identity(root) or (None, None))[1]
+ else:
+ root = None
+ # The snapshot the load reads: a pinned ``revision`` among the load's kwargs, else the cached main (whose
+ # .bin files are then checked against the Hub's current main, what an unpinned load resolves to).
+ revision = passed.get("revision") if isinstance(passed, dict) else None
+ for manifest in _MANIFESTS:
+ root = _cached_snapshot_dir(text, manifest, cache_dir, revision)
+ if root is not None:
+ break
+ if root is not None and local_pipeline_safetensors_complete(root, passed):
+ return {"use_safetensors": True}
+ variant = passed.get("variant") if isinstance(passed, dict) else None
+ # A variant load reads ``*.<variant>.*`` files, which the default-name .bin check says nothing about.
+ if root is not None and not variant and _snapshot_identity(root) is not None:
+ # Every weight component already cached as a Hub-verified .bin: reuse it, download nothing new.
+ components = _manifest_components(root) or []
+ skip = set(passed or ())
+ needed = [
+ name
+ for name, class_name in components
+ if name not in skip and not _weightless(class_name)
+ ]
+ token = passed.get("token") if isinstance(passed, dict) else None
+ offline = bool(passed.get("local_files_only")) if isinstance(passed, dict) else False
+ if needed and verified_bin_components(
+ root, needed, token = token, offline = offline, revision = revision
+ ) == set(needed):
+ return {"use_safetensors": False}
+ except Exception: # noqa: BLE001 - unknown keeps the library default
+ pass
+ return {}
+
+
+def component_safetensors_kwargs(
+ source: Any,
+ subfolder: str,
+ *,
+ cache_dir: Optional[str] = None,
+ model_cls: Any = None,
+ local_files_only: bool = False,
+ revision: Optional[str] = None,
+) -> dict:
+ """``{"use_safetensors": True}`` for ``from_pretrained(source, subfolder = subfolder)`` when that component folder
+ (local, or in the cached snapshot of a Hub id) holds a safetensors weight; ``{}`` otherwise (and for a
+ ``model_cls`` that is not a diffusers ``ModelMixin``)."""
+ try:
+ if not _takes_use_safetensors(model_cls, "ModelMixin"):
+ return {}
+ text = str(source or "").strip()
+ if not text or not subfolder:
+ return {}
+ if os.path.isdir(os.path.expanduser(text)):
+ folder: Optional[Path] = Path(text).expanduser() / subfolder
+ revision = (_snapshot_identity(folder.parent) or (None, revision))[1]
+ else:
+ snap = _cached_snapshot_dir(text, f"{subfolder}/config.json", cache_dir, revision)
+ folder = snap / subfolder if snap is not None else None
+ if folder is not None and component_weight_state(folder) is True:
+ return {"use_safetensors": True}
+ if folder is not None and verified_bin_components(
+ folder.parent, [subfolder], offline = local_files_only, revision = revision
+ ):
+ return {"use_safetensors": False}
+ except Exception: # noqa: BLE001
+ pass
+ return {}
+
+
+def pipeline_pickle_files(filenames: Iterable[str]) -> list:
+ """Pipeline repo files to skip: pickles in a component folder or beside a same-stem root ``.safetensors``, and
+ ``.bin`` beside the component's own safetensors. A pickle with no twin is the only copy and stays."""
+ names = [n for n in filenames if isinstance(n, str)]
+ if not any(n in _MANIFESTS for n in names):
+ return []
+ st_dirs = {n.rsplit("/", 1)[0] if "/" in n else "" for n in names if _is_model_safetensors(n)}
+ st_stems = {
+ n[: -len(".safetensors")].lower() for n in names if n.lower().endswith(".safetensors")
+ }
+ out = []
+ for n in names:
+ low = n.lower()
+ folder = n.rsplit("/", 1)[0] if "/" in n else ""
+ if low.endswith(_NEVER_READ_PICKLES):
+ if folder or low.rsplit(".", 1)[0] in st_stems:
+ out.append(n)
+ elif (low.endswith(".bin") or low.endswith(".bin.index.json")) and folder in st_dirs:
+ out.append(n)
+ return out
+
+
+def is_never_read_pickle(name: Optional[str]) -> bool:
+ """A pickle container (``.pt`` / ``.pth`` / ``.ckpt`` / ``.pkl``) no diffusers component load reads."""
+ return bool(name) and is_pickle_weight(name) and str(name).lower().endswith(_NEVER_READ_PICKLES)
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 84cef6d0ed9..3e57c1c1cb6 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -486,6 +486,13 @@ from .diffusion_comfy_quant import (
refuse_comfy_quant,
)
from .diffusion_single_file_trust import assert_safetensors_file, single_file_load_allowed
+from .diffusers_safetensors import (
+ component_safetensors_kwargs,
+ component_weight_state,
+ is_never_read_pickle,
+ pipeline_pickle_files,
+ pipeline_safetensors_kwargs,
+)
from .diffusion_prequant import (
PrequantIntoError,
PrequantSource,
@@ -6961,7 +6968,12 @@ class DiffusionBackend:
)
# Under the repo it describes: a mirror's commit says nothing about a cached upstream snapshot.
_record_revision(revisions_out, listing_repo, info)
- return [s for s in info.siblings if not _listing_extra(listing_repo, s.rfilename)]
+ skipped = set(pipeline_pickle_files(s.rfilename for s in info.siblings))
+ return [
+ s
+ for s in info.siblings
+ if not _listing_extra(listing_repo, s.rfilename) and s.rfilename not in skipped
+ ]
listing_repo = prefer_ungated_mirror(base_repo, hf_token)
base_siblings = _base_listing(listing_repo)
@@ -9912,7 +9924,14 @@ class DiffusionBackend:
try:
with _converted_shards_pread(dtype, small_host):
pipe = pipeline_cls.from_pretrained(
- _base_local_dir or fetch_base, **pipe_kwargs
+ _base_local_dir or fetch_base,
+ **pipe_kwargs,
+ **pipeline_safetensors_kwargs(
+ _base_local_dir or fetch_base,
+ passed = pipe_kwargs,
+ cache_dir = pipe_kwargs.get("cache_dir"),
+ pipeline_cls = pipeline_cls,
+ ),
)
finally:
self._stop_load_prefetch()
@@ -10278,7 +10297,14 @@ class DiffusionBackend:
)
with _converted_shards_pread(dtype, small_host):
pipe = pipeline_cls.from_pretrained(
- _base_local_dir or fetch_base, **pipe_kwargs
+ _base_local_dir or fetch_base,
+ **pipe_kwargs,
+ **pipeline_safetensors_kwargs(
+ _base_local_dir or fetch_base,
+ passed = pipe_kwargs,
+ cache_dir = pipe_kwargs.get("cache_dir"),
+ pipeline_cls = pipeline_cls,
+ ),
)
if small_host is not None and small_host.engaged:
plan = self._apply_small_host_route(
@@ -11957,6 +11983,13 @@ class DiffusionBackend:
cache_dir = hub_cache_dir(),
# Dense bf16 transformer is the largest fetch on this path, so API-initiated loads are refused here.
local_files_only = local_files_only,
+ **component_safetensors_kwargs(
+ fetch_base,
+ "transformer",
+ cache_dir = hub_cache_dir(),
+ model_cls = transformer_cls,
+ local_files_only = local_files_only,
+ ),
)
check_cancelled()
pipe = self._assemble_pipe(
@@ -12444,7 +12477,16 @@ class DiffusionBackend:
)
)
with _converted_shards_pread(dtype, None):
- pipe = pipeline_cls.from_pretrained(base_local_dir or base, **pipe_kwargs)
+ pipe = pipeline_cls.from_pretrained(
+ base_local_dir or base,
+ **pipe_kwargs,
+ **pipeline_safetensors_kwargs(
+ base_local_dir or base,
+ passed = pipe_kwargs,
+ cache_dir = pipe_kwargs.get("cache_dir"),
+ pipeline_cls = pipeline_cls,
+ ),
+ )
check_cancelled()
pipe.to(device)
return pipe
@@ -13626,7 +13668,7 @@ class DiffusionBackend:
cn_dtype = getattr(torch, str(state.dtype).replace("torch.", ""), None)
# Force safetensors for untrusted remote repos; if the Hub scan failed open, pickles still load.
cn_from_pretrained_kwargs: dict[str, Any] = {"cache_dir": hub_cache_dir()}
- if remote_cn:
+ if remote_cn or component_weight_state(resolved_cn.path) is True:
cn_from_pretrained_kwargs["use_safetensors"] = True
cn_model = getattr(diffusers, model_cls_name).from_pretrained(
resolved_cn.path,
@@ -16385,6 +16427,8 @@ def _base_file_downloaded(
# from_single_file(config=repo, subfolder="transformer") always fetches this config. Video keeps
# the same exception in video.py's snapshot filter.
return True
+ if is_never_read_pickle(rfilename):
+ return False
if rfilename.startswith("transformer/"):
return include_transformer
if "/" not in rfilename: # top-level: only the pipeline manifest is fetched
@@ -16432,7 +16476,9 @@ def _pipeline_file_downloaded(rfilename: str) -> bool:
name = lower.rsplit("/", 1)[1]
if name.startswith(("openvino_", "flax_")):
return False
- if name.endswith((".onnx", ".onnx_data", ".pb", ".msgpack", ".h5", ".ckpt")):
+ if name.endswith((".onnx", ".onnx_data", ".pb", ".msgpack", ".h5")) or is_never_read_pickle(
+ name
+ ):
return False
if ".fp16." in name or ".bf16." in name or ".non_ema." in name:
return False
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 50b4b3dab01..090ffa75125 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -640,7 +640,9 @@ def prefer_cached_pickle_twins(
log: bool = True,
roots: Optional[Sequence[Optional[str]]] = None,
) -> list:
- """Same-stem cached .pt/.pth moves ahead of its uncached safetensors twin; the twin stays behind it as 404 fallback; PREFER_SAFETENSORS_ENV disables."""
+ """Safetensors stays first, except a same-stem cached .pt/.pth whose sha256 matches the Hub's record moves ahead of
+ its uncached safetensors twin (reused, not re-downloaded); the twin stays behind it as 404 fallback. An unverified
+ cached pickle (offline, no LFS record, mismatch) keeps its place behind the twin. PREFER_SAFETENSORS_ENV disables."""
import os
out = [n for n in names if n]
@@ -659,6 +661,18 @@ def prefer_cached_pickle_twins(
except Exception: # noqa: BLE001 - an unanswerable question is a no
return False
+ def _verified(name: str) -> bool:
+ from .cached_weight_sha import cached_copy_matches_hub
+
+ if _LOCAL_FILES_ONLY.get():
+ return False # a load that may not reach the Hub cannot ask it for the hash
+ token = _LOAD_HF_TOKEN.get()
+ for root in roots:
+ hit = _hub_name_cached(repo_id, name, root)
+ if hit is not None and cached_copy_matches_hub(repo_id, name, hit, token = token):
+ return True
+ return False
+
# A cached container of an artifact beats an uncached one of the SAME artifact: never download a second copy.
for comfy in [n for n in out if is_comfy_prequant_filename(n)]:
stem = _artifact_stem(comfy)
@@ -711,7 +725,8 @@ def prefer_cached_pickle_twins(
),
None,
)
- if twin is None or _cached(st):
+ # Only a cached pickle the Hub's sha256 vouches for is reused; unverified, safetensors stays first.
+ if twin is None or _cached(st) or not _verified(twin):
continue
out.remove(twin)
out.insert(out.index(st), twin)
@@ -723,8 +738,8 @@ def prefer_cached_pickle_twins(
import logging
sink = logging.getLogger(__name__)
sink.info(
- "diffusion.prequant_cached_pickle: %s: using the cached %s; the %s twin is not "
- "downloaded (set %s=1 to fetch it instead)",
+ "diffusion.prequant_cached_pickle: %s: using the cached %s (sha256 matches the Hub); the "
+ "%s twin is not downloaded (set %s=1 to fetch it instead)",
repo_id,
twin,
st,
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index c00520d834b..5a59fa8c8c5 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -1561,6 +1561,9 @@ def _fetch_dense_components(
fetched = snapshot_download(
repo_id,
allow_patterns = [f"{c}/*" for c in components],
+ ignore_patterns = _dense_component_pickles(
+ repo_id, components, revision, hf_token, cache_dir
+ ),
token = hf_token,
cache_dir = cache_dir,
revision = revision,
@@ -1573,6 +1576,60 @@ def _fetch_dense_components(
return False
+def _dense_component_pickles(
+ repo_id: str,
+ components: list,
+ revision: Optional[str],
+ hf_token: Optional[str],
+ cache_dir: Optional[str] = None,
+) -> list:
+ """ignore_patterns for a dense encoder fetch: every .pt / .pth / .ckpt / .pkl, plus a component's .bin shards when
+ the repo lists a safetensors weight for it. A component whose .bin is already cached and matches the Hub's sha256
+ (and whose safetensors is not cached) skips its safetensors instead, so nothing is downloaded twice. Without a
+ listing only the never-read pickles are dropped, so a component whose only copy is a .bin still loads."""
+ from .diffusers_safetensors import (
+ _is_model_safetensors,
+ pipeline_pickle_files,
+ verified_bin_components,
+ )
+
+ patterns = [
+ f"{c}/*{suffix}"
+ for c in components
+ for suffix in (".pt", ".pth", ".ckpt", ".pkl", ".pickle")
+ ]
+ try:
+ from huggingface_hub import HfApi
+ names = HfApi().list_repo_files(repo_id, revision = revision, token = hf_token)
+ except Exception: # noqa: BLE001 - no listing: the .bin of a component without safetensors must stay fetchable
+ return patterns
+ reused: set = set()
+ try:
+ import os
+
+ from huggingface_hub import try_to_load_from_cache
+
+ hit = try_to_load_from_cache(
+ repo_id, "model_index.json", cache_dir = cache_dir, revision = revision
+ )
+ if isinstance(hit, str):
+ reused = verified_bin_components(
+ os.path.dirname(hit), components, token = hf_token, revision = revision
+ )
+ except Exception: # noqa: BLE001 - unverified: safetensors stays first
+ reused = set()
+ wanted = tuple(f"{c}/" for c in components)
+ for n in pipeline_pickle_files(list(names) + ["model_index.json"]):
+ if n.startswith(wanted) and n.split("/", 1)[0] not in reused:
+ patterns.append(n)
+ for n in names:
+ if n.split("/", 1)[0] in reused and (
+ _is_model_safetensors(n) or n.endswith(".safetensors.index.json")
+ ):
+ patterns.append(n)
+ return patterns
+
+
def _held_locally(repo_id: str, name: Optional[str], hf_token: Optional[str]) -> bool:
"""Whether ``repo_id/name`` is on this machine, without touching the network: the operator's mirror, or either
cache root the loader reuses (the live one and huggingface_hub's import-time one, as _download_checkpoint_name)."""
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 06e877fe6f6..92670fec5b1 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -75,6 +75,7 @@ from .diffusion_comfy_quant import (
refuse_comfy_quant,
)
from .diffusion_single_file_trust import assert_safetensors_file, single_file_load_allowed
+from .diffusers_safetensors import component_safetensors_kwargs, pipeline_safetensors_kwargs
from .diffusion_prequant import scoped_local_files_only
from .video_moe_pair import moe_expert_of, moe_partner_filename, moe_pick_pairs
from .diffusion_cache import (
@@ -10042,6 +10043,15 @@ class VideoBackend:
else {}
)
+ def _safetensors_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
+ """use_safetensors=True when the base folder ships safetensors for every component this load reads."""
+ return pipeline_safetensors_kwargs(
+ _base_local_dir or fetch_base,
+ passed = {**_pinned_base_kwargs(), **kwargs},
+ cache_dir = kwargs.get("cache_dir"),
+ pipeline_cls = pipeline_cls,
+ )
+
pipe_kwargs: dict[str, Any] = {
"torch_dtype": dtype,
"cache_dir": hub_cache_dir(),
@@ -10262,6 +10272,7 @@ class VideoBackend:
_base_local_dir or fetch_base,
**_pinned_base_kwargs(),
**pipe_kwargs,
+ **_safetensors_kwargs(pipe_kwargs),
)
except Exception as exc:
refusal = (
@@ -10494,6 +10505,7 @@ class VideoBackend:
_base_local_dir or fetch_base,
**_pinned_base_kwargs(),
**{**pipe_kwargs, "transformer": ltx25_transformer},
+ **_safetensors_kwargs({**pipe_kwargs, "transformer": ltx25_transformer}),
)
else:
@@ -10539,6 +10551,7 @@ class VideoBackend:
_base_local_dir or fetch_base,
**_pinned_base_kwargs(),
**{**pipe_kwargs, **experts},
+ **_safetensors_kwargs({**pipe_kwargs, **experts}),
)
# Dtype dict already loads the Wan VAE in float32; this backstops any path bypassing it (e.g. vae=).
@@ -10659,6 +10672,14 @@ class VideoBackend:
local_files_only = local_files_only,
**({"token": hf_token} if hf_token else {}),
**_pinned_base_kwargs(),
+ **component_safetensors_kwargs(
+ _base_local_dir or fetch_base,
+ component,
+ cache_dir = hub_cache_dir(),
+ model_cls = dense_cls,
+ local_files_only = local_files_only,
+ revision = _pinned_base_kwargs().get("revision"),
+ ),
)
}
)
diff --git a/studio/backend/hub/utils/snapshot_filters.py b/studio/backend/hub/utils/snapshot_filters.py
index 6769faa96ab..c94bc177875 100644
--- a/studio/backend/hub/utils/snapshot_filters.py
+++ b/studio/backend/hub/utils/snapshot_filters.py
@@ -163,9 +163,19 @@ def resolve_snapshot_ignore_patterns_for_files(filenames: Iterable[str]) -> list
# Only when the root is itself a checkpoint, or the folders are the only runnable copies.
if "config.json" in names:
ignore.extend(f"{folder}/*" for folder in precision_variant_subfolders(names))
+ ignore.extend(diffusers_pickle_files(names))
return ignore
+def diffusers_pickle_files(filenames: Iterable[str]) -> list[str]:
+ """For a diffusers pipeline repo (``model_index.json`` / ``modular_model_index.json`` at its root): the pickles a
+ safetensors copy makes redundant, as exact names (a component folder's .pt / .pth / .ckpt / .pkl, a root one beside
+ its same-stem .safetensors, a .bin beside the component's safetensors weights). A pickle that is the only copy
+ stays. Empty for every other repo (LLM, audio, datasets keep their own rules above)."""
+ from core.inference.diffusers_safetensors import pipeline_pickle_files
+ return pipeline_pickle_files(filenames)
+
+
def sibling_matches_ignore(filename: str, ignore_patterns: Iterable[str]) -> bool:
return any(fnmatchcase(filename, pattern) for pattern in ignore_patterns)
diff --git a/studio/backend/tests/test_cached_pickle_sha_reuse.py b/studio/backend/tests/test_cached_pickle_sha_reuse.py
new file mode 100644
index 00000000000..45b65860a3e
--- /dev/null
+++ b/studio/backend/tests/test_cached_pickle_sha_reuse.py
@@ -0,0 +1,478 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Safetensors first; a .pt / .bin already on disk is reused only when its sha256 matches the Hub's LFS record.
+
+The Hub boundary is faked at huggingface_hub itself (``HfApi.get_paths_info``, ``try_to_load_from_cache``,
+``hf_hub_download``) and the local files are really hashed, so these run unchanged against any tree."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import types
+
+import huggingface_hub
+import pytest
+from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
+
+import core.inference.diffusion_prequant as pq
+
+REPO = "unsloth/Model-INT8"
+PT_BYTES = b"pickle checkpoint bytes"
+
+
+def _sha(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+class Hub:
+ """Cache + listing + LFS records for one repo; records downloads and metadata calls."""
+
+ def __init__(self, root):
+ self.root = root
+ self.cached: dict = {}
+ self.lfs: dict = {}
+ self.hosted: set = set()
+ self.downloads: list = []
+ self.info_calls: list = []
+ self.offline = False
+
+ def put(
+ self,
+ name,
+ data,
+ *,
+ cache = True,
+ record = True,
+ ):
+ self.hosted.add(name)
+ if record:
+ self.lfs[name] = _sha(data)
+ if cache:
+ path = self.root / "cache" / name
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_bytes(data)
+ self.cached[name] = str(path)
+
+ def try_to_load_from_cache(
+ self,
+ repo_id,
+ filename,
+ cache_dir = None,
+ **_,
+ ):
+ return self.cached.get(filename) if repo_id == REPO else None
+
+ def hf_hub_download(
+ self,
+ repo_id,
+ filename,
+ local_files_only = False,
+ **_,
+ ):
+ hit = self.cached.get(filename)
+ if local_files_only or self.offline:
+ self.downloads.append((filename, "local"))
+ if hit is None:
+ raise LocalEntryNotFoundError(f"{filename} not cached")
+ return hit
+ if filename not in self.hosted:
+ raise EntryNotFoundError(f"404 {filename}")
+ if hit is None:
+ self.downloads.append((filename, "fetched"))
+ path = self.root / "cache" / filename
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_bytes(b"fresh " + filename.encode())
+ self.cached[filename] = str(path)
+ return str(path)
+ self.downloads.append((filename, "revalidated"))
+ return hit
+
+ def get_paths_info(self, repo_id, paths, **_):
+ self.info_calls.append(tuple(paths))
+ if self.offline:
+ raise OSError("no network")
+ out = []
+ for p in paths:
+ if p in self.hosted:
+ sha = self.lfs.get(p)
+ lfs = {"size": 1, "sha256": sha, "pointer_size": 1} if sha else None
+ out.append(types.SimpleNamespace(path = p, lfs = lfs))
+ return out
+
+ @property
+ def fetched(self):
+ return [n for n, how in self.downloads if how == "fetched"]
+
+
+def hub(tmp_path, monkeypatch):
+ fake = Hub(tmp_path)
+ monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", fake.try_to_load_from_cache)
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake.hf_hub_download)
+ monkeypatch.setattr(
+ huggingface_hub.HfApi, "get_paths_info", lambda self, *a, **k: fake.get_paths_info(*a, **k)
+ )
+ monkeypatch.setattr(
+ pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True
+ )
+ monkeypatch.delenv(pq.PREQUANT_MIRROR_ENV, raising = False)
+ monkeypatch.delenv("UNSLOTH_PREQUANT_PREFER_SAFETENSORS", raising = False)
+ monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))
+ getattr(pq, "_logged_twin_choices", set()).clear()
+ try:
+ import core.inference.cached_weight_sha as cws
+
+ cws._hub.clear()
+ cws._local.clear()
+ monkeypatch.setattr(cws, "_persisted", None)
+ except ImportError:
+ pass
+ return fake
+
+
+def _src():
+ return pq.PrequantSource(
+ kind = "repo",
+ location = REPO,
+ filename = "Model-INT8.safetensors",
+ fallback_filenames = ("Model-INT8.pt",),
+ )
+
+
+def _resolve(**kw):
+ return pq._resolve_checkpoint_path(_src(), None, None, scheme = "int8", **kw)
+
+
+def test_safetensors_is_downloaded_when_nothing_is_cached(hub):
+ hub.put("Model-INT8.safetensors", b"st", cache = False)
+ hub.put("Model-INT8.pt", PT_BYTES, cache = False)
+ assert _resolve().endswith("Model-INT8.safetensors")
+ assert hub.fetched == ["Model-INT8.safetensors"]
+
+
+def test_a_cached_pickle_matching_the_hub_sha_is_reused_without_a_download(hub):
+ hub.put("Model-INT8.safetensors", b"st", cache = False)
+ hub.put("Model-INT8.pt", PT_BYTES)
+ assert _resolve() == hub.cached["Model-INT8.pt"]
+ assert hub.fetched == []
+
+
+def test_a_cached_pickle_with_a_different_sha_is_not_reused(hub):
+ hub.put("Model-INT8.safetensors", b"st", cache = False)
+ hub.put("Model-INT8.pt", PT_BYTES)
+ hub.lfs["Model-INT8.pt"] = _sha(b"the Hub has a newer upload")
+ assert _resolve().endswith("Model-INT8.safetensors")
+ assert hub.fetched == ["Model-INT8.safetensors"]
+
+
+def test_a_cached_pickle_without_an_lfs_record_is_not_reused(hub):
+ hub.put("Model-INT8.safetensors", b"st", cache = False)
+ hub.put("Model-INT8.pt", PT_BYTES, record = False)
+ assert _resolve().endswith("Model-INT8.safetensors")
+ assert hub.fetched == ["Model-INT8.safetensors"]
+
+
+def test_offline_the_hash_is_unanswerable_so_safetensors_is_tried_first(hub, monkeypatch):
+ hub.put("Model-INT8.safetensors", b"st", cache = False)
+ hub.put("Model-INT8.pt", PT_BYTES)
+ hub.offline = True
+ # Fail open: the safetensors name is asked for first; it cannot be fetched offline, so the cached pickle loads.
+ assert _resolve() == hub.cached["Model-INT8.pt"]
+ assert [n for n, _ in hub.downloads][0] == "Model-INT8.safetensors"
+ assert hub.fetched == []
+
+
+def test_a_local_files_only_load_never_asks_the_hub_for_the_hash(hub):
+ hub.put("Model-INT8.safetensors", b"st", cache = False)
+ hub.put("Model-INT8.pt", PT_BYTES)
+ token = pq._LOCAL_FILES_ONLY.set(True)
+ try:
+ assert _resolve(local_files_only = True) == hub.cached["Model-INT8.pt"]
+ finally:
+ pq._LOCAL_FILES_ONLY.reset(token)
+ assert hub.info_calls == []
+ assert hub.fetched == []
+
+
+def test_a_pickle_only_repo_still_loads_its_pickle(hub):
+ hub.put("Model-INT8.pt", PT_BYTES, cache = False)
+ assert _resolve().endswith("Model-INT8.pt")
+ assert hub.fetched == ["Model-INT8.pt"]
+
+
+def test_both_cached_safetensors_wins_and_nothing_is_hashed(hub):
+ hub.put("Model-INT8.safetensors", b"st")
+ hub.put("Model-INT8.pt", PT_BYTES)
+ assert _resolve() == hub.cached["Model-INT8.safetensors"]
+ assert hub.info_calls == []
+
+
+# --- the hash helper ---------------------------------------------------------------------------------------------
+
+
+def test_the_local_hash_is_computed_once_and_remembered(hub, tmp_path, monkeypatch):
+ import core.inference.cached_weight_sha as cws
+
+ path = tmp_path / "w.bin"
+ path.write_bytes(PT_BYTES)
+ opened = []
+ real_open = open
+
+ def counting_open(p, *a, **k):
+ if str(p) == os.path.realpath(path):
+ opened.append(p)
+ return real_open(p, *a, **k)
+
+ monkeypatch.setattr("builtins.open", counting_open)
+ assert cws.file_sha256(str(path)) == _sha(PT_BYTES)
+ assert cws.file_sha256(str(path)) == _sha(PT_BYTES)
+ assert len(opened) == 1
+ # A fresh process reads the persisted record instead of rehashing.
+ cws._local.clear()
+ monkeypatch.setattr(cws, "_persisted", None)
+ assert cws.file_sha256(str(path)) == _sha(PT_BYTES)
+ assert len(opened) == 1
+ record = json.loads(cws._store_path().read_text())
+ assert record[os.path.realpath(path)][2] == _sha(PT_BYTES)
+ # A changed file is rehashed.
+ path.write_bytes(b"changed!" * 10)
+ assert cws.file_sha256(str(path)) == _sha(b"changed!" * 10)
+
+
+def test_a_blob_is_judged_by_its_bytes_not_its_name(hub, tmp_path):
+ import core.inference.cached_weight_sha as cws
+
+ # huggingface_hub 1.x names cache blobs by a Xet hash, not the sha256.
+ blob = tmp_path / "blobs" / "c5" / ("c" * 64)
+ blob.parent.mkdir(parents = True)
+ blob.write_bytes(PT_BYTES)
+ assert cws.cached_copy_matches_hub(REPO, "x.pt", str(blob), expected = _sha(PT_BYTES))
+ assert not cws.cached_copy_matches_hub(REPO, "x.pt", str(blob), expected = _sha(b"other"))
+
+
+def test_the_reuse_switch_turns_verification_off(hub, monkeypatch):
+ import core.inference.cached_weight_sha as cws
+
+ hub.put("Model-INT8.safetensors", b"st", cache = False)
+ hub.put("Model-INT8.pt", PT_BYTES)
+ monkeypatch.setenv(cws.ENV_DISABLE, "0")
+ assert _resolve().endswith("Model-INT8.safetensors")
+ assert hub.fetched == ["Model-INT8.safetensors"]
+
+
+def test_hub_lookups_are_cached_per_process(hub):
+ import core.inference.cached_weight_sha as cws
+
+ hub.put("Model-INT8.pt", PT_BYTES)
+ assert cws.hub_sha256s(REPO, ["Model-INT8.pt"]) == {"Model-INT8.pt": _sha(PT_BYTES)}
+ assert cws.hub_sha256s(REPO, ["Model-INT8.pt"]) == {"Model-INT8.pt": _sha(PT_BYTES)}
+ assert len(hub.info_calls) == 1
+
+
+# --- diffusers pipelines: a cached, verified .bin is loaded instead of downloading safetensors --------------------
+
+
+def _snapshot(tmp_path, components):
+ root = tmp_path / "hub" / "models--o--pipe" / "snapshots" / ("c" * 40)
+ root.mkdir(parents = True)
+ index = {
+ "_class_name": "FluxPipeline",
+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
+ }
+ for name, files in components.items():
+ index[name] = ["diffusers", "Y"]
+ (root / name).mkdir()
+ (root / name / "config.json").write_text("{}")
+ for fname, data in files.items():
+ (root / name / fname).write_bytes(data)
+ (root / "scheduler").mkdir()
+ (root / "model_index.json").write_text(json.dumps(index))
+ return root
+
+
+def _hub_records(
+ monkeypatch,
+ records,
+ calls = None,
+):
+ def info(self, repo_id, paths, **kw):
+ if calls is not None:
+ calls.append((repo_id, kw.get("revision"), tuple(paths)))
+ return [
+ types.SimpleNamespace(path = p, lfs = {"sha256": records[p]} if records.get(p) else None)
+ for p in paths
+ ]
+
+ monkeypatch.setattr(huggingface_hub.HfApi, "get_paths_info", info)
+
+
+def test_a_snapshot_of_verified_bins_loads_them_instead_of_downloading_safetensors(
+ hub, tmp_path, monkeypatch
+):
+ from core.inference import diffusers_safetensors as ds
+
+ root = _snapshot(
+ tmp_path,
+ {
+ "transformer": {"diffusion_pytorch_model.bin": b"t"},
+ "text_encoder": {"pytorch_model.bin": b"e"},
+ },
+ )
+ calls = []
+ _hub_records(
+ monkeypatch,
+ {
+ "transformer/diffusion_pytorch_model.bin": _sha(b"t"),
+ "text_encoder/pytorch_model.bin": _sha(b"e"),
+ },
+ calls,
+ )
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": False}
+ # A snapshot folder is read as is: checked against its own commit.
+ assert calls and calls[0][0] == "o/pipe" and calls[0][1] == "c" * 40
+ assert ds.component_safetensors_kwargs(str(root), "transformer") == {"use_safetensors": False}
+
+
+def test_one_mismatched_bin_keeps_the_library_default(hub, tmp_path, monkeypatch):
+ from core.inference import diffusers_safetensors as ds
+
+ root = _snapshot(
+ tmp_path,
+ {
+ "transformer": {"diffusion_pytorch_model.bin": b"t"},
+ "text_encoder": {"pytorch_model.bin": b"e"},
+ },
+ )
+ _hub_records(
+ monkeypatch,
+ {
+ "transformer/diffusion_pytorch_model.bin": _sha(b"t"),
+ "text_encoder/pytorch_model.bin": _sha(b"other"),
+ },
+ )
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+ assert ds.component_safetensors_kwargs(str(root), "text_encoder") == {}
+
+
+def test_a_partly_safetensors_snapshot_never_forces_bins(hub, tmp_path, monkeypatch):
+ from core.inference import diffusers_safetensors as ds
+
+ root = _snapshot(
+ tmp_path,
+ {
+ "transformer": {"diffusion_pytorch_model.bin": b"t"},
+ "vae": {"diffusion_pytorch_model.safetensors": b"v"},
+ },
+ )
+ _hub_records(monkeypatch, {"transformer/diffusion_pytorch_model.bin": _sha(b"t")})
+ # The vae has no .bin to reuse, so use_safetensors=False would fetch one: keep the default.
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+
+
+def test_offline_a_snapshot_cached_only_as_bins_loads_them(hub, tmp_path, monkeypatch):
+ from core.inference import diffusers_safetensors as ds
+ import huggingface_hub.constants as constants
+
+ root = _snapshot(tmp_path, {"transformer": {"diffusion_pytorch_model.bin": b"t"}})
+ calls = []
+ _hub_records(monkeypatch, {}, calls)
+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)
+ # Nothing can be fetched or hashed: the cached .bin is the only way the load succeeds.
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": False}
+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", False)
+ assert ds.pipeline_safetensors_kwargs(str(root), passed = {"local_files_only": True}) == {
+ "use_safetensors": False
+ }
+ assert calls == []
+ # A variant load (fp16 twins) is never pointed at the default-name .bin files.
+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)
+ assert ds.pipeline_safetensors_kwargs(str(root), passed = {"variant": "fp16"}) == {}
+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", False)
+ # Online with no matching record the library default stays.
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+
+
+def test_a_plain_local_folder_of_bins_is_never_hub_verified(hub, tmp_path, monkeypatch):
+ from core.inference import diffusers_safetensors as ds
+
+ root = tmp_path / "local_pipe"
+ (root / "transformer").mkdir(parents = True)
+ (root / "transformer" / "diffusion_pytorch_model.bin").write_bytes(b"t")
+ (root / "model_index.json").write_text(json.dumps({"transformer": ["diffusers", "Y"]}))
+ _hub_records(monkeypatch, {})
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+
+
+def test_the_dense_encoder_fetch_skips_safetensors_for_a_verified_cached_bin(
+ hub, tmp_path, monkeypatch
+):
+ from core.inference import diffusion_te_prequant as tp
+
+ root = _snapshot(tmp_path, {"text_encoder": {"pytorch_model.bin": b"e"}})
+ listing = [
+ "model_index.json",
+ "text_encoder/model.safetensors",
+ "text_encoder/pytorch_model.bin",
+ ]
+ monkeypatch.setattr(huggingface_hub.HfApi, "list_repo_files", lambda self, *a, **k: listing)
+ monkeypatch.setattr(
+ huggingface_hub,
+ "try_to_load_from_cache",
+ lambda repo_id, filename, **k: str(root / filename)
+ if filename == "model_index.json"
+ else None,
+ )
+ _hub_records(monkeypatch, {"text_encoder/pytorch_model.bin": _sha(b"e")})
+ patterns = tp._dense_component_pickles("o/pipe", ["text_encoder"], None, None)
+ assert "text_encoder/model.safetensors" in patterns
+ assert "text_encoder/pytorch_model.bin" not in patterns
+ _hub_records(monkeypatch, {"text_encoder/pytorch_model.bin": _sha(b"different")})
+ import core.inference.cached_weight_sha as cws
+
+ cws._hub.clear()
+ patterns = tp._dense_component_pickles("o/pipe", ["text_encoder"], None, None)
+ assert "text_encoder/pytorch_model.bin" in patterns
+ assert "text_encoder/model.safetensors" not in patterns
+
+
+def test_an_unpinned_hub_id_is_checked_against_the_hubs_current_main(hub, tmp_path, monkeypatch):
+ from core.inference import diffusers_safetensors as ds
+
+ root = _snapshot(tmp_path, {"transformer": {"diffusion_pytorch_model.bin": b"t"}})
+ (root.parents[1] / "refs").mkdir()
+ (root.parents[1] / "refs" / "main").write_text("c" * 40)
+ import huggingface_hub.file_download as fd
+ import core.inference.cached_weight_sha as cws
+
+ cache = str(tmp_path / "hub")
+ calls = []
+ # The Hub's current main no longer lists the .bin (it moved to safetensors only): no reuse.
+ _hub_records(monkeypatch, {}, calls)
+ real = fd.try_to_load_from_cache
+ monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", real)
+ assert ds.pipeline_safetensors_kwargs("o/pipe", cache_dir = cache) == {}
+ assert calls and calls[-1][1] is None # current main, not the cached commit
+ cws._hub.clear() # the unanswered lookup above is otherwise remembered for a minute
+ _hub_records(monkeypatch, {"transformer/diffusion_pytorch_model.bin": _sha(b"t")}, calls)
+ assert ds.pipeline_safetensors_kwargs("o/pipe", cache_dir = cache) == {"use_safetensors": False}
+ assert calls[-1][1] is None
+
+
+def test_a_main_answer_is_refreshed_but_a_commit_answer_is_kept(hub, monkeypatch):
+ import core.inference.cached_weight_sha as cws
+
+ hub.put("Model-INT8.pt", PT_BYTES)
+ clock = [1000.0]
+ monkeypatch.setattr(cws.time, "monotonic", lambda: clock[0])
+ cws.hub_sha256s(REPO, ["Model-INT8.pt"])
+ cws.hub_sha256s(REPO, ["Model-INT8.pt"], revision = "c" * 40)
+ assert len(hub.info_calls) == 2
+ clock[0] += cws._MUTABLE_TTL_S + 1
+ cws.hub_sha256s(REPO, ["Model-INT8.pt"], revision = "c" * 40)
+ assert len(hub.info_calls) == 2
+ cws.hub_sha256s(REPO, ["Model-INT8.pt"])
+ assert len(hub.info_calls) == 3
diff --git a/studio/backend/tests/test_prequant_prefer_cached_pickle.py b/studio/backend/tests/test_prequant_prefer_cached_pickle.py
index 85daf66436b..6c67bc7e76d 100644
--- a/studio/backend/tests/test_prequant_prefer_cached_pickle.py
+++ b/studio/backend/tests/test_prequant_prefer_cached_pickle.py
@@ -1,16 +1,19 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Safetensors is preferred, so a cached .pt repo must not re-download its safetensors twin."""
+"""Safetensors is preferred, but a cached .pt whose sha256 matches the Hub's record is reused rather than
+re-downloading its safetensors twin; an unverified one (mismatch, no record, offline) keeps safetensors first."""
from __future__ import annotations
+import hashlib
import types
import huggingface_hub
import pytest
from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
+import core.inference.cached_weight_sha as cws
import core.inference.diffusion_prequant as pq
import core.inference.diffusion_te_prequant as tpq
@@ -34,6 +37,9 @@ class FakeHub:
# names a 404 was recorded for, as huggingface_hub's .no_exist markers
self.absent: set = set()
self.downloads: list = []
+ # sha256 the Hub records per name; None = no LFS record. Default: the bytes cache() writes.
+ self.hub_sha: dict = {}
+ self.sha_calls: list = []
for name in cached:
self.cache(name)
@@ -90,6 +96,19 @@ def hub(tmp_path, monkeypatch):
fake = FakeHub(tmp_path)
monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", fake.try_to_load_from_cache)
monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake.hf_hub_download)
+
+ def hub_sha256s(repo_id, filenames, **_):
+ fake.sha_calls.append(tuple(filenames))
+ out = {}
+ for name in filenames:
+ sha = fake.hub_sha.get(name, hashlib.sha256(b"x").hexdigest())
+ if sha:
+ out[name] = sha
+ return out
+
+ monkeypatch.setattr(cws, "hub_sha256s", hub_sha256s)
+ monkeypatch.setattr(cws, "_store_path", lambda: tmp_path / "weight_sha256.json")
+ monkeypatch.setattr(cws, "_persisted", None)
monkeypatch.setattr(
pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True
)
diff --git a/studio/backend/tests/test_safetensors_first_downloads.py b/studio/backend/tests/test_safetensors_first_downloads.py
new file mode 100644
index 00000000000..f110e96facb
--- /dev/null
+++ b/studio/backend/tests/test_safetensors_first_downloads.py
@@ -0,0 +1,404 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Image / video downloads and diffusers loads put .safetensors first: a pickle beside its safetensors twin is never
+fetched, a pickle that is the only copy still is, and use_safetensors=True is passed where the folder ships
+safetensors."""
+
+from __future__ import annotations
+
+import ast
+import json
+import struct
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+from core.inference import diffusers_safetensors as ds
+from hub.utils.snapshot_filters import (
+ resolve_snapshot_ignore_patterns_for_files,
+ sibling_matches_ignore,
+)
+
+_BACKEND = Path(__file__).resolve().parents[1]
+
+
+def _write_safetensors(path: Path) -> None:
+ header = json.dumps({"w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode()
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_bytes(struct.pack("<Q", len(header)) + header + b"\0\0\0\0")
+
+
+def _pipeline(tmp_path: Path, components: dict) -> Path:
+ root = tmp_path / "pipe"
+ root.mkdir()
+ index = {
+ "_class_name": "FluxPipeline",
+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
+ }
+ for name, files in components.items():
+ index[name] = ["diffusers", "Y"]
+ (root / name).mkdir()
+ (root / name / "config.json").write_text("{}")
+ for f in files:
+ if f.endswith(".safetensors"):
+ _write_safetensors(root / name / f)
+ else:
+ (root / name / f).write_bytes(b"x")
+ (root / "scheduler").mkdir()
+ (root / "scheduler" / "scheduler_config.json").write_text("{}")
+ (root / "model_index.json").write_text(json.dumps(index))
+ return root
+
+
+# --- download filters -------------------------------------------------------------------------------------------
+
+
+_DIFFUSERS_REPO = [
+ "model_index.json",
+ "unet/config.json",
+ "unet/diffusion_pytorch_model.safetensors",
+ "unet/diffusion_pytorch_model.bin",
+ "text_encoder/model.safetensors",
+ "text_encoder/pytorch_model.bin",
+ "safety_checker/pytorch_model.bin", # its only copy: kept
+ "vae/diffusion_pytorch_model.fp16.safetensors", # a variant does not make the .bin redundant
+ "vae/diffusion_pytorch_model.bin",
+ "sd_xl_base_1.0.ckpt", # root pickle with no safetensors twin: the only copy, kept
+ "image_encoder/open_clip.pth",
+ "Wan2.2-T2V-A14B-FP8.pt", # pre-quant hosted only as .pt: kept
+ "Wan2.2-T2V-A14B-INT8.pt",
+ "Wan2.2-T2V-A14B-INT8.safetensors", # its twin makes the .pt redundant
+]
+
+
+def test_a_pipeline_repo_never_downloads_a_pickle_beside_safetensors():
+ skipped = set(ds.pipeline_pickle_files(_DIFFUSERS_REPO))
+ assert skipped == {
+ "unet/diffusion_pytorch_model.bin",
+ "text_encoder/pytorch_model.bin",
+ "image_encoder/open_clip.pth",
+ "Wan2.2-T2V-A14B-INT8.pt",
+ }
+
+
+def test_a_bin_only_component_still_downloads():
+ skipped = set(ds.pipeline_pickle_files(_DIFFUSERS_REPO))
+ assert "safety_checker/pytorch_model.bin" not in skipped
+ assert "vae/diffusion_pytorch_model.bin" not in skipped
+ assert "sd_xl_base_1.0.ckpt" not in skipped
+ assert "Wan2.2-T2V-A14B-FP8.pt" not in skipped
+
+
+def test_a_modular_pipeline_repo_is_filtered_too():
+ names = [
+ "modular_model_index.json",
+ "transformer/diffusion_pytorch_model.safetensors",
+ "transformer/diffusion_pytorch_model.bin",
+ "x.pt",
+ "x.safetensors",
+ ]
+ assert set(ds.pipeline_pickle_files(names)) == {
+ "transformer/diffusion_pytorch_model.bin",
+ "x.pt",
+ }
+
+
+def test_an_unrelated_safetensors_never_drops_the_only_bin(tmp_path):
+ # An adapter / aux .safetensors beside a .bin-only component is not that component's weights.
+ names = [
+ "model_index.json",
+ "transformer/diffusion_pytorch_model.bin",
+ "transformer/adapter_model.safetensors",
+ "text_encoder/model-00001-of-00002.safetensors",
+ "text_encoder/model.safetensors.index.json",
+ "text_encoder/pytorch_model.bin",
+ ]
+ assert set(ds.pipeline_pickle_files(names)) == {"text_encoder/pytorch_model.bin"}
+ folder = tmp_path / "transformer"
+ folder.mkdir()
+ (folder / "diffusion_pytorch_model.bin").write_bytes(b"x")
+ (folder / "adapter_model.safetensors").write_bytes(b"x")
+ assert ds.component_weight_state(folder) is False
+ (folder / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
+ assert ds.component_weight_state(folder) is True
+
+
+def test_a_non_pipeline_repo_keeps_its_files():
+ # LLM / audio repos keep their own rules: no manifest, nothing extra is skipped here.
+ names = [
+ "config.json",
+ "model.safetensors",
+ "consolidated.pth",
+ "tokenizer.model",
+ "optimizer.pt",
+ ]
+ assert ds.pipeline_pickle_files(names) == []
+
+
+def test_the_hub_snapshot_filter_skips_diffusers_pickles():
+ ignore = resolve_snapshot_ignore_patterns_for_files(_DIFFUSERS_REPO)
+ for name in ("unet/diffusion_pytorch_model.bin", "Wan2.2-T2V-A14B-INT8.pt"):
+ assert sibling_matches_ignore(name, ignore), name
+ for name in (
+ "unet/diffusion_pytorch_model.safetensors",
+ "safety_checker/pytorch_model.bin",
+ "Wan2.2-T2V-A14B-FP8.pt",
+ "sd_xl_base_1.0.ckpt",
+ ):
+ assert not sibling_matches_ignore(name, ignore), name
+
+
+def test_the_hub_snapshot_filter_is_unchanged_for_an_llm_repo():
+ names = [
+ "config.json",
+ "model.safetensors",
+ "pytorch_model.bin",
+ "training_args.bin",
+ "extra.pt",
+ ]
+ ignore = resolve_snapshot_ignore_patterns_for_files(names)
+ assert not sibling_matches_ignore("extra.pt", ignore)
+ assert not sibling_matches_ignore("training_args.bin", ignore)
+
+
+def test_the_frontend_size_estimate_mirrors_the_pipeline_pickle_filter():
+ src = (
+ _BACKEND.parent / "frontend" / "src" / "features" / "hub" / "lib" / "dataset-size.ts"
+ ).read_text(encoding = "utf-8")
+ assert "function diffusersPickleFiles(" in src
+ assert '"model_index.json", "modular_model_index.json"' in src
+ assert r"/\.(pt|pth|ckpt|pkl|pickle)$/i" in src
+ # The same component-weight matcher as the backend: an adapter / variant safetensors never drops the .bin.
+ backend = ds._MODEL_SAFETENSORS.pattern
+ assert (
+ backend == r"^(diffusion_pytorch_model|model)(-\d+-of-\d+)?\.safetensors(\.index\.json)?$"
+ )
+ assert "/" + backend.replace("\\d", "\\d") + "/" in src
+ assert "MODEL_SAFETENSORS_RE.test(basename(n))" in src
+ assert "pipelinePickles.has(filename)" in src
+ assert "safetensorsStems.has(" in src
+
+
+def test_pipeline_and_base_file_filters_drop_never_read_pickles():
+ from core.inference.diffusion import _base_file_downloaded, _pipeline_file_downloaded
+
+ for name in ("unet/x.pt", "vae/y.pth", "text_encoder/z.ckpt", "transformer/w.pkl"):
+ assert not _pipeline_file_downloaded(name), name
+ assert not _base_file_downloaded(name, include_transformer = True), name
+ assert _pipeline_file_downloaded("unet/diffusion_pytorch_model.safetensors")
+ assert _base_file_downloaded("vae/diffusion_pytorch_model.safetensors")
+
+
+def test_the_dense_encoder_fetch_ignores_pickles(monkeypatch):
+ from core.inference import diffusion_te_prequant as tp
+
+ listing = [
+ "text_encoder/model.safetensors",
+ "text_encoder/pytorch_model.bin",
+ "text_encoder_2/pytorch_model.bin",
+ ]
+
+ class _Api:
+ def list_repo_files(self, *_a, **_k):
+ return listing
+
+ monkeypatch.setitem(sys.modules, "huggingface_hub", types.SimpleNamespace(HfApi = _Api))
+ patterns = tp._dense_component_pickles("o/r", ["text_encoder", "text_encoder_2"], None, None)
+ assert "text_encoder/pytorch_model.bin" in patterns
+ assert "text_encoder_2/pytorch_model.bin" not in patterns # its only copy
+ assert "text_encoder/*.pt" in patterns and "text_encoder_2/*.ckpt" in patterns
+
+
+def test_the_dense_encoder_fetch_without_a_listing_drops_only_never_read_pickles(monkeypatch):
+ from core.inference import diffusion_te_prequant as tp
+
+ class _Api:
+ def list_repo_files(self, *_a, **_k):
+ raise OSError("offline")
+
+ monkeypatch.setitem(sys.modules, "huggingface_hub", types.SimpleNamespace(HfApi = _Api))
+ patterns = tp._dense_component_pickles("o/r", ["text_encoder"], None, None)
+ assert not any(p.endswith(".bin") for p in patterns)
+ assert "text_encoder/*.pt" in patterns
+
+
+# --- use_safetensors=True ---------------------------------------------------------------------------------------
+
+
+def test_a_folder_shipping_safetensors_everywhere_loads_with_use_safetensors(tmp_path):
+ root = _pipeline(
+ tmp_path,
+ {
+ "transformer": ["diffusion_pytorch_model.safetensors"],
+ "text_encoder": ["model.safetensors", "pytorch_model.bin"],
+ "vae": ["diffusion_pytorch_model.safetensors"],
+ },
+ )
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": True}
+
+
+def test_a_bin_only_component_keeps_the_library_default(tmp_path):
+ root = _pipeline(
+ tmp_path,
+ {
+ "transformer": ["diffusion_pytorch_model.safetensors"],
+ "text_encoder": ["pytorch_model.bin"],
+ },
+ )
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+ assert ds.pipeline_safetensors_kwargs(str(root), passed = {"text_encoder": object()}) == {
+ "use_safetensors": True
+ }
+
+
+def test_a_partly_cached_snapshot_is_unknown_not_complete(tmp_path):
+ """A weight-bearing component that is missing, or present with its config only (a partly cached snapshot),
+ could still be .bin-only on the Hub: never force safetensors on it."""
+ root = _pipeline(tmp_path, {"transformer": ["diffusion_pytorch_model.safetensors"]})
+ index = json.loads((root / "model_index.json").read_text())
+ index["text_encoder"] = ["transformers", "CLIPTextModel"]
+ index["tokenizer"] = ["transformers", "CLIPTokenizer"]
+ (root / "model_index.json").write_text(json.dumps(index))
+ # tokenizer and scheduler hold no weights, so their absence or config-only folder is no evidence either way
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+ (root / "text_encoder").mkdir()
+ (root / "text_encoder" / "config.json").write_text("{}")
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+ _write_safetensors(root / "text_encoder" / "model.safetensors")
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": True}
+
+
+def test_a_variant_only_safetensors_does_not_count(tmp_path):
+ root = _pipeline(
+ tmp_path,
+ {"vae": ["diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.bin"]},
+ )
+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}
+
+
+def test_an_unknown_source_keeps_the_library_default(tmp_path):
+ assert ds.pipeline_safetensors_kwargs("nobody/not-cached-anywhere-xyz") == {}
+ assert ds.pipeline_safetensors_kwargs(str(tmp_path)) == {} # no manifest
+ assert ds.pipeline_safetensors_kwargs("") == {}
+
+
+def test_a_non_diffusers_pipeline_class_never_gets_the_kwarg(tmp_path, monkeypatch):
+ root = _pipeline(tmp_path, {"transformer": ["diffusion_pytorch_model.safetensors"]})
+
+ class DiffusionPipeline:
+ pass
+
+ class FluxPipeline(DiffusionPipeline):
+ pass
+
+ class Custom:
+ pass
+
+ monkeypatch.setitem(
+ sys.modules, "diffusers", types.SimpleNamespace(DiffusionPipeline = DiffusionPipeline)
+ )
+ assert ds.pipeline_safetensors_kwargs(str(root), pipeline_cls = Custom) == {}
+ assert ds.pipeline_safetensors_kwargs(str(root), pipeline_cls = FluxPipeline) == {
+ "use_safetensors": True
+ }
+
+
+def test_a_component_load_takes_use_safetensors_only_when_its_folder_ships_them(tmp_path):
+ root = _pipeline(
+ tmp_path,
+ {
+ "transformer": ["diffusion_pytorch_model.safetensors"],
+ "text_encoder": ["pytorch_model.bin"],
+ },
+ )
+ assert ds.component_safetensors_kwargs(str(root), "transformer") == {"use_safetensors": True}
+ assert ds.component_safetensors_kwargs(str(root), "text_encoder") == {}
+ assert ds.component_safetensors_kwargs(str(root), "missing") == {}
+
+
+def test_a_hub_id_uses_its_cached_snapshot(tmp_path, monkeypatch):
+ root = _pipeline(tmp_path, {"transformer": ["diffusion_pytorch_model.safetensors"]})
+
+ def _cached(
+ repo_id,
+ filename,
+ cache_dir = None,
+ ):
+ path = root / filename
+ return str(path) if path.is_file() else None
+
+ monkeypatch.setitem(
+ sys.modules, "huggingface_hub", types.SimpleNamespace(try_to_load_from_cache = _cached)
+ )
+ assert ds.pipeline_safetensors_kwargs("org/model") == {"use_safetensors": True}
+ assert ds.component_safetensors_kwargs("org/model", "transformer") == {"use_safetensors": True}
+
+
+def _calls_with_helper(path: Path, helper: str) -> int:
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ count = 0
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "from_pretrained":
+ for kw in node.keywords:
+ if kw.arg is None and isinstance(kw.value, ast.Call):
+ name = getattr(kw.value.func, "id", None) or getattr(
+ kw.value.func, "attr", None
+ )
+ if name == helper:
+ count += 1
+ return count
+
+
+def test_every_dense_diffusers_load_asks_for_safetensors():
+ image = _BACKEND / "core" / "inference" / "diffusion.py"
+ video = _BACKEND / "core" / "inference" / "video.py"
+ assert _calls_with_helper(image, "pipeline_safetensors_kwargs") == 3
+ assert _calls_with_helper(image, "component_safetensors_kwargs") == 1
+ assert _calls_with_helper(video, "_safetensors_kwargs") == 3
+ assert _calls_with_helper(video, "component_safetensors_kwargs") == 1
+
+
+def test_a_pinned_revision_is_judged_on_its_own_snapshot(tmp_path):
+ # Cached main ships safetensors, the pinned commit only a .bin: the pinned load must not be forced to safetensors.
+ cache = tmp_path / "hub"
+ repo = cache / "models--o--pipe"
+ main_rev, pinned = "a" * 40, "b" * 40
+ for rev, weight in (
+ (main_rev, "diffusion_pytorch_model.safetensors"),
+ (pinned, "diffusion_pytorch_model.bin"),
+ ):
+ snap = repo / "snapshots" / rev
+ (snap / "transformer").mkdir(parents = True)
+ (snap / "transformer" / "config.json").write_text("{}")
+ if weight.endswith(".safetensors"):
+ _write_safetensors(snap / "transformer" / weight)
+ else:
+ (snap / "transformer" / weight).write_bytes(b"x")
+ (snap / "model_index.json").write_text(json.dumps({"transformer": ["diffusers", "Y"]}))
+ (repo / "refs").mkdir()
+ (repo / "refs" / "main").write_text(main_rev)
+ assert ds.pipeline_safetensors_kwargs("o/pipe", cache_dir = str(cache)) == {
+ "use_safetensors": True
+ }
+ assert ds.pipeline_safetensors_kwargs(
+ "o/pipe", passed = {"revision": pinned}, cache_dir = str(cache)
+ ) != {"use_safetensors": True}
+ assert ds.component_safetensors_kwargs(
+ "o/pipe", "transformer", cache_dir = str(cache), revision = pinned
+ ) != {"use_safetensors": True}
+ assert ds.component_safetensors_kwargs("o/pipe", "transformer", cache_dir = str(cache)) == {
+ "use_safetensors": True
+ }
+
+
+def test_an_orphan_shard_without_its_index_is_not_safetensors_weights(tmp_path):
+ folder = tmp_path / "transformer"
+ folder.mkdir()
+ (folder / "diffusion_pytorch_model.bin").write_bytes(b"x")
+ (folder / "diffusion_pytorch_model-00001-of-00002.safetensors").write_bytes(b"x")
+ assert ds.component_weight_state(folder) is False
+ (folder / "diffusion_pytorch_model.safetensors.index.json").write_text("{}")
+ assert ds.component_weight_state(folder) is True
diff --git a/studio/backend/tests/test_video_prequant_safetensors_twins.py b/studio/backend/tests/test_video_prequant_safetensors_twins.py
index c6e6b7a174a..8d157e4c263 100644
--- a/studio/backend/tests/test_video_prequant_safetensors_twins.py
+++ b/studio/backend/tests/test_video_prequant_safetensors_twins.py
@@ -6,6 +6,7 @@ a new user fetches only the twin, a cached .pt is kept, an unreadable twin is sk
from __future__ import annotations
+import hashlib
import threading
import types
@@ -13,6 +14,7 @@ import huggingface_hub
import pytest
from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
+import core.inference.cached_weight_sha as cws
import core.inference.diffusion_prequant as pq
from core.inference import video_families as vf
@@ -260,6 +262,14 @@ def _setup(
hub = FakeHub(tmp_path, src.location, hosted)
monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", hub.try_to_load_from_cache)
monkeypatch.setattr(huggingface_hub, "hf_hub_download", hub.hf_hub_download)
+ # The Hub records the sha256 of the bytes the fake cache writes: a cached pickle is a verified copy.
+ monkeypatch.setattr(
+ cws,
+ "hub_sha256s",
+ lambda repo_id, names, **_: {n: hashlib.sha256(b"x").hexdigest() for n in names},
+ )
+ monkeypatch.setattr(cws, "_store_path", lambda: tmp_path / "weight_sha256.json")
+ monkeypatch.setattr(cws, "_persisted", None)
monkeypatch.setattr(
pq,
"restricted_prequant_load_supported",
diff --git a/studio/frontend/src/features/hub/lib/dataset-size.ts b/studio/frontend/src/features/hub/lib/dataset-size.ts
index c64275f0b75..7ef4f9a002b 100644
--- a/studio/frontend/src/features/hub/lib/dataset-size.ts
+++ b/studio/frontend/src/features/hub/lib/dataset-size.ts
@@ -318,6 +318,49 @@ function redundantTorchBinFiles(names: string[]): Set<string> {
return out;
}
+// Mirrors pipeline_pickle_files (diffusers_safetensors.py, via diffusers_pickle_files in snapshot_filters.py):
+// a diffusers pipeline repo skips a .pt / .pth / .ckpt / .pkl inside a component folder or beside its same-stem
+// .safetensors twin (a root pickle with no twin is the only copy and stays), and a .bin beside safetensors weights.
+const PIPELINE_MANIFESTS = ["model_index.json", "modular_model_index.json"];
+const NEVER_READ_PICKLE_RE = /\.(pt|pth|ckpt|pkl|pickle)$/i;
+// Mirrors _MODEL_SAFETENSORS: only a component's own non-variant weights make its .bin redundant.
+const MODEL_SAFETENSORS_RE =
+ /^(diffusion_pytorch_model|model)(-\d+-of-\d+)?\.safetensors(\.index\.json)?$/;
+
+function folderOf(name: string): string {
+ const slash = name.lastIndexOf("/");
+ return slash >= 0 ? name.slice(0, slash) : "";
+}
+
+function diffusersPickleFiles(names: string[]): Set<string> {
+ const out = new Set<string>();
+ if (!names.some((n) => PIPELINE_MANIFESTS.includes(n))) return out;
+ const safetensorsFolders = new Set(
+ names
+ .filter((n) => MODEL_SAFETENSORS_RE.test(basename(n)))
+ .map(folderOf),
+ );
+ const safetensorsStems = new Set(
+ names
+ .filter((n) => n.toLowerCase().endsWith(".safetensors"))
+ .map((n) => n.slice(0, -".safetensors".length).toLowerCase()),
+ );
+ for (const n of names) {
+ const lower = n.toLowerCase();
+ if (NEVER_READ_PICKLE_RE.test(lower)) {
+ if (folderOf(n) !== "" || safetensorsStems.has(lower.slice(0, lower.lastIndexOf(".")))) {
+ out.add(n);
+ }
+ } else if (
+ (lower.endsWith(".bin") || lower.endsWith(".bin.index.json")) &&
+ safetensorsFolders.has(folderOf(n))
+ ) {
+ out.add(n);
+ }
+ }
+ return out;
+}
+
// Mirrors PRECISION_FOLDER_SOURCE and precision_variant_subfolders in snapshot_filters.py.
const PRECISION_FOLDER_SOURCE =
"(?:mlx[-_]?)?(?:[0-9]{1,2}[-_]?bits?(?:[-_][a-z0-9]+)*|bf16|fp16|f16|fp32|f32|fp8|mxfp4|nvfp4)";
@@ -380,6 +423,7 @@ function isSnapshotIgnored(
skipDuplicateFormats: boolean,
redundantBins: Set<string>,
precisionFolders: Set<string> = new Set(),
+ pipelinePickles: Set<string> = new Set(),
): boolean {
const lower = filename.toLowerCase();
const folder = filename.includes("/") ? filename.slice(0, filename.indexOf("/")) : "";
@@ -390,6 +434,7 @@ function isSnapshotIgnored(
lower.startsWith("openvino/") ||
lower.startsWith("mlx/") ||
lower.endsWith(".bin.index.json.bak") ||
+ pipelinePickles.has(filename) ||
(skipConsolidated && lower.startsWith("consolidated")) ||
(skipDuplicateFormats &&
(DUPLICATE_WEIGHT_FORMAT_RE.test(filename) || redundantBins.has(filename) ||
@@ -433,6 +478,7 @@ export function fetchModelSize(
const precisionFolders = names.includes("config.json")
? precisionVariantFolders(names)
: new Set<string>();
+ const pipelinePickles = diffusersPickleFiles(names);
let total = 0;
let weights = 0;
for (const s of siblings) {
@@ -446,6 +492,7 @@ export function fetchModelSize(
skipDuplicateFormats,
redundantBins,
precisionFolders,
+ pipelinePickles,
)
) {
continue;
diff --git a/studio/frontend/tests/hub-model-size-diffusers-pickles.test.ts b/studio/frontend/tests/hub-model-size-diffusers-pickles.test.ts
new file mode 100644
index 00000000000..b7c4c727f43
--- /dev/null
+++ b/studio/frontend/tests/hub-model-size-diffusers-pickles.test.ts
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+
+const { fetchModelSize } = await import(
+ "../src/features/hub/lib/dataset-size.ts"
+);
+
+function stubSiblings(sizes: Record<string, number>): void {
+ globalThis.fetch = (async () =>
+ new Response(
+ JSON.stringify({
+ siblings: Object.entries(sizes).map(([rfilename, size]) => ({
+ rfilename,
+ size,
+ })),
+ }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ )) as typeof fetch;
+}
+
+test("a diffusers pipeline skips the pickles its safetensors make redundant", async () => {
+ stubSiblings({
+ "model_index.json": 1,
+ "unet/diffusion_pytorch_model.safetensors": 100,
+ "unet/diffusion_pytorch_model.bin": 100,
+ "v1-5-pruned.safetensors": 50,
+ "v1-5-pruned.ckpt": 50,
+ "Model-FP8.pt": 30,
+ });
+ // The root .pt with no safetensors twin is the only copy and stays.
+ assert.deepEqual(await fetchModelSize("acme/sd-shaped"), {
+ totalBytes: 1 + 100 + 50 + 30,
+ weightsBytes: 100 + 50 + 30,
+ });
+});
+
+test("an adapter safetensors never drops a component's only .bin", async () => {
+ stubSiblings({
+ "model_index.json": 1,
+ "transformer/adapter_model.safetensors": 5,
+ "transformer/diffusion_pytorch_model.bin": 200,
+ "vae/diffusion_pytorch_model.fp16.safetensors": 10,
+ "vae/diffusion_pytorch_model.bin": 20,
+ });
+ assert.deepEqual(await fetchModelSize("acme/adapter-shaped"), {
+ totalBytes: 1 + 5 + 200 + 10 + 20,
+ weightsBytes: 5 + 200 + 10 + 20,
+ });
+});