Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: load image / video / text-encoder weights from .safetensors first, reuse a cached .pt / .bin when its sha256 matches the Hub (#17061)
Daniel Han2fb404b85ca0
diff --git a/studio/backend/core/inference/cached_weight_sha.py b/studio/backend/core/inference/cached_weight_sha.pynew file mode 100644index 00000000000..bffc0167602--- /dev/null+++ b/studio/backend/core/inference/cached_weight_sha.py@@ -0,0 +1,198 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Verify a cached ``.pt`` / ``.bin`` is byte-identical to the Hub's copy (LFS sha256) so a load can reuse it.++Digests are memoised per (real path, size, mtime) in memory and on disk. Any unanswerable case (offline, no LFS+record, error) is "not verified", so the caller keeps safetensors first. Never raises."""++from __future__ import annotations++import hashlib+import json+import os+import threading+import time+from pathlib import Path+from typing import Iterable, Optional++ENV_DISABLE = "UNSLOTH_REUSE_CACHED_PICKLE_WEIGHTS"+_CHUNK = 8 * 1024 * 1024+# Hub misses retry after _MISS_TTL_S, branch answers after _MUTABLE_TTL_S; commit-id answers never expire.+_MISS_TTL_S = 60.0+_MUTABLE_TTL_S = 300.0+_FILE_NAME = "weight_sha256.json"+_MAX_ENTRIES = 512++_lock = threading.Lock()+_hub: dict = {}+_local: dict = {}+_persisted: Optional[dict] = None+++def reuse_enabled() -> bool:+ return (os.environ.get(ENV_DISABLE) or "").strip().lower() not in ("0", "false", "no", "off")+++def _offline() -> bool:+ try:+ from huggingface_hub import constants+ return bool(constants.HF_HUB_OFFLINE)+ except Exception: # noqa: BLE001 - no hub library: nothing to compare against+ return True+++def hub_sha256s(+ repo_id: Optional[str],+ filenames: Iterable[str],+ *,+ revision: Optional[str] = None,+ token: Optional[str] = None,+) -> dict:+ """``{filename: sha256}`` for the LFS files of ``repo_id`` among ``filenames``, one Hub call for the uncached+ ones. A file the Hub stores without an LFS record, or any failure, is simply absent from the answer."""+ names = [str(n) for n in dict.fromkeys(filenames) if n]+ if not repo_id or not names or _offline():+ return {}+ now = time.monotonic()+ pinned = (+ bool(revision)+ and len(str(revision)) == 40+ and all(c in "0123456789abcdef" for c in str(revision))+ )+ out: dict = {}+ wanted: list = []+ with _lock:+ for name in names:+ hit = _hub.get((repo_id, revision, name))+ if (+ hit is None+ or (hit[0] is None and now - hit[1] > _MISS_TTL_S)+ or (not pinned and now - hit[1] > _MUTABLE_TTL_S)+ ):+ wanted.append(name)+ elif hit[0]:+ out[name] = hit[0]+ if not wanted:+ return out+ answered: dict = {}+ try:+ from huggingface_hub import HfApi+ for entry in HfApi().get_paths_info(+ repo_id, wanted, revision = revision, token = token or None+ ):+ lfs = getattr(entry, "lfs", None)+ sha = lfs.get("sha256") if isinstance(lfs, dict) else getattr(lfs, "sha256", None)+ path = getattr(entry, "path", None)+ if path and isinstance(sha, str) and len(sha) == 64:+ answered[path] = sha.lower()+ except Exception: # noqa: BLE001 - offline, gated, rate limited: unverified, retried later+ with _lock:+ for name in wanted:+ _hub[(repo_id, revision, name)] = (None, now)+ return out+ with _lock:+ for name in wanted:+ sha = answered.get(name)+ # A listed non-LFS file is answered for good (""), a missing one retried after the TTL.+ _hub[(repo_id, revision, name)] = (sha or None, now)+ if sha:+ out[name] = sha+ return out+++def _store_path() -> Optional[Path]:+ try:+ from utils.paths.storage_roots import cache_root+ return cache_root() / _FILE_NAME+ except Exception: # noqa: BLE001 - outside the Studio backend: memory only+ return None+++def _load_persisted() -> dict:+ global _persisted+ if _persisted is None:+ _persisted = {}+ path = _store_path()+ try:+ if path is not None and path.is_file():+ data = json.loads(path.read_text(encoding = "utf-8"))+ if isinstance(data, dict):+ _persisted = {+ k: v for k, v in data.items() if isinstance(v, list) and len(v) == 3+ }+ except Exception: # noqa: BLE001 - a bad record file is an empty one+ _persisted = {}+ return _persisted+++def _save_persisted(data: dict) -> None:+ path = _store_path()+ if path is None:+ return+ try:+ path.parent.mkdir(parents = True, exist_ok = True)+ keep = dict(list(data.items())[-_MAX_ENTRIES:])+ tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")+ tmp.write_text(json.dumps(keep), encoding = "utf-8")+ os.replace(tmp, path)+ except Exception: # noqa: BLE001 - the digest is still remembered in memory+ pass+++def file_sha256(path: str) -> Optional[str]:+ """The sha256 of ``path``'s bytes (symlinks followed), from the record when the size and mtime still match."""+ try:+ real = os.path.realpath(path)+ st = os.stat(real)+ except OSError:+ return None+ key = (real, int(st.st_size), int(st.st_mtime_ns))+ with _lock:+ hit = _local.get(key)+ if hit is not None:+ return hit+ rec = _load_persisted().get(real)+ if rec and int(rec[0]) == key[1] and int(rec[1]) == key[2]:+ _local[key] = str(rec[2])+ return str(rec[2])+ digest = hashlib.sha256()+ try:+ with open(real, "rb") as handle:+ for chunk in iter(lambda: handle.read(_CHUNK), b""):+ digest.update(chunk)+ except OSError:+ return None+ sha = digest.hexdigest()+ with _lock:+ _local[key] = sha+ data = _load_persisted()+ data.pop(real, None)+ data[real] = [key[1], key[2], sha]+ _save_persisted(data)+ return sha+++def cached_copy_matches_hub(+ repo_id: Optional[str],+ filename: Optional[str],+ path: Optional[str],+ *,+ revision: Optional[str] = None,+ token: Optional[str] = None,+ expected: Optional[str] = None,+) -> bool:+ """True only when the Hub records an sha256 for ``repo_id/filename`` and ``path`` hashes to it."""+ if not reuse_enabled() or not repo_id or not filename or not path:+ return False+ try:+ sha = expected or hub_sha256s(repo_id, [filename], revision = revision, token = token).get(+ filename+ )+ if not sha:+ return False+ # Always the bytes, never the cache blob's name: newer huggingface_hub caches name blobs by a Xet / etag+ # hash that is not the file's sha256.+ return file_sha256(path) == sha+ except Exception: # noqa: BLE001 - unverified+ return Falsediff --git a/studio/backend/core/inference/diffusers_safetensors.py b/studio/backend/core/inference/diffusers_safetensors.pynew file mode 100644index 00000000000..bda97d66672--- /dev/null+++ b/studio/backend/core/inference/diffusers_safetensors.py@@ -0,0 +1,359 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""When a diffusers load may pass ``use_safetensors``, and which repo files a pipeline download skips.++``True`` turns the silent pickle fallback into an error, so it is passed only when every weight-bearing component of+the folder ships ``.safetensors``; an uninspectable folder keeps the library default. ``False`` is passed only to+reuse cached ``.bin`` weights whose sha256 matches the Hub (or offline, where the default would refuse the load).+No torch, never raises."""++from __future__ import annotations++import json+import os+import re+from collections.abc import Iterable+from pathlib import Path+from typing import Any, Optional++_PICKLE_WEIGHT_SUFFIXES = (".pt", ".pth", ".ckpt", ".bin", ".pkl", ".pickle")+++def is_pickle_weight(name: Optional[Any]) -> bool:+ return str(name or "").strip().lower().endswith(_PICKLE_WEIGHT_SUFFIXES)+++_MANIFESTS = ("model_index.json", "modular_model_index.json")+_WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pt", ".pth", ".ckpt", ".pkl", ".pickle")+# Pickle containers no diffusers / transformers component load reads: always skipped in a pipeline download.+_NEVER_READ_PICKLES = (".pt", ".pth", ".ckpt", ".pkl", ".pickle")+++# The weight names a component load resolves (diffusers ``diffusion_pytorch_model``, transformers ``model``), single or+# sharded. dtype-variant twins (``model.fp16.safetensors``) never match, and any other ``.safetensors``+# (``adapter_model.safetensors``, an aux file) is not the component's weights.+_MODEL_SAFETENSORS = re.compile(+ r"^(diffusion_pytorch_model|model)(-\d+-of-\d+)?\.safetensors(\.index\.json)?$"+)+++def _is_model_safetensors(name: str) -> bool:+ return bool(_MODEL_SAFETENSORS.match(name.rsplit("/", 1)[-1]))+++# The pickle twins of those names (``diffusion_pytorch_model.bin``, ``pytorch_model-00001-of-00002.bin``).+_MODEL_BIN = re.compile(r"^(diffusion_pytorch_model|pytorch_model)(-\d+-of-\d+)?\.bin$")+++def _is_model_bin(name: str) -> bool:+ return bool(_MODEL_BIN.match(name.rsplit("/", 1)[-1]))+++def _snapshot_identity(root: Path) -> Optional[tuple]:+ """``(repo_id, commit)`` of a Hub cache snapshot folder (``.../models--a--b/snapshots/<commit>``), else None."""+ try:+ parts = Path(os.path.realpath(str(root))).parts+ if len(parts) < 3 or parts[-2] != "snapshots" or not parts[-3].startswith("models--"):+ return None+ return parts[-3][len("models--") :].replace("--", "/"), parts[-1]+ except Exception: # noqa: BLE001+ return None+++def _hub_offline() -> bool:+ try:+ from huggingface_hub import constants+ return bool(constants.HF_HUB_OFFLINE)+ except Exception: # noqa: BLE001+ return True+++def verified_bin_components(+ root: Any,+ components: Iterable[str],+ *,+ token: Optional[str] = None,+ offline: bool = False,+ revision: Optional[str] = None,+) -> set:+ """The ``components`` of the cached snapshot ``root`` whose weights are on disk only as ``.bin`` (no+ ``.safetensors`` weight cached) and every such ``.bin`` hashes to the sha256 the Hub records for it at the+ revision the load will read (``revision``, or the Hub's current ``main`` when the load is unpinned, never the+ cached snapshot's possibly stale commit). Empty for a folder that is not a Hub cache snapshot or on any doubt. ``offline`` (nothing can+ be downloaded, so the alternative to the cached ``.bin`` is a failed load) accepts the cached ``.bin`` unhashed."""+ try:+ from .cached_weight_sha import cached_copy_matches_hub, hub_sha256s, reuse_enabled++ if not reuse_enabled():+ return set()+ ident = _snapshot_identity(Path(str(root)))+ if ident is None:+ return set()+ repo_id, _commit = ident+ offline = offline or _hub_offline()+ wanted: dict = {}+ for component in components:+ folder = Path(str(root)) / component+ if not folder.is_dir() or component_weight_state(folder) is not False:+ continue+ bins = [p.name for p in folder.iterdir() if p.is_file() and _is_model_bin(p.name)]+ if bins:+ wanted[component] = [f"{component}/{b}" for b in bins]+ if not wanted:+ return set()+ if offline:+ return set(wanted)+ shas = hub_sha256s(+ repo_id, [n for names in wanted.values() for n in names], revision = revision, token = token+ )+ return {+ component+ for component, names in wanted.items()+ if all(+ n in shas+ and cached_copy_matches_hub(+ repo_id, n, str(Path(str(root)) / n), revision = revision, expected = shas[n]+ )+ for n in names+ )+ }+ except Exception: # noqa: BLE001 - unverified: safetensors stays first+ return set()+++def component_weight_state(folder: Any) -> Optional[bool]:+ """True: ``folder`` holds the component's own non-variant ``.safetensors`` weight (or its index); False: it holds+ weights and none of them is that; None: no weight file at all (or unreadable)."""+ try:+ names = [p.name for p in Path(str(folder)).iterdir() if p.is_file()]+ except OSError:+ return None+ weights = [+ n for n in names if n.lower().endswith(_WEIGHT_SUFFIXES) or n.endswith(".index.json")+ ]+ if not weights:+ return None+ # A lone shard without its index is not loadable (an interrupted download): only a whole file or an index counts.+ if any(+ _is_model_safetensors(n) and not re.search(r"-\d+-of-\d+\.safetensors$", n) for n in weights+ ):+ return True+ return False if any(not n.endswith(".index.json") for n in weights) else None+++def _manifest_components(root: Path) -> Optional[list]:+ for manifest in _MANIFESTS:+ path = root / manifest+ if not path.is_file():+ continue+ try:+ data = json.loads(path.read_text(encoding = "utf-8"))+ except Exception: # noqa: BLE001 - unreadable manifest: unknown+ return None+ if not isinstance(data, dict):+ return None+ out = []+ for key, value in data.items():+ if key.startswith("_"):+ continue+ # classic manifests: [library, class]; modular ones: [library, class, {spec}] or nested specs+ if isinstance(value, (list, tuple)) and value and value[0] is not None:+ out.append((key, value[1] if len(value) > 1 else None))+ return out+ return None+++# Manifest classes that never hold weights (a scheduler / tokenizer folder is config only).+_WEIGHTLESS_CLASS_SUFFIXES = (+ "Scheduler",+ "Tokenizer",+ "TokenizerFast",+ "Processor",+ "FeatureExtractor",+)+++def _weightless(class_name: Any) -> bool:+ return isinstance(class_name, str) and class_name.endswith(_WEIGHTLESS_CLASS_SUFFIXES)+++def local_pipeline_safetensors_complete(root: Any, passed: Iterable[str] = ()) -> bool:+ """True when ``root`` is a local diffusers pipeline folder whose every manifest component that holds weights+ (and is not supplied pre-built in ``passed``) has a ``.safetensors`` weight. False when unknown (a weight-bearing+ component missing or without weights, e.g. a partly cached snapshot) or any component is pickle-only."""+ try:+ base = Path(str(root)).expanduser()+ if not base.is_dir():+ return False+ except (OSError, ValueError):+ return False+ components = _manifest_components(base)+ if components is None:+ return False+ skip = set(passed or ())+ seen = False+ for name, class_name in components:+ if name in skip:+ continue+ folder = base / name+ state = component_weight_state(folder) if folder.is_dir() else None+ if state is False:+ return False+ if state is None:+ if _weightless(class_name):+ continue+ return False+ seen = True+ return seen+++def _cached_snapshot_dir(+ repo_id: str,+ probe: str,+ cache_dir: Optional[str],+ revision: Optional[str] = None,+) -> Optional[Path]:+ """The cached snapshot folder of ``revision`` (default: the cached ``main``) holding ``repo_id/probe``, or None.+ Offline: reads the cache only."""+ try:+ from huggingface_hub import try_to_load_from_cache+ except Exception: # noqa: BLE001+ return None+ for root in dict.fromkeys((cache_dir, None)):+ try:+ hit = try_to_load_from_cache(+ repo_id, probe, cache_dir = root, **({"revision": revision} if revision else {})+ )+ except Exception: # noqa: BLE001+ continue+ if isinstance(hit, str) and os.path.isfile(hit):+ return Path(hit).parents[len(probe.split("/")) - 1]+ return None+++def _takes_use_safetensors(cls: Any, base_name: str) -> bool:+ """Only diffusers' own ``DiffusionPipeline`` / ``ModelMixin`` subclasses are known to accept the kwarg."""+ if cls is None:+ return True+ try:+ import diffusers+ base = getattr(diffusers, base_name, None)+ return isinstance(cls, type) and base is not None and issubclass(cls, base)+ except Exception: # noqa: BLE001+ return False+++def pipeline_safetensors_kwargs(+ source: Any,+ *,+ passed: Iterable[str] = (),+ cache_dir: Optional[str] = None,+ pipeline_cls: Any = None,+) -> dict:+ """``{"use_safetensors": True}`` for a pipeline ``from_pretrained(source)`` whose folder (``source`` itself, or the+ cached snapshot of a Hub id) ships safetensors for every weight-bearing component; ``{}`` otherwise (and for a+ ``pipeline_cls`` that is not a diffusers ``DiffusionPipeline``)."""+ try:+ if not _takes_use_safetensors(pipeline_cls, "DiffusionPipeline"):+ return {}+ text = str(source or "").strip()+ if not text:+ return {}+ if os.path.isdir(os.path.expanduser(text)):+ root: Optional[Path] = Path(text).expanduser()+ # A snapshot folder is read as is: its own commit is the revision the load reads.+ revision = (_snapshot_identity(root) or (None, None))[1]+ else:+ root = None+ # The snapshot the load reads: a pinned ``revision`` among the load's kwargs, else the cached main (whose+ # .bin files are then checked against the Hub's current main, what an unpinned load resolves to).+ revision = passed.get("revision") if isinstance(passed, dict) else None+ for manifest in _MANIFESTS:+ root = _cached_snapshot_dir(text, manifest, cache_dir, revision)+ if root is not None:+ break+ if root is not None and local_pipeline_safetensors_complete(root, passed):+ return {"use_safetensors": True}+ variant = passed.get("variant") if isinstance(passed, dict) else None+ # A variant load reads ``*.<variant>.*`` files, which the default-name .bin check says nothing about.+ if root is not None and not variant and _snapshot_identity(root) is not None:+ # Every weight component already cached as a Hub-verified .bin: reuse it, download nothing new.+ components = _manifest_components(root) or []+ skip = set(passed or ())+ needed = [+ name+ for name, class_name in components+ if name not in skip and not _weightless(class_name)+ ]+ token = passed.get("token") if isinstance(passed, dict) else None+ offline = bool(passed.get("local_files_only")) if isinstance(passed, dict) else False+ if needed and verified_bin_components(+ root, needed, token = token, offline = offline, revision = revision+ ) == set(needed):+ return {"use_safetensors": False}+ except Exception: # noqa: BLE001 - unknown keeps the library default+ pass+ return {}+++def component_safetensors_kwargs(+ source: Any,+ subfolder: str,+ *,+ cache_dir: Optional[str] = None,+ model_cls: Any = None,+ local_files_only: bool = False,+ revision: Optional[str] = None,+) -> dict:+ """``{"use_safetensors": True}`` for ``from_pretrained(source, subfolder = subfolder)`` when that component folder+ (local, or in the cached snapshot of a Hub id) holds a safetensors weight; ``{}`` otherwise (and for a+ ``model_cls`` that is not a diffusers ``ModelMixin``)."""+ try:+ if not _takes_use_safetensors(model_cls, "ModelMixin"):+ return {}+ text = str(source or "").strip()+ if not text or not subfolder:+ return {}+ if os.path.isdir(os.path.expanduser(text)):+ folder: Optional[Path] = Path(text).expanduser() / subfolder+ revision = (_snapshot_identity(folder.parent) or (None, revision))[1]+ else:+ snap = _cached_snapshot_dir(text, f"{subfolder}/config.json", cache_dir, revision)+ folder = snap / subfolder if snap is not None else None+ if folder is not None and component_weight_state(folder) is True:+ return {"use_safetensors": True}+ if folder is not None and verified_bin_components(+ folder.parent, [subfolder], offline = local_files_only, revision = revision+ ):+ return {"use_safetensors": False}+ except Exception: # noqa: BLE001+ pass+ return {}+++def pipeline_pickle_files(filenames: Iterable[str]) -> list:+ """Pipeline repo files to skip: pickles in a component folder or beside a same-stem root ``.safetensors``, and+ ``.bin`` beside the component's own safetensors. A pickle with no twin is the only copy and stays."""+ names = [n for n in filenames if isinstance(n, str)]+ if not any(n in _MANIFESTS for n in names):+ return []+ st_dirs = {n.rsplit("/", 1)[0] if "/" in n else "" for n in names if _is_model_safetensors(n)}+ st_stems = {+ n[: -len(".safetensors")].lower() for n in names if n.lower().endswith(".safetensors")+ }+ out = []+ for n in names:+ low = n.lower()+ folder = n.rsplit("/", 1)[0] if "/" in n else ""+ if low.endswith(_NEVER_READ_PICKLES):+ if folder or low.rsplit(".", 1)[0] in st_stems:+ out.append(n)+ elif (low.endswith(".bin") or low.endswith(".bin.index.json")) and folder in st_dirs:+ out.append(n)+ return out+++def is_never_read_pickle(name: Optional[str]) -> bool:+ """A pickle container (``.pt`` / ``.pth`` / ``.ckpt`` / ``.pkl``) no diffusers component load reads."""+ return bool(name) and is_pickle_weight(name) and str(name).lower().endswith(_NEVER_READ_PICKLES)diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 84cef6d0ed9..3e57c1c1cb6 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -486,6 +486,13 @@ from .diffusion_comfy_quant import (refuse_comfy_quant,)from .diffusion_single_file_trust import assert_safetensors_file, single_file_load_allowed+from .diffusers_safetensors import (+ component_safetensors_kwargs,+ component_weight_state,+ is_never_read_pickle,+ pipeline_pickle_files,+ pipeline_safetensors_kwargs,+)from .diffusion_prequant import (PrequantIntoError,PrequantSource,@@ -6961,7 +6968,12 @@ class DiffusionBackend:)# Under the repo it describes: a mirror's commit says nothing about a cached upstream snapshot._record_revision(revisions_out, listing_repo, info)- return [s for s in info.siblings if not _listing_extra(listing_repo, s.rfilename)]+ skipped = set(pipeline_pickle_files(s.rfilename for s in info.siblings))+ return [+ s+ for s in info.siblings+ if not _listing_extra(listing_repo, s.rfilename) and s.rfilename not in skipped+ ]listing_repo = prefer_ungated_mirror(base_repo, hf_token)base_siblings = _base_listing(listing_repo)@@ -9912,7 +9924,14 @@ class DiffusionBackend:try:with _converted_shards_pread(dtype, small_host):pipe = pipeline_cls.from_pretrained(- _base_local_dir or fetch_base, **pipe_kwargs+ _base_local_dir or fetch_base,+ **pipe_kwargs,+ **pipeline_safetensors_kwargs(+ _base_local_dir or fetch_base,+ passed = pipe_kwargs,+ cache_dir = pipe_kwargs.get("cache_dir"),+ pipeline_cls = pipeline_cls,+ ),)finally:self._stop_load_prefetch()@@ -10278,7 +10297,14 @@ class DiffusionBackend:)with _converted_shards_pread(dtype, small_host):pipe = pipeline_cls.from_pretrained(- _base_local_dir or fetch_base, **pipe_kwargs+ _base_local_dir or fetch_base,+ **pipe_kwargs,+ **pipeline_safetensors_kwargs(+ _base_local_dir or fetch_base,+ passed = pipe_kwargs,+ cache_dir = pipe_kwargs.get("cache_dir"),+ pipeline_cls = pipeline_cls,+ ),)if small_host is not None and small_host.engaged:plan = self._apply_small_host_route(@@ -11957,6 +11983,13 @@ class DiffusionBackend:cache_dir = hub_cache_dir(),# Dense bf16 transformer is the largest fetch on this path, so API-initiated loads are refused here.local_files_only = local_files_only,+ **component_safetensors_kwargs(+ fetch_base,+ "transformer",+ cache_dir = hub_cache_dir(),+ model_cls = transformer_cls,+ local_files_only = local_files_only,+ ),)check_cancelled()pipe = self._assemble_pipe(@@ -12444,7 +12477,16 @@ class DiffusionBackend:))with _converted_shards_pread(dtype, None):- pipe = pipeline_cls.from_pretrained(base_local_dir or base, **pipe_kwargs)+ pipe = pipeline_cls.from_pretrained(+ base_local_dir or base,+ **pipe_kwargs,+ **pipeline_safetensors_kwargs(+ base_local_dir or base,+ passed = pipe_kwargs,+ cache_dir = pipe_kwargs.get("cache_dir"),+ pipeline_cls = pipeline_cls,+ ),+ )check_cancelled()pipe.to(device)return pipe@@ -13626,7 +13668,7 @@ class DiffusionBackend:cn_dtype = getattr(torch, str(state.dtype).replace("torch.", ""), None)# Force safetensors for untrusted remote repos; if the Hub scan failed open, pickles still load.cn_from_pretrained_kwargs: dict[str, Any] = {"cache_dir": hub_cache_dir()}- if remote_cn:+ if remote_cn or component_weight_state(resolved_cn.path) is True:cn_from_pretrained_kwargs["use_safetensors"] = Truecn_model = getattr(diffusers, model_cls_name).from_pretrained(resolved_cn.path,@@ -16385,6 +16427,8 @@ def _base_file_downloaded(# from_single_file(config=repo, subfolder="transformer") always fetches this config. Video keeps# the same exception in video.py's snapshot filter.return True+ if is_never_read_pickle(rfilename):+ return Falseif rfilename.startswith("transformer/"):return include_transformerif "/" not in rfilename: # top-level: only the pipeline manifest is fetched@@ -16432,7 +16476,9 @@ def _pipeline_file_downloaded(rfilename: str) -> bool:name = lower.rsplit("/", 1)[1]if name.startswith(("openvino_", "flax_")):return False- if name.endswith((".onnx", ".onnx_data", ".pb", ".msgpack", ".h5", ".ckpt")):+ if name.endswith((".onnx", ".onnx_data", ".pb", ".msgpack", ".h5")) or is_never_read_pickle(+ name+ ):return Falseif ".fp16." in name or ".bf16." in name or ".non_ema." in name:return Falsediff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex 50b4b3dab01..090ffa75125 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -640,7 +640,9 @@ def prefer_cached_pickle_twins(log: bool = True,roots: Optional[Sequence[Optional[str]]] = None,) -> list:- """Same-stem cached .pt/.pth moves ahead of its uncached safetensors twin; the twin stays behind it as 404 fallback; PREFER_SAFETENSORS_ENV disables."""+ """Safetensors stays first, except a same-stem cached .pt/.pth whose sha256 matches the Hub's record moves ahead of+ its uncached safetensors twin (reused, not re-downloaded); the twin stays behind it as 404 fallback. An unverified+ cached pickle (offline, no LFS record, mismatch) keeps its place behind the twin. PREFER_SAFETENSORS_ENV disables."""import osout = [n for n in names if n]@@ -659,6 +661,18 @@ def prefer_cached_pickle_twins(except Exception: # noqa: BLE001 - an unanswerable question is a noreturn False+ def _verified(name: str) -> bool:+ from .cached_weight_sha import cached_copy_matches_hub++ if _LOCAL_FILES_ONLY.get():+ return False # a load that may not reach the Hub cannot ask it for the hash+ token = _LOAD_HF_TOKEN.get()+ for root in roots:+ hit = _hub_name_cached(repo_id, name, root)+ if hit is not None and cached_copy_matches_hub(repo_id, name, hit, token = token):+ return True+ return False+# A cached container of an artifact beats an uncached one of the SAME artifact: never download a second copy.for comfy in [n for n in out if is_comfy_prequant_filename(n)]:stem = _artifact_stem(comfy)@@ -711,7 +725,8 @@ def prefer_cached_pickle_twins(),None,)- if twin is None or _cached(st):+ # Only a cached pickle the Hub's sha256 vouches for is reused; unverified, safetensors stays first.+ if twin is None or _cached(st) or not _verified(twin):continueout.remove(twin)out.insert(out.index(st), twin)@@ -723,8 +738,8 @@ def prefer_cached_pickle_twins(import loggingsink = logging.getLogger(__name__)sink.info(- "diffusion.prequant_cached_pickle: %s: using the cached %s; the %s twin is not "- "downloaded (set %s=1 to fetch it instead)",+ "diffusion.prequant_cached_pickle: %s: using the cached %s (sha256 matches the Hub); the "+ "%s twin is not downloaded (set %s=1 to fetch it instead)",repo_id,twin,st,diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex c00520d834b..5a59fa8c8c5 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -1561,6 +1561,9 @@ def _fetch_dense_components(fetched = snapshot_download(repo_id,allow_patterns = [f"{c}/*" for c in components],+ ignore_patterns = _dense_component_pickles(+ repo_id, components, revision, hf_token, cache_dir+ ),token = hf_token,cache_dir = cache_dir,revision = revision,@@ -1573,6 +1576,60 @@ def _fetch_dense_components(return False+def _dense_component_pickles(+ repo_id: str,+ components: list,+ revision: Optional[str],+ hf_token: Optional[str],+ cache_dir: Optional[str] = None,+) -> list:+ """ignore_patterns for a dense encoder fetch: every .pt / .pth / .ckpt / .pkl, plus a component's .bin shards when+ the repo lists a safetensors weight for it. A component whose .bin is already cached and matches the Hub's sha256+ (and whose safetensors is not cached) skips its safetensors instead, so nothing is downloaded twice. Without a+ listing only the never-read pickles are dropped, so a component whose only copy is a .bin still loads."""+ from .diffusers_safetensors import (+ _is_model_safetensors,+ pipeline_pickle_files,+ verified_bin_components,+ )++ patterns = [+ f"{c}/*{suffix}"+ for c in components+ for suffix in (".pt", ".pth", ".ckpt", ".pkl", ".pickle")+ ]+ try:+ from huggingface_hub import HfApi+ names = HfApi().list_repo_files(repo_id, revision = revision, token = hf_token)+ except Exception: # noqa: BLE001 - no listing: the .bin of a component without safetensors must stay fetchable+ return patterns+ reused: set = set()+ try:+ import os++ from huggingface_hub import try_to_load_from_cache++ hit = try_to_load_from_cache(+ repo_id, "model_index.json", cache_dir = cache_dir, revision = revision+ )+ if isinstance(hit, str):+ reused = verified_bin_components(+ os.path.dirname(hit), components, token = hf_token, revision = revision+ )+ except Exception: # noqa: BLE001 - unverified: safetensors stays first+ reused = set()+ wanted = tuple(f"{c}/" for c in components)+ for n in pipeline_pickle_files(list(names) + ["model_index.json"]):+ if n.startswith(wanted) and n.split("/", 1)[0] not in reused:+ patterns.append(n)+ for n in names:+ if n.split("/", 1)[0] in reused and (+ _is_model_safetensors(n) or n.endswith(".safetensors.index.json")+ ):+ patterns.append(n)+ return patterns++def _held_locally(repo_id: str, name: Optional[str], hf_token: Optional[str]) -> bool:"""Whether ``repo_id/name`` is on this machine, without touching the network: the operator's mirror, or eithercache root the loader reuses (the live one and huggingface_hub's import-time one, as _download_checkpoint_name)."""diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 06e877fe6f6..92670fec5b1 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -75,6 +75,7 @@ from .diffusion_comfy_quant import (refuse_comfy_quant,)from .diffusion_single_file_trust import assert_safetensors_file, single_file_load_allowed+from .diffusers_safetensors import component_safetensors_kwargs, pipeline_safetensors_kwargsfrom .diffusion_prequant import scoped_local_files_onlyfrom .video_moe_pair import moe_expert_of, moe_partner_filename, moe_pick_pairsfrom .diffusion_cache import (@@ -10042,6 +10043,15 @@ class VideoBackend:else {})+ def _safetensors_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:+ """use_safetensors=True when the base folder ships safetensors for every component this load reads."""+ return pipeline_safetensors_kwargs(+ _base_local_dir or fetch_base,+ passed = {**_pinned_base_kwargs(), **kwargs},+ cache_dir = kwargs.get("cache_dir"),+ pipeline_cls = pipeline_cls,+ )+pipe_kwargs: dict[str, Any] = {"torch_dtype": dtype,"cache_dir": hub_cache_dir(),@@ -10262,6 +10272,7 @@ class VideoBackend:_base_local_dir or fetch_base,**_pinned_base_kwargs(),**pipe_kwargs,+ **_safetensors_kwargs(pipe_kwargs),)except Exception as exc:refusal = (@@ -10494,6 +10505,7 @@ class VideoBackend:_base_local_dir or fetch_base,**_pinned_base_kwargs(),**{**pipe_kwargs, "transformer": ltx25_transformer},+ **_safetensors_kwargs({**pipe_kwargs, "transformer": ltx25_transformer}),)else:@@ -10539,6 +10551,7 @@ class VideoBackend:_base_local_dir or fetch_base,**_pinned_base_kwargs(),**{**pipe_kwargs, **experts},+ **_safetensors_kwargs({**pipe_kwargs, **experts}),)# Dtype dict already loads the Wan VAE in float32; this backstops any path bypassing it (e.g. vae=).@@ -10659,6 +10672,14 @@ class VideoBackend:local_files_only = local_files_only,**({"token": hf_token} if hf_token else {}),**_pinned_base_kwargs(),+ **component_safetensors_kwargs(+ _base_local_dir or fetch_base,+ component,+ cache_dir = hub_cache_dir(),+ model_cls = dense_cls,+ local_files_only = local_files_only,+ revision = _pinned_base_kwargs().get("revision"),+ ),)})diff --git a/studio/backend/hub/utils/snapshot_filters.py b/studio/backend/hub/utils/snapshot_filters.pyindex 6769faa96ab..c94bc177875 100644--- a/studio/backend/hub/utils/snapshot_filters.py+++ b/studio/backend/hub/utils/snapshot_filters.py@@ -163,9 +163,19 @@ def resolve_snapshot_ignore_patterns_for_files(filenames: Iterable[str]) -> list# Only when the root is itself a checkpoint, or the folders are the only runnable copies.if "config.json" in names:ignore.extend(f"{folder}/*" for folder in precision_variant_subfolders(names))+ ignore.extend(diffusers_pickle_files(names))return ignore+def diffusers_pickle_files(filenames: Iterable[str]) -> list[str]:+ """For a diffusers pipeline repo (``model_index.json`` / ``modular_model_index.json`` at its root): the pickles a+ safetensors copy makes redundant, as exact names (a component folder's .pt / .pth / .ckpt / .pkl, a root one beside+ its same-stem .safetensors, a .bin beside the component's safetensors weights). A pickle that is the only copy+ stays. Empty for every other repo (LLM, audio, datasets keep their own rules above)."""+ from core.inference.diffusers_safetensors import pipeline_pickle_files+ return pipeline_pickle_files(filenames)++def sibling_matches_ignore(filename: str, ignore_patterns: Iterable[str]) -> bool:return any(fnmatchcase(filename, pattern) for pattern in ignore_patterns)diff --git a/studio/backend/tests/test_cached_pickle_sha_reuse.py b/studio/backend/tests/test_cached_pickle_sha_reuse.pynew file mode 100644index 00000000000..45b65860a3e--- /dev/null+++ b/studio/backend/tests/test_cached_pickle_sha_reuse.py@@ -0,0 +1,478 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Safetensors first; a .pt / .bin already on disk is reused only when its sha256 matches the Hub's LFS record.++The Hub boundary is faked at huggingface_hub itself (``HfApi.get_paths_info``, ``try_to_load_from_cache``,+``hf_hub_download``) and the local files are really hashed, so these run unchanged against any tree."""++from __future__ import annotations++import hashlib+import json+import os+import types++import huggingface_hub+import pytest+from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError++import core.inference.diffusion_prequant as pq++REPO = "unsloth/Model-INT8"+PT_BYTES = b"pickle checkpoint bytes"+++def _sha(data: bytes) -> str:+ return hashlib.sha256(data).hexdigest()+++class Hub:+ """Cache + listing + LFS records for one repo; records downloads and metadata calls."""++ def __init__(self, root):+ self.root = root+ self.cached: dict = {}+ self.lfs: dict = {}+ self.hosted: set = set()+ self.downloads: list = []+ self.info_calls: list = []+ self.offline = False++ def put(+ self,+ name,+ data,+ *,+ cache = True,+ record = True,+ ):+ self.hosted.add(name)+ if record:+ self.lfs[name] = _sha(data)+ if cache:+ path = self.root / "cache" / name+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(data)+ self.cached[name] = str(path)++ def try_to_load_from_cache(+ self,+ repo_id,+ filename,+ cache_dir = None,+ **_,+ ):+ return self.cached.get(filename) if repo_id == REPO else None++ def hf_hub_download(+ self,+ repo_id,+ filename,+ local_files_only = False,+ **_,+ ):+ hit = self.cached.get(filename)+ if local_files_only or self.offline:+ self.downloads.append((filename, "local"))+ if hit is None:+ raise LocalEntryNotFoundError(f"{filename} not cached")+ return hit+ if filename not in self.hosted:+ raise EntryNotFoundError(f"404 {filename}")+ if hit is None:+ self.downloads.append((filename, "fetched"))+ path = self.root / "cache" / filename+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(b"fresh " + filename.encode())+ self.cached[filename] = str(path)+ return str(path)+ self.downloads.append((filename, "revalidated"))+ return hit++ def get_paths_info(self, repo_id, paths, **_):+ self.info_calls.append(tuple(paths))+ if self.offline:+ raise OSError("no network")+ out = []+ for p in paths:+ if p in self.hosted:+ sha = self.lfs.get(p)+ lfs = {"size": 1, "sha256": sha, "pointer_size": 1} if sha else None+ out.append(types.SimpleNamespace(path = p, lfs = lfs))+ return out++ @property+ def fetched(self):+ return [n for n, how in self.downloads if how == "fetched"]+++def hub(tmp_path, monkeypatch):+ fake = Hub(tmp_path)+ monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", fake.try_to_load_from_cache)+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake.hf_hub_download)+ monkeypatch.setattr(+ huggingface_hub.HfApi, "get_paths_info", lambda self, *a, **k: fake.get_paths_info(*a, **k)+ )+ monkeypatch.setattr(+ pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True+ )+ monkeypatch.delenv(pq.PREQUANT_MIRROR_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_PREQUANT_PREFER_SAFETENSORS", raising = False)+ monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))+ getattr(pq, "_logged_twin_choices", set()).clear()+ try:+ import core.inference.cached_weight_sha as cws++ cws._hub.clear()+ cws._local.clear()+ monkeypatch.setattr(cws, "_persisted", None)+ except ImportError:+ pass+ return fake+++def _src():+ return pq.PrequantSource(+ kind = "repo",+ location = REPO,+ filename = "Model-INT8.safetensors",+ fallback_filenames = ("Model-INT8.pt",),+ )+++def _resolve(**kw):+ return pq._resolve_checkpoint_path(_src(), None, None, scheme = "int8", **kw)+++def test_safetensors_is_downloaded_when_nothing_is_cached(hub):+ hub.put("Model-INT8.safetensors", b"st", cache = False)+ hub.put("Model-INT8.pt", PT_BYTES, cache = False)+ assert _resolve().endswith("Model-INT8.safetensors")+ assert hub.fetched == ["Model-INT8.safetensors"]+++def test_a_cached_pickle_matching_the_hub_sha_is_reused_without_a_download(hub):+ hub.put("Model-INT8.safetensors", b"st", cache = False)+ hub.put("Model-INT8.pt", PT_BYTES)+ assert _resolve() == hub.cached["Model-INT8.pt"]+ assert hub.fetched == []+++def test_a_cached_pickle_with_a_different_sha_is_not_reused(hub):+ hub.put("Model-INT8.safetensors", b"st", cache = False)+ hub.put("Model-INT8.pt", PT_BYTES)+ hub.lfs["Model-INT8.pt"] = _sha(b"the Hub has a newer upload")+ assert _resolve().endswith("Model-INT8.safetensors")+ assert hub.fetched == ["Model-INT8.safetensors"]+++def test_a_cached_pickle_without_an_lfs_record_is_not_reused(hub):+ hub.put("Model-INT8.safetensors", b"st", cache = False)+ hub.put("Model-INT8.pt", PT_BYTES, record = False)+ assert _resolve().endswith("Model-INT8.safetensors")+ assert hub.fetched == ["Model-INT8.safetensors"]+++def test_offline_the_hash_is_unanswerable_so_safetensors_is_tried_first(hub, monkeypatch):+ hub.put("Model-INT8.safetensors", b"st", cache = False)+ hub.put("Model-INT8.pt", PT_BYTES)+ hub.offline = True+ # Fail open: the safetensors name is asked for first; it cannot be fetched offline, so the cached pickle loads.+ assert _resolve() == hub.cached["Model-INT8.pt"]+ assert [n for n, _ in hub.downloads][0] == "Model-INT8.safetensors"+ assert hub.fetched == []+++def test_a_local_files_only_load_never_asks_the_hub_for_the_hash(hub):+ hub.put("Model-INT8.safetensors", b"st", cache = False)+ hub.put("Model-INT8.pt", PT_BYTES)+ token = pq._LOCAL_FILES_ONLY.set(True)+ try:+ assert _resolve(local_files_only = True) == hub.cached["Model-INT8.pt"]+ finally:+ pq._LOCAL_FILES_ONLY.reset(token)+ assert hub.info_calls == []+ assert hub.fetched == []+++def test_a_pickle_only_repo_still_loads_its_pickle(hub):+ hub.put("Model-INT8.pt", PT_BYTES, cache = False)+ assert _resolve().endswith("Model-INT8.pt")+ assert hub.fetched == ["Model-INT8.pt"]+++def test_both_cached_safetensors_wins_and_nothing_is_hashed(hub):+ hub.put("Model-INT8.safetensors", b"st")+ hub.put("Model-INT8.pt", PT_BYTES)+ assert _resolve() == hub.cached["Model-INT8.safetensors"]+ assert hub.info_calls == []+++# --- the hash helper ---------------------------------------------------------------------------------------------+++def test_the_local_hash_is_computed_once_and_remembered(hub, tmp_path, monkeypatch):+ import core.inference.cached_weight_sha as cws++ path = tmp_path / "w.bin"+ path.write_bytes(PT_BYTES)+ opened = []+ real_open = open++ def counting_open(p, *a, **k):+ if str(p) == os.path.realpath(path):+ opened.append(p)+ return real_open(p, *a, **k)++ monkeypatch.setattr("builtins.open", counting_open)+ assert cws.file_sha256(str(path)) == _sha(PT_BYTES)+ assert cws.file_sha256(str(path)) == _sha(PT_BYTES)+ assert len(opened) == 1+ # A fresh process reads the persisted record instead of rehashing.+ cws._local.clear()+ monkeypatch.setattr(cws, "_persisted", None)+ assert cws.file_sha256(str(path)) == _sha(PT_BYTES)+ assert len(opened) == 1+ record = json.loads(cws._store_path().read_text())+ assert record[os.path.realpath(path)][2] == _sha(PT_BYTES)+ # A changed file is rehashed.+ path.write_bytes(b"changed!" * 10)+ assert cws.file_sha256(str(path)) == _sha(b"changed!" * 10)+++def test_a_blob_is_judged_by_its_bytes_not_its_name(hub, tmp_path):+ import core.inference.cached_weight_sha as cws++ # huggingface_hub 1.x names cache blobs by a Xet hash, not the sha256.+ blob = tmp_path / "blobs" / "c5" / ("c" * 64)+ blob.parent.mkdir(parents = True)+ blob.write_bytes(PT_BYTES)+ assert cws.cached_copy_matches_hub(REPO, "x.pt", str(blob), expected = _sha(PT_BYTES))+ assert not cws.cached_copy_matches_hub(REPO, "x.pt", str(blob), expected = _sha(b"other"))+++def test_the_reuse_switch_turns_verification_off(hub, monkeypatch):+ import core.inference.cached_weight_sha as cws++ hub.put("Model-INT8.safetensors", b"st", cache = False)+ hub.put("Model-INT8.pt", PT_BYTES)+ monkeypatch.setenv(cws.ENV_DISABLE, "0")+ assert _resolve().endswith("Model-INT8.safetensors")+ assert hub.fetched == ["Model-INT8.safetensors"]+++def test_hub_lookups_are_cached_per_process(hub):+ import core.inference.cached_weight_sha as cws++ hub.put("Model-INT8.pt", PT_BYTES)+ assert cws.hub_sha256s(REPO, ["Model-INT8.pt"]) == {"Model-INT8.pt": _sha(PT_BYTES)}+ assert cws.hub_sha256s(REPO, ["Model-INT8.pt"]) == {"Model-INT8.pt": _sha(PT_BYTES)}+ assert len(hub.info_calls) == 1+++# --- diffusers pipelines: a cached, verified .bin is loaded instead of downloading safetensors --------------------+++def _snapshot(tmp_path, components):+ root = tmp_path / "hub" / "models--o--pipe" / "snapshots" / ("c" * 40)+ root.mkdir(parents = True)+ index = {+ "_class_name": "FluxPipeline",+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],+ }+ for name, files in components.items():+ index[name] = ["diffusers", "Y"]+ (root / name).mkdir()+ (root / name / "config.json").write_text("{}")+ for fname, data in files.items():+ (root / name / fname).write_bytes(data)+ (root / "scheduler").mkdir()+ (root / "model_index.json").write_text(json.dumps(index))+ return root+++def _hub_records(+ monkeypatch,+ records,+ calls = None,+):+ def info(self, repo_id, paths, **kw):+ if calls is not None:+ calls.append((repo_id, kw.get("revision"), tuple(paths)))+ return [+ types.SimpleNamespace(path = p, lfs = {"sha256": records[p]} if records.get(p) else None)+ for p in paths+ ]++ monkeypatch.setattr(huggingface_hub.HfApi, "get_paths_info", info)+++def test_a_snapshot_of_verified_bins_loads_them_instead_of_downloading_safetensors(+ hub, tmp_path, monkeypatch+):+ from core.inference import diffusers_safetensors as ds++ root = _snapshot(+ tmp_path,+ {+ "transformer": {"diffusion_pytorch_model.bin": b"t"},+ "text_encoder": {"pytorch_model.bin": b"e"},+ },+ )+ calls = []+ _hub_records(+ monkeypatch,+ {+ "transformer/diffusion_pytorch_model.bin": _sha(b"t"),+ "text_encoder/pytorch_model.bin": _sha(b"e"),+ },+ calls,+ )+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": False}+ # A snapshot folder is read as is: checked against its own commit.+ assert calls and calls[0][0] == "o/pipe" and calls[0][1] == "c" * 40+ assert ds.component_safetensors_kwargs(str(root), "transformer") == {"use_safetensors": False}+++def test_one_mismatched_bin_keeps_the_library_default(hub, tmp_path, monkeypatch):+ from core.inference import diffusers_safetensors as ds++ root = _snapshot(+ tmp_path,+ {+ "transformer": {"diffusion_pytorch_model.bin": b"t"},+ "text_encoder": {"pytorch_model.bin": b"e"},+ },+ )+ _hub_records(+ monkeypatch,+ {+ "transformer/diffusion_pytorch_model.bin": _sha(b"t"),+ "text_encoder/pytorch_model.bin": _sha(b"other"),+ },+ )+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+ assert ds.component_safetensors_kwargs(str(root), "text_encoder") == {}+++def test_a_partly_safetensors_snapshot_never_forces_bins(hub, tmp_path, monkeypatch):+ from core.inference import diffusers_safetensors as ds++ root = _snapshot(+ tmp_path,+ {+ "transformer": {"diffusion_pytorch_model.bin": b"t"},+ "vae": {"diffusion_pytorch_model.safetensors": b"v"},+ },+ )+ _hub_records(monkeypatch, {"transformer/diffusion_pytorch_model.bin": _sha(b"t")})+ # The vae has no .bin to reuse, so use_safetensors=False would fetch one: keep the default.+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+++def test_offline_a_snapshot_cached_only_as_bins_loads_them(hub, tmp_path, monkeypatch):+ from core.inference import diffusers_safetensors as ds+ import huggingface_hub.constants as constants++ root = _snapshot(tmp_path, {"transformer": {"diffusion_pytorch_model.bin": b"t"}})+ calls = []+ _hub_records(monkeypatch, {}, calls)+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)+ # Nothing can be fetched or hashed: the cached .bin is the only way the load succeeds.+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": False}+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", False)+ assert ds.pipeline_safetensors_kwargs(str(root), passed = {"local_files_only": True}) == {+ "use_safetensors": False+ }+ assert calls == []+ # A variant load (fp16 twins) is never pointed at the default-name .bin files.+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)+ assert ds.pipeline_safetensors_kwargs(str(root), passed = {"variant": "fp16"}) == {}+ monkeypatch.setattr(constants, "HF_HUB_OFFLINE", False)+ # Online with no matching record the library default stays.+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+++def test_a_plain_local_folder_of_bins_is_never_hub_verified(hub, tmp_path, monkeypatch):+ from core.inference import diffusers_safetensors as ds++ root = tmp_path / "local_pipe"+ (root / "transformer").mkdir(parents = True)+ (root / "transformer" / "diffusion_pytorch_model.bin").write_bytes(b"t")+ (root / "model_index.json").write_text(json.dumps({"transformer": ["diffusers", "Y"]}))+ _hub_records(monkeypatch, {})+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+++def test_the_dense_encoder_fetch_skips_safetensors_for_a_verified_cached_bin(+ hub, tmp_path, monkeypatch+):+ from core.inference import diffusion_te_prequant as tp++ root = _snapshot(tmp_path, {"text_encoder": {"pytorch_model.bin": b"e"}})+ listing = [+ "model_index.json",+ "text_encoder/model.safetensors",+ "text_encoder/pytorch_model.bin",+ ]+ monkeypatch.setattr(huggingface_hub.HfApi, "list_repo_files", lambda self, *a, **k: listing)+ monkeypatch.setattr(+ huggingface_hub,+ "try_to_load_from_cache",+ lambda repo_id, filename, **k: str(root / filename)+ if filename == "model_index.json"+ else None,+ )+ _hub_records(monkeypatch, {"text_encoder/pytorch_model.bin": _sha(b"e")})+ patterns = tp._dense_component_pickles("o/pipe", ["text_encoder"], None, None)+ assert "text_encoder/model.safetensors" in patterns+ assert "text_encoder/pytorch_model.bin" not in patterns+ _hub_records(monkeypatch, {"text_encoder/pytorch_model.bin": _sha(b"different")})+ import core.inference.cached_weight_sha as cws++ cws._hub.clear()+ patterns = tp._dense_component_pickles("o/pipe", ["text_encoder"], None, None)+ assert "text_encoder/pytorch_model.bin" in patterns+ assert "text_encoder/model.safetensors" not in patterns+++def test_an_unpinned_hub_id_is_checked_against_the_hubs_current_main(hub, tmp_path, monkeypatch):+ from core.inference import diffusers_safetensors as ds++ root = _snapshot(tmp_path, {"transformer": {"diffusion_pytorch_model.bin": b"t"}})+ (root.parents[1] / "refs").mkdir()+ (root.parents[1] / "refs" / "main").write_text("c" * 40)+ import huggingface_hub.file_download as fd+ import core.inference.cached_weight_sha as cws++ cache = str(tmp_path / "hub")+ calls = []+ # The Hub's current main no longer lists the .bin (it moved to safetensors only): no reuse.+ _hub_records(monkeypatch, {}, calls)+ real = fd.try_to_load_from_cache+ monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", real)+ assert ds.pipeline_safetensors_kwargs("o/pipe", cache_dir = cache) == {}+ assert calls and calls[-1][1] is None # current main, not the cached commit+ cws._hub.clear() # the unanswered lookup above is otherwise remembered for a minute+ _hub_records(monkeypatch, {"transformer/diffusion_pytorch_model.bin": _sha(b"t")}, calls)+ assert ds.pipeline_safetensors_kwargs("o/pipe", cache_dir = cache) == {"use_safetensors": False}+ assert calls[-1][1] is None+++def test_a_main_answer_is_refreshed_but_a_commit_answer_is_kept(hub, monkeypatch):+ import core.inference.cached_weight_sha as cws++ hub.put("Model-INT8.pt", PT_BYTES)+ clock = [1000.0]+ monkeypatch.setattr(cws.time, "monotonic", lambda: clock[0])+ cws.hub_sha256s(REPO, ["Model-INT8.pt"])+ cws.hub_sha256s(REPO, ["Model-INT8.pt"], revision = "c" * 40)+ assert len(hub.info_calls) == 2+ clock[0] += cws._MUTABLE_TTL_S + 1+ cws.hub_sha256s(REPO, ["Model-INT8.pt"], revision = "c" * 40)+ assert len(hub.info_calls) == 2+ cws.hub_sha256s(REPO, ["Model-INT8.pt"])+ assert len(hub.info_calls) == 3diff --git a/studio/backend/tests/test_prequant_prefer_cached_pickle.py b/studio/backend/tests/test_prequant_prefer_cached_pickle.pyindex 85daf66436b..6c67bc7e76d 100644--- a/studio/backend/tests/test_prequant_prefer_cached_pickle.py+++ b/studio/backend/tests/test_prequant_prefer_cached_pickle.py@@ -1,16 +1,19 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Safetensors is preferred, so a cached .pt repo must not re-download its safetensors twin."""+"""Safetensors is preferred, but a cached .pt whose sha256 matches the Hub's record is reused rather than+re-downloading its safetensors twin; an unverified one (mismatch, no record, offline) keeps safetensors first."""from __future__ import annotations+import hashlibimport typesimport huggingface_hubimport pytestfrom huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError+import core.inference.cached_weight_sha as cwsimport core.inference.diffusion_prequant as pqimport core.inference.diffusion_te_prequant as tpq@@ -34,6 +37,9 @@ class FakeHub:# names a 404 was recorded for, as huggingface_hub's .no_exist markersself.absent: set = set()self.downloads: list = []+ # sha256 the Hub records per name; None = no LFS record. Default: the bytes cache() writes.+ self.hub_sha: dict = {}+ self.sha_calls: list = []for name in cached:self.cache(name)@@ -90,6 +96,19 @@ def hub(tmp_path, monkeypatch):fake = FakeHub(tmp_path)monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", fake.try_to_load_from_cache)monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake.hf_hub_download)++ def hub_sha256s(repo_id, filenames, **_):+ fake.sha_calls.append(tuple(filenames))+ out = {}+ for name in filenames:+ sha = fake.hub_sha.get(name, hashlib.sha256(b"x").hexdigest())+ if sha:+ out[name] = sha+ return out++ monkeypatch.setattr(cws, "hub_sha256s", hub_sha256s)+ monkeypatch.setattr(cws, "_store_path", lambda: tmp_path / "weight_sha256.json")+ monkeypatch.setattr(cws, "_persisted", None)monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True)diff --git a/studio/backend/tests/test_safetensors_first_downloads.py b/studio/backend/tests/test_safetensors_first_downloads.pynew file mode 100644index 00000000000..f110e96facb--- /dev/null+++ b/studio/backend/tests/test_safetensors_first_downloads.py@@ -0,0 +1,404 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Image / video downloads and diffusers loads put .safetensors first: a pickle beside its safetensors twin is never+fetched, a pickle that is the only copy still is, and use_safetensors=True is passed where the folder ships+safetensors."""++from __future__ import annotations++import ast+import json+import struct+import sys+import types+from pathlib import Path++import pytest++from core.inference import diffusers_safetensors as ds+from hub.utils.snapshot_filters import (+ resolve_snapshot_ignore_patterns_for_files,+ sibling_matches_ignore,+)++_BACKEND = Path(__file__).resolve().parents[1]+++def _write_safetensors(path: Path) -> None:+ header = json.dumps({"w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode()+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(struct.pack("<Q", len(header)) + header + b"\0\0\0\0")+++def _pipeline(tmp_path: Path, components: dict) -> Path:+ root = tmp_path / "pipe"+ root.mkdir()+ index = {+ "_class_name": "FluxPipeline",+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],+ }+ for name, files in components.items():+ index[name] = ["diffusers", "Y"]+ (root / name).mkdir()+ (root / name / "config.json").write_text("{}")+ for f in files:+ if f.endswith(".safetensors"):+ _write_safetensors(root / name / f)+ else:+ (root / name / f).write_bytes(b"x")+ (root / "scheduler").mkdir()+ (root / "scheduler" / "scheduler_config.json").write_text("{}")+ (root / "model_index.json").write_text(json.dumps(index))+ return root+++# --- download filters -------------------------------------------------------------------------------------------+++_DIFFUSERS_REPO = [+ "model_index.json",+ "unet/config.json",+ "unet/diffusion_pytorch_model.safetensors",+ "unet/diffusion_pytorch_model.bin",+ "text_encoder/model.safetensors",+ "text_encoder/pytorch_model.bin",+ "safety_checker/pytorch_model.bin", # its only copy: kept+ "vae/diffusion_pytorch_model.fp16.safetensors", # a variant does not make the .bin redundant+ "vae/diffusion_pytorch_model.bin",+ "sd_xl_base_1.0.ckpt", # root pickle with no safetensors twin: the only copy, kept+ "image_encoder/open_clip.pth",+ "Wan2.2-T2V-A14B-FP8.pt", # pre-quant hosted only as .pt: kept+ "Wan2.2-T2V-A14B-INT8.pt",+ "Wan2.2-T2V-A14B-INT8.safetensors", # its twin makes the .pt redundant+]+++def test_a_pipeline_repo_never_downloads_a_pickle_beside_safetensors():+ skipped = set(ds.pipeline_pickle_files(_DIFFUSERS_REPO))+ assert skipped == {+ "unet/diffusion_pytorch_model.bin",+ "text_encoder/pytorch_model.bin",+ "image_encoder/open_clip.pth",+ "Wan2.2-T2V-A14B-INT8.pt",+ }+++def test_a_bin_only_component_still_downloads():+ skipped = set(ds.pipeline_pickle_files(_DIFFUSERS_REPO))+ assert "safety_checker/pytorch_model.bin" not in skipped+ assert "vae/diffusion_pytorch_model.bin" not in skipped+ assert "sd_xl_base_1.0.ckpt" not in skipped+ assert "Wan2.2-T2V-A14B-FP8.pt" not in skipped+++def test_a_modular_pipeline_repo_is_filtered_too():+ names = [+ "modular_model_index.json",+ "transformer/diffusion_pytorch_model.safetensors",+ "transformer/diffusion_pytorch_model.bin",+ "x.pt",+ "x.safetensors",+ ]+ assert set(ds.pipeline_pickle_files(names)) == {+ "transformer/diffusion_pytorch_model.bin",+ "x.pt",+ }+++def test_an_unrelated_safetensors_never_drops_the_only_bin(tmp_path):+ # An adapter / aux .safetensors beside a .bin-only component is not that component's weights.+ names = [+ "model_index.json",+ "transformer/diffusion_pytorch_model.bin",+ "transformer/adapter_model.safetensors",+ "text_encoder/model-00001-of-00002.safetensors",+ "text_encoder/model.safetensors.index.json",+ "text_encoder/pytorch_model.bin",+ ]+ assert set(ds.pipeline_pickle_files(names)) == {"text_encoder/pytorch_model.bin"}+ folder = tmp_path / "transformer"+ folder.mkdir()+ (folder / "diffusion_pytorch_model.bin").write_bytes(b"x")+ (folder / "adapter_model.safetensors").write_bytes(b"x")+ assert ds.component_weight_state(folder) is False+ (folder / "diffusion_pytorch_model.safetensors").write_bytes(b"x")+ assert ds.component_weight_state(folder) is True+++def test_a_non_pipeline_repo_keeps_its_files():+ # LLM / audio repos keep their own rules: no manifest, nothing extra is skipped here.+ names = [+ "config.json",+ "model.safetensors",+ "consolidated.pth",+ "tokenizer.model",+ "optimizer.pt",+ ]+ assert ds.pipeline_pickle_files(names) == []+++def test_the_hub_snapshot_filter_skips_diffusers_pickles():+ ignore = resolve_snapshot_ignore_patterns_for_files(_DIFFUSERS_REPO)+ for name in ("unet/diffusion_pytorch_model.bin", "Wan2.2-T2V-A14B-INT8.pt"):+ assert sibling_matches_ignore(name, ignore), name+ for name in (+ "unet/diffusion_pytorch_model.safetensors",+ "safety_checker/pytorch_model.bin",+ "Wan2.2-T2V-A14B-FP8.pt",+ "sd_xl_base_1.0.ckpt",+ ):+ assert not sibling_matches_ignore(name, ignore), name+++def test_the_hub_snapshot_filter_is_unchanged_for_an_llm_repo():+ names = [+ "config.json",+ "model.safetensors",+ "pytorch_model.bin",+ "training_args.bin",+ "extra.pt",+ ]+ ignore = resolve_snapshot_ignore_patterns_for_files(names)+ assert not sibling_matches_ignore("extra.pt", ignore)+ assert not sibling_matches_ignore("training_args.bin", ignore)+++def test_the_frontend_size_estimate_mirrors_the_pipeline_pickle_filter():+ src = (+ _BACKEND.parent / "frontend" / "src" / "features" / "hub" / "lib" / "dataset-size.ts"+ ).read_text(encoding = "utf-8")+ assert "function diffusersPickleFiles(" in src+ assert '"model_index.json", "modular_model_index.json"' in src+ assert r"/\.(pt|pth|ckpt|pkl|pickle)$/i" in src+ # The same component-weight matcher as the backend: an adapter / variant safetensors never drops the .bin.+ backend = ds._MODEL_SAFETENSORS.pattern+ assert (+ backend == r"^(diffusion_pytorch_model|model)(-\d+-of-\d+)?\.safetensors(\.index\.json)?$"+ )+ assert "/" + backend.replace("\\d", "\\d") + "/" in src+ assert "MODEL_SAFETENSORS_RE.test(basename(n))" in src+ assert "pipelinePickles.has(filename)" in src+ assert "safetensorsStems.has(" in src+++def test_pipeline_and_base_file_filters_drop_never_read_pickles():+ from core.inference.diffusion import _base_file_downloaded, _pipeline_file_downloaded++ for name in ("unet/x.pt", "vae/y.pth", "text_encoder/z.ckpt", "transformer/w.pkl"):+ assert not _pipeline_file_downloaded(name), name+ assert not _base_file_downloaded(name, include_transformer = True), name+ assert _pipeline_file_downloaded("unet/diffusion_pytorch_model.safetensors")+ assert _base_file_downloaded("vae/diffusion_pytorch_model.safetensors")+++def test_the_dense_encoder_fetch_ignores_pickles(monkeypatch):+ from core.inference import diffusion_te_prequant as tp++ listing = [+ "text_encoder/model.safetensors",+ "text_encoder/pytorch_model.bin",+ "text_encoder_2/pytorch_model.bin",+ ]++ class _Api:+ def list_repo_files(self, *_a, **_k):+ return listing++ monkeypatch.setitem(sys.modules, "huggingface_hub", types.SimpleNamespace(HfApi = _Api))+ patterns = tp._dense_component_pickles("o/r", ["text_encoder", "text_encoder_2"], None, None)+ assert "text_encoder/pytorch_model.bin" in patterns+ assert "text_encoder_2/pytorch_model.bin" not in patterns # its only copy+ assert "text_encoder/*.pt" in patterns and "text_encoder_2/*.ckpt" in patterns+++def test_the_dense_encoder_fetch_without_a_listing_drops_only_never_read_pickles(monkeypatch):+ from core.inference import diffusion_te_prequant as tp++ class _Api:+ def list_repo_files(self, *_a, **_k):+ raise OSError("offline")++ monkeypatch.setitem(sys.modules, "huggingface_hub", types.SimpleNamespace(HfApi = _Api))+ patterns = tp._dense_component_pickles("o/r", ["text_encoder"], None, None)+ assert not any(p.endswith(".bin") for p in patterns)+ assert "text_encoder/*.pt" in patterns+++# --- use_safetensors=True ---------------------------------------------------------------------------------------+++def test_a_folder_shipping_safetensors_everywhere_loads_with_use_safetensors(tmp_path):+ root = _pipeline(+ tmp_path,+ {+ "transformer": ["diffusion_pytorch_model.safetensors"],+ "text_encoder": ["model.safetensors", "pytorch_model.bin"],+ "vae": ["diffusion_pytorch_model.safetensors"],+ },+ )+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": True}+++def test_a_bin_only_component_keeps_the_library_default(tmp_path):+ root = _pipeline(+ tmp_path,+ {+ "transformer": ["diffusion_pytorch_model.safetensors"],+ "text_encoder": ["pytorch_model.bin"],+ },+ )+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+ assert ds.pipeline_safetensors_kwargs(str(root), passed = {"text_encoder": object()}) == {+ "use_safetensors": True+ }+++def test_a_partly_cached_snapshot_is_unknown_not_complete(tmp_path):+ """A weight-bearing component that is missing, or present with its config only (a partly cached snapshot),+ could still be .bin-only on the Hub: never force safetensors on it."""+ root = _pipeline(tmp_path, {"transformer": ["diffusion_pytorch_model.safetensors"]})+ index = json.loads((root / "model_index.json").read_text())+ index["text_encoder"] = ["transformers", "CLIPTextModel"]+ index["tokenizer"] = ["transformers", "CLIPTokenizer"]+ (root / "model_index.json").write_text(json.dumps(index))+ # tokenizer and scheduler hold no weights, so their absence or config-only folder is no evidence either way+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+ (root / "text_encoder").mkdir()+ (root / "text_encoder" / "config.json").write_text("{}")+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+ _write_safetensors(root / "text_encoder" / "model.safetensors")+ assert ds.pipeline_safetensors_kwargs(str(root)) == {"use_safetensors": True}+++def test_a_variant_only_safetensors_does_not_count(tmp_path):+ root = _pipeline(+ tmp_path,+ {"vae": ["diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.bin"]},+ )+ assert ds.pipeline_safetensors_kwargs(str(root)) == {}+++def test_an_unknown_source_keeps_the_library_default(tmp_path):+ assert ds.pipeline_safetensors_kwargs("nobody/not-cached-anywhere-xyz") == {}+ assert ds.pipeline_safetensors_kwargs(str(tmp_path)) == {} # no manifest+ assert ds.pipeline_safetensors_kwargs("") == {}+++def test_a_non_diffusers_pipeline_class_never_gets_the_kwarg(tmp_path, monkeypatch):+ root = _pipeline(tmp_path, {"transformer": ["diffusion_pytorch_model.safetensors"]})++ class DiffusionPipeline:+ pass++ class FluxPipeline(DiffusionPipeline):+ pass++ class Custom:+ pass++ monkeypatch.setitem(+ sys.modules, "diffusers", types.SimpleNamespace(DiffusionPipeline = DiffusionPipeline)+ )+ assert ds.pipeline_safetensors_kwargs(str(root), pipeline_cls = Custom) == {}+ assert ds.pipeline_safetensors_kwargs(str(root), pipeline_cls = FluxPipeline) == {+ "use_safetensors": True+ }+++def test_a_component_load_takes_use_safetensors_only_when_its_folder_ships_them(tmp_path):+ root = _pipeline(+ tmp_path,+ {+ "transformer": ["diffusion_pytorch_model.safetensors"],+ "text_encoder": ["pytorch_model.bin"],+ },+ )+ assert ds.component_safetensors_kwargs(str(root), "transformer") == {"use_safetensors": True}+ assert ds.component_safetensors_kwargs(str(root), "text_encoder") == {}+ assert ds.component_safetensors_kwargs(str(root), "missing") == {}+++def test_a_hub_id_uses_its_cached_snapshot(tmp_path, monkeypatch):+ root = _pipeline(tmp_path, {"transformer": ["diffusion_pytorch_model.safetensors"]})++ def _cached(+ repo_id,+ filename,+ cache_dir = None,+ ):+ path = root / filename+ return str(path) if path.is_file() else None++ monkeypatch.setitem(+ sys.modules, "huggingface_hub", types.SimpleNamespace(try_to_load_from_cache = _cached)+ )+ assert ds.pipeline_safetensors_kwargs("org/model") == {"use_safetensors": True}+ assert ds.component_safetensors_kwargs("org/model", "transformer") == {"use_safetensors": True}+++def _calls_with_helper(path: Path, helper: str) -> int:+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ count = 0+ for node in ast.walk(tree):+ if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "from_pretrained":+ for kw in node.keywords:+ if kw.arg is None and isinstance(kw.value, ast.Call):+ name = getattr(kw.value.func, "id", None) or getattr(+ kw.value.func, "attr", None+ )+ if name == helper:+ count += 1+ return count+++def test_every_dense_diffusers_load_asks_for_safetensors():+ image = _BACKEND / "core" / "inference" / "diffusion.py"+ video = _BACKEND / "core" / "inference" / "video.py"+ assert _calls_with_helper(image, "pipeline_safetensors_kwargs") == 3+ assert _calls_with_helper(image, "component_safetensors_kwargs") == 1+ assert _calls_with_helper(video, "_safetensors_kwargs") == 3+ assert _calls_with_helper(video, "component_safetensors_kwargs") == 1+++def test_a_pinned_revision_is_judged_on_its_own_snapshot(tmp_path):+ # Cached main ships safetensors, the pinned commit only a .bin: the pinned load must not be forced to safetensors.+ cache = tmp_path / "hub"+ repo = cache / "models--o--pipe"+ main_rev, pinned = "a" * 40, "b" * 40+ for rev, weight in (+ (main_rev, "diffusion_pytorch_model.safetensors"),+ (pinned, "diffusion_pytorch_model.bin"),+ ):+ snap = repo / "snapshots" / rev+ (snap / "transformer").mkdir(parents = True)+ (snap / "transformer" / "config.json").write_text("{}")+ if weight.endswith(".safetensors"):+ _write_safetensors(snap / "transformer" / weight)+ else:+ (snap / "transformer" / weight).write_bytes(b"x")+ (snap / "model_index.json").write_text(json.dumps({"transformer": ["diffusers", "Y"]}))+ (repo / "refs").mkdir()+ (repo / "refs" / "main").write_text(main_rev)+ assert ds.pipeline_safetensors_kwargs("o/pipe", cache_dir = str(cache)) == {+ "use_safetensors": True+ }+ assert ds.pipeline_safetensors_kwargs(+ "o/pipe", passed = {"revision": pinned}, cache_dir = str(cache)+ ) != {"use_safetensors": True}+ assert ds.component_safetensors_kwargs(+ "o/pipe", "transformer", cache_dir = str(cache), revision = pinned+ ) != {"use_safetensors": True}+ assert ds.component_safetensors_kwargs("o/pipe", "transformer", cache_dir = str(cache)) == {+ "use_safetensors": True+ }+++def test_an_orphan_shard_without_its_index_is_not_safetensors_weights(tmp_path):+ folder = tmp_path / "transformer"+ folder.mkdir()+ (folder / "diffusion_pytorch_model.bin").write_bytes(b"x")+ (folder / "diffusion_pytorch_model-00001-of-00002.safetensors").write_bytes(b"x")+ assert ds.component_weight_state(folder) is False+ (folder / "diffusion_pytorch_model.safetensors.index.json").write_text("{}")+ assert ds.component_weight_state(folder) is Truediff --git a/studio/backend/tests/test_video_prequant_safetensors_twins.py b/studio/backend/tests/test_video_prequant_safetensors_twins.pyindex c6e6b7a174a..8d157e4c263 100644--- a/studio/backend/tests/test_video_prequant_safetensors_twins.py+++ b/studio/backend/tests/test_video_prequant_safetensors_twins.py@@ -6,6 +6,7 @@ a new user fetches only the twin, a cached .pt is kept, an unreadable twin is skfrom __future__ import annotations+import hashlibimport threadingimport types@@ -13,6 +14,7 @@ import huggingface_hubimport pytestfrom huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError+import core.inference.cached_weight_sha as cwsimport core.inference.diffusion_prequant as pqfrom core.inference import video_families as vf@@ -260,6 +262,14 @@ def _setup(hub = FakeHub(tmp_path, src.location, hosted)monkeypatch.setattr(huggingface_hub, "try_to_load_from_cache", hub.try_to_load_from_cache)monkeypatch.setattr(huggingface_hub, "hf_hub_download", hub.hf_hub_download)+ # The Hub records the sha256 of the bytes the fake cache writes: a cached pickle is a verified copy.+ monkeypatch.setattr(+ cws,+ "hub_sha256s",+ lambda repo_id, names, **_: {n: hashlib.sha256(b"x").hexdigest() for n in names},+ )+ monkeypatch.setattr(cws, "_store_path", lambda: tmp_path / "weight_sha256.json")+ monkeypatch.setattr(cws, "_persisted", None)monkeypatch.setattr(pq,"restricted_prequant_load_supported",diff --git a/studio/frontend/src/features/hub/lib/dataset-size.ts b/studio/frontend/src/features/hub/lib/dataset-size.tsindex c64275f0b75..7ef4f9a002b 100644--- a/studio/frontend/src/features/hub/lib/dataset-size.ts+++ b/studio/frontend/src/features/hub/lib/dataset-size.ts@@ -318,6 +318,49 @@ function redundantTorchBinFiles(names: string[]): Set<string> {return out;}+// Mirrors pipeline_pickle_files (diffusers_safetensors.py, via diffusers_pickle_files in snapshot_filters.py):+// a diffusers pipeline repo skips a .pt / .pth / .ckpt / .pkl inside a component folder or beside its same-stem+// .safetensors twin (a root pickle with no twin is the only copy and stays), and a .bin beside safetensors weights.+const PIPELINE_MANIFESTS = ["model_index.json", "modular_model_index.json"];+const NEVER_READ_PICKLE_RE = /\.(pt|pth|ckpt|pkl|pickle)$/i;+// Mirrors _MODEL_SAFETENSORS: only a component's own non-variant weights make its .bin redundant.+const MODEL_SAFETENSORS_RE =+ /^(diffusion_pytorch_model|model)(-\d+-of-\d+)?\.safetensors(\.index\.json)?$/;++function folderOf(name: string): string {+ const slash = name.lastIndexOf("/");+ return slash >= 0 ? name.slice(0, slash) : "";+}++function diffusersPickleFiles(names: string[]): Set<string> {+ const out = new Set<string>();+ if (!names.some((n) => PIPELINE_MANIFESTS.includes(n))) return out;+ const safetensorsFolders = new Set(+ names+ .filter((n) => MODEL_SAFETENSORS_RE.test(basename(n)))+ .map(folderOf),+ );+ const safetensorsStems = new Set(+ names+ .filter((n) => n.toLowerCase().endsWith(".safetensors"))+ .map((n) => n.slice(0, -".safetensors".length).toLowerCase()),+ );+ for (const n of names) {+ const lower = n.toLowerCase();+ if (NEVER_READ_PICKLE_RE.test(lower)) {+ if (folderOf(n) !== "" || safetensorsStems.has(lower.slice(0, lower.lastIndexOf(".")))) {+ out.add(n);+ }+ } else if (+ (lower.endsWith(".bin") || lower.endsWith(".bin.index.json")) &&+ safetensorsFolders.has(folderOf(n))+ ) {+ out.add(n);+ }+ }+ return out;+}+// Mirrors PRECISION_FOLDER_SOURCE and precision_variant_subfolders in snapshot_filters.py.const PRECISION_FOLDER_SOURCE ="(?:mlx[-_]?)?(?:[0-9]{1,2}[-_]?bits?(?:[-_][a-z0-9]+)*|bf16|fp16|f16|fp32|f32|fp8|mxfp4|nvfp4)";@@ -380,6 +423,7 @@ function isSnapshotIgnored(skipDuplicateFormats: boolean,redundantBins: Set<string>,precisionFolders: Set<string> = new Set(),+ pipelinePickles: Set<string> = new Set(),): boolean {const lower = filename.toLowerCase();const folder = filename.includes("/") ? filename.slice(0, filename.indexOf("/")) : "";@@ -390,6 +434,7 @@ function isSnapshotIgnored(lower.startsWith("openvino/") ||lower.startsWith("mlx/") ||lower.endsWith(".bin.index.json.bak") ||+ pipelinePickles.has(filename) ||(skipConsolidated && lower.startsWith("consolidated")) ||(skipDuplicateFormats &&(DUPLICATE_WEIGHT_FORMAT_RE.test(filename) || redundantBins.has(filename) ||@@ -433,6 +478,7 @@ export function fetchModelSize(const precisionFolders = names.includes("config.json")? precisionVariantFolders(names): new Set<string>();+ const pipelinePickles = diffusersPickleFiles(names);let total = 0;let weights = 0;for (const s of siblings) {@@ -446,6 +492,7 @@ export function fetchModelSize(skipDuplicateFormats,redundantBins,precisionFolders,+ pipelinePickles,)) {continue;diff --git a/studio/frontend/tests/hub-model-size-diffusers-pickles.test.ts b/studio/frontend/tests/hub-model-size-diffusers-pickles.test.tsnew file mode 100644index 00000000000..b7c4c727f43--- /dev/null+++ b/studio/frontend/tests/hub-model-size-diffusers-pickles.test.ts@@ -0,0 +1,56 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();++const { fetchModelSize } = await import(+ "../src/features/hub/lib/dataset-size.ts"+);++function stubSiblings(sizes: Record<string, number>): void {+ globalThis.fetch = (async () =>+ new Response(+ JSON.stringify({+ siblings: Object.entries(sizes).map(([rfilename, size]) => ({+ rfilename,+ size,+ })),+ }),+ { status: 200, headers: { "content-type": "application/json" } },+ )) as typeof fetch;+}++test("a diffusers pipeline skips the pickles its safetensors make redundant", async () => {+ stubSiblings({+ "model_index.json": 1,+ "unet/diffusion_pytorch_model.safetensors": 100,+ "unet/diffusion_pytorch_model.bin": 100,+ "v1-5-pruned.safetensors": 50,+ "v1-5-pruned.ckpt": 50,+ "Model-FP8.pt": 30,+ });+ // The root .pt with no safetensors twin is the only copy and stays.+ assert.deepEqual(await fetchModelSize("acme/sd-shaped"), {+ totalBytes: 1 + 100 + 50 + 30,+ weightsBytes: 100 + 50 + 30,+ });+});++test("an adapter safetensors never drops a component's only .bin", async () => {+ stubSiblings({+ "model_index.json": 1,+ "transformer/adapter_model.safetensors": 5,+ "transformer/diffusion_pytorch_model.bin": 200,+ "vae/diffusion_pytorch_model.fp16.safetensors": 10,+ "vae/diffusion_pytorch_model.bin": 20,+ });+ assert.deepEqual(await fetchModelSize("acme/adapter-shaped"), {+ totalBytes: 1 + 5 + 200 + 10 + 20,+ weightsBytes: 5 + 200 + 10 + 20,+ });+});