Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cache fp32-shard video components cast to float16 for later loads (#15303)

Daniel Hand4be24722242
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.py
new file mode 100644
index 00000000000..15046aebdf5
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_cast_cache.py
@@ -0,0 +1,814 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Local float16 cache of components loaded from fp32 shards (e.g. Wan), written from the live tensors after publish.
+
+Later loads read the half-size file at the same dtype (a no-op cast, bit-identical); a sampled source comparison
+refuses a module changed in place. Shares ``diffusion_prequant_cache``'s root, locks, manifest and LRU.
+Kill switch: ``UNSLOTH_DIFFUSION_CAST_CACHE=0`` (``UNSLOTH_DIFFUSION_PREQUANT_CACHE=0`` disables it too).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import struct
+import tempfile
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+from . import diffusion_prequant_cache as _cache
+
+CAST_CACHE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE"
+CAST_LAYOUT_VERSION = 1
+CAST_META_KEY = "unsloth_cast_component"
+_LIBRARIES = ("diffusers", "transformers")
+_STEMS = {"diffusers": "diffusion_pytorch_model", "transformers": "model"}
+_WIDE_TAGS = ("F32", "F64")
+_FLOAT_TAGS = ("F64", "F32", "F16", "BF16", "F8_E4M3", "F8_E5M2", "F8_E8M0")
+# Per-shard source sample cap; a bigger read would grow the page cache.
+_SAMPLE_MAX_BYTES = 64 << 20
+_DROP_EVERY_BYTES = 256 << 20
+# Per-component wait for group-offloaded tensors to return to host before reading the device copy.
+_ONLOAD_WAIT_SECONDS = 10.0
+
+
+def enabled() -> bool:
+ raw = (os.environ.get(CAST_CACHE_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no", "none") and _cache.enabled()
+
+
+def _is_fp16(dtype: Any) -> bool:
+ return str(dtype) in ("torch.float16", "float16")
+
+
+def _component_dtype(torch_dtype: Any, component: str) -> Any:
+ if isinstance(torch_dtype, dict):
+ return torch_dtype.get(component, torch_dtype.get("default"))
+ return torch_dtype
+
+
+def _minor(module_name: str) -> Optional[str]:
+ try:
+ import importlib.metadata as md
+ return _cache._minor(md.version(module_name))
+ except Exception: # noqa: BLE001
+ return None
+
+
+@dataclass
+class CastCandidate:
+ component: str
+ library: str
+ class_name: str
+ cls: Any
+ dtype: Any
+ source: str
+ shards: list
+ config_path: str
+ key: str
+ parts: dict
+ names: list = field(default_factory = list)
+
+
+def _read_json(path: Optional[str]) -> Optional[dict]:
+ if not path:
+ return None
+ try:
+ with open(path, "r", encoding = "utf-8") as fh:
+ data = json.load(fh)
+ return data if isinstance(data, dict) else None
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _shards_of(
+ source: str, component: str, stem: str, cache_dir: Optional[str], revision: Optional[str]
+) -> Optional[list]:
+ """The shard files ``from_pretrained(source/component)`` reads, or None when unsure (both layouts, or any missing)."""
+ index = _cache._local_file(
+ source, f"{component}/{stem}.safetensors.index.json", cache_dir, revision
+ )
+ single = _cache._local_file(source, f"{component}/{stem}.safetensors", cache_dir, revision)
+ if index is not None and single is not None:
+ return None
+ if single is not None:
+ return [str(single)]
+ weight_map = (_read_json(index) or {}).get("weight_map") or {}
+ names = sorted(set(str(v) for v in weight_map.values()))
+ if not names:
+ return None
+ paths = [_cache._local_file(source, f"{component}/{n}", cache_dir, revision) for n in names]
+ return [str(p) for p in paths] if all(paths) else None
+
+
+def _shard_header(path: str) -> Optional[dict]:
+ raw = _cache.read_header_bytes(path)
+ if raw is None:
+ return None
+ try:
+ header = json.loads(raw[8:].decode("utf-8"))
+ except Exception: # noqa: BLE001
+ return None
+ header.pop("__metadata__", None)
+ return header
+
+
+def _all_wide(shards: list) -> bool:
+ """Every floating tensor of every shard is wider than float16 (the only case the cast halves)."""
+ seen = False
+ for shard in shards:
+ header = _shard_header(shard)
+ if header is None:
+ return False
+ for entry in header.values():
+ tag = entry.get("dtype") if isinstance(entry, dict) else None
+ if tag in _FLOAT_TAGS:
+ if tag not in _WIDE_TAGS:
+ return False
+ seen = True
+ return seen
+
+
+def _load_class(library: str, class_name: str) -> Any:
+ try:
+ if library == "diffusers":
+ import diffusers as module
+ else:
+ import transformers as module
+ return getattr(module, class_name, None)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def candidates(
+ source: Any,
+ *,
+ torch_dtype: Any,
+ cache_dir: Optional[str] = None,
+ revision: Optional[str] = None,
+ skip: Iterable[str] = (),
+ base_repo: Optional[str] = None,
+ logger: Any = None,
+) -> list:
+ """Components at ``source`` that load fp32 shards at float16, with cache keys. Local reads only; never raises."""
+ if not enabled() or not source:
+ return []
+ out: list = []
+ try:
+ import torch
+
+ source = str(source)
+ index = _read_json(_cache._local_file(source, "model_index.json", cache_dir, revision))
+ if index is None:
+ return []
+ skip = set(skip)
+ for component, spec in sorted(index.items()):
+ if component.startswith("_") or component in skip:
+ continue
+ if not isinstance(spec, (list, tuple)) or len(spec) != 2:
+ continue
+ library, class_name = spec
+ if library not in _LIBRARIES or not isinstance(class_name, str):
+ continue
+ dtype = _component_dtype(torch_dtype, component)
+ if not _is_fp16(dtype):
+ continue
+ cls = _load_class(library, class_name)
+ if cls is None or not isinstance(cls, type) or not issubclass(cls, torch.nn.Module):
+ continue
+ shards = _shards_of(source, component, _STEMS[library], cache_dir, revision)
+ if not shards or not _all_wide(shards):
+ continue
+ config_path = _cache._local_file(
+ source, f"{component}/config.json", cache_dir, revision
+ )
+ if config_path is None:
+ continue
+ fp = _cache.source_fingerprint(shards)
+ if fp is None:
+ continue
+ parts = {
+ "kind": "cast",
+ "layout": CAST_LAYOUT_VERSION,
+ "policy": _cache.POLICY_VERSION,
+ "component": component,
+ "library": library,
+ "class": class_name,
+ "dtype": str(dtype).replace("torch.", ""),
+ "keep_in_fp32": sorted(
+ str(m) for m in (getattr(cls, "_keep_in_fp32_modules", None) or ())
+ ),
+ "source": fp,
+ "config": hashlib.sha256(Path(config_path).read_bytes()).hexdigest(),
+ "base": _cache.base_identity(base_repo or source),
+ "torch": _cache._minor(torch.__version__),
+ library: _minor(library),
+ }
+ out.append(
+ CastCandidate(
+ component = component,
+ library = library,
+ class_name = class_name,
+ cls = cls,
+ dtype = dtype,
+ source = source,
+ shards = shards,
+ config_path = str(config_path),
+ key = _cache.cache_key(parts),
+ parts = parts,
+ )
+ )
+ except Exception as exc: # noqa: BLE001
+ _log(logger, "warning", f"candidates skipped ({type(exc).__name__}: {str(exc)[:200]})")
+ return []
+ return out
+
+
+def _log(logger: Any, level: str, msg: str) -> None:
+ if logger is not None:
+ try:
+ getattr(logger, level)("diffusion.cast_cache: %s", msg)
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _header_metadata(path: Any) -> Optional[dict]:
+ raw = _cache.read_header_bytes(path)
+ if raw is None:
+ return None
+ try:
+ meta = json.loads(raw[8:].decode("utf-8")).get("__metadata__") or {}
+ return json.loads(meta[CAST_META_KEY])
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _weights_link(kdir: Path, library: str) -> None:
+ """``from_pretrained`` reads ``<stem>.safetensors``; the entry's artifact is ``model.safetensors``."""
+ name = f"{_STEMS[library]}.safetensors"
+ if name == _cache.FILE_NAME:
+ return
+ link = kdir / name
+ if link.exists():
+ return
+ try:
+ os.symlink(_cache.FILE_NAME, link)
+ except OSError:
+ os.link(kdir / _cache.FILE_NAME, link) # no symlink rights (Windows): a hard link
+
+
+def _load_kwargs(cand: CastCandidate) -> dict:
+ kwargs: dict = {"low_cpu_mem_usage": True, "use_safetensors": True, "output_loading_info": True}
+ use_dtype_kw = False
+ if cand.library == "transformers":
+ try:
+ import transformers
+ from packaging import version
+ use_dtype_kw = version.parse(transformers.__version__) >= version.parse("4.56.0")
+ except Exception: # noqa: BLE001
+ use_dtype_kw = False
+ kwargs["dtype" if use_dtype_kw else "torch_dtype"] = cand.dtype
+ return kwargs
+
+
+def read(
+ cand: CastCandidate,
+ *,
+ logger: Any = None,
+ cancel_check: Optional[Any] = None,
+) -> Any:
+ """The cached cast ``cand.component``, or None (a miss: the dense load runs). Never raises."""
+ if not enabled():
+ return None
+ try:
+ root = _cache.cache_root()
+ if not (root / cand.key / _cache.MANIFEST_NAME).is_file():
+ return None
+ if _cache._hub_source_stale(cand.shards, logger):
+ return None
+ with _cache._cache_lock():
+ artifact = _cache._entry(cand.key)
+ if artifact is None:
+ return None
+ with _cache._lock:
+ if os.path.realpath(str(artifact)) in _cache._REJECTED:
+ return None
+ _cache._touch(artifact.parent)
+ meta = _header_metadata(artifact)
+ if not isinstance(meta, dict) or meta.get("key") != cand.key:
+ _cache.note_rejected(artifact, logger)
+ return None
+ if not _cache.verify_entry(artifact, logger):
+ _cache.note_rejected(artifact, logger)
+ return None
+ kdir = artifact.parent
+ if not (kdir / "config.json").is_file():
+ _cache.note_rejected(artifact, logger)
+ return None
+ _weights_link(kdir, cand.library)
+ t0 = time.perf_counter()
+ model, info = cand.cls.from_pretrained(str(kdir), **_load_kwargs(cand))
+ bad = {
+ k: list(v)
+ for k, v in (info or {}).items()
+ if k in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs") and v
+ }
+ if bad:
+ _log(
+ logger,
+ "warning",
+ f"{cand.component}: entry did not load cleanly ({str(bad)[:200]}); not used",
+ )
+ _cache.note_rejected(artifact, logger)
+ return None
+ if any(t.device.type == "meta" for t in (*model.parameters(), *model.buffers())):
+ _cache.note_rejected(artifact, logger)
+ return None
+ model.eval()
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: read the cached {meta.get('dtype')} cast ({artifact.stat().st_size / 2**30:.2f} GiB) "
+ f"in {time.perf_counter() - t0:.1f} s instead of the {meta.get('source_gib', '?')} GiB source shards",
+ )
+ _cache.note_hit(artifact, logger)
+ return model
+ except Exception as exc: # noqa: BLE001 - an unusable entry is a miss
+ _log(
+ logger,
+ "warning",
+ f"{cand.component}: read failed: {type(exc).__name__}: {str(exc)[:200]}",
+ )
+ return None
+
+
+def read_all(
+ cands: list,
+ *,
+ logger: Any = None,
+ cancel_check: Optional[Any] = None,
+) -> dict:
+ """``{component: module}`` of every cached candidate. Never raises."""
+ out: dict = {}
+ for cand in cands or ():
+ if cancel_check is not None:
+ cancel_check()
+ module = read(cand, logger = logger, cancel_check = cancel_check)
+ if module is not None:
+ out[cand.component] = module
+ return out
+
+
+def _tied_keys(module: Any) -> set:
+ keys: set = set()
+ for attr in ("_tied_weights_keys", "all_tied_weights_keys"):
+ value = getattr(module, attr, None)
+ if isinstance(value, dict):
+ keys.update(str(k) for k in value)
+ elif isinstance(value, (list, tuple, set)):
+ keys.update(str(k) for k in value)
+ return keys
+
+
+def _resolve(module: Any, name: str) -> Any:
+ parent_name, _, leaf = name.rpartition(".")
+ parent = module.get_submodule(parent_name) if parent_name else module
+ tensor = (
+ parent._parameters.get(leaf) if leaf in parent._parameters else parent._buffers.get(leaf)
+ )
+ if tensor is None:
+ raise KeyError(name)
+ return tensor.data if hasattr(tensor, "data") else tensor
+
+
+def _state_names(module: Any) -> Optional[list]:
+ """``[(name, dtype, shape)]`` to write: the state dict minus tied aliases ``from_pretrained`` restores."""
+ import torch
+
+ tied = _tied_keys(module)
+ seen: dict = {}
+ out = []
+ for name, tensor in module.state_dict(keep_vars = True).items():
+ if type(tensor) not in (torch.Tensor, torch.nn.Parameter) or tensor.device.type == "meta":
+ return None
+ ident = (tensor.data_ptr(), tensor.dtype, tuple(tensor.shape), tensor.device)
+ if tensor.numel() and ident in seen and name in tied:
+ continue
+ seen.setdefault(ident, name)
+ out.append((name, tensor.dtype, tuple(tensor.shape)))
+ return out
+
+
+def capture(
+ pipe: Any,
+ cands: list,
+ *,
+ logger: Any = None,
+) -> list:
+ """Right after ``from_pretrained``: record what each missed candidate's module holds (names only, no tensors)."""
+ out = []
+ for cand in cands or ():
+ try:
+ module = getattr(pipe, cand.component, None)
+ if module is None or type(module).__name__ != cand.class_name:
+ continue
+ names = _state_names(module)
+ if not names:
+ continue
+ cand.names = names
+ out.append(cand)
+ except Exception as exc: # noqa: BLE001
+ _log(logger, "info", f"{cand.component}: not cached ({type(exc).__name__}: {exc})")
+ return out
+
+
+def _hooked(module: Any) -> bool:
+ return any(
+ getattr(m, "_diffusers_hook", None) is not None or getattr(m, "_hf_hook", None) is not None
+ for m in module.modules()
+ )
+
+
+class _Reader:
+ """Host copies of one component's live tensors; device copies are read on a side stream after one device
+ synchronise (an upload may be queued on another stream), never behind the render on the default stream."""
+
+ def __init__(self, module: Any, check: Any) -> None:
+ self.module = module
+ self.check = check
+ self.wait = _ONLOAD_WAIT_SECONDS if _hooked(module) else 0.0
+ self.synced = False
+ self.stream = None
+
+ def tensor(self, name: str) -> Any:
+ while True:
+ tensor = _resolve(self.module, name)
+ if tensor.device.type == "cpu":
+ return tensor
+ if self.wait <= 0:
+ break
+ self.check()
+ time.sleep(0.25)
+ self.wait -= 0.25
+ import torch
+
+ if tensor.device.type != "cuda":
+ return tensor.to("cpu")
+ if not self.synced:
+ torch.cuda.synchronize(tensor.device)
+ self.synced = True
+ self.stream = torch.cuda.Stream(device = tensor.device)
+ with torch.cuda.stream(self.stream):
+ host = tensor.to("cpu", non_blocking = False)
+ self.stream.synchronize()
+ return host
+
+
+def _sample_keys(header: dict, names: dict) -> list:
+ """The largest cast tensor up to ``_SAMPLE_MAX_BYTES`` and the smallest tensor kept wide, of one shard."""
+ import torch
+
+ cast, wide = [], []
+ for key, entry in header.items():
+ if key not in names or entry.get("dtype") not in _WIDE_TAGS:
+ continue
+ begin, end = (int(v) for v in entry["data_offsets"])
+ if end - begin > _SAMPLE_MAX_BYTES:
+ continue
+ (wide if names[key] in (torch.float32, torch.float64) else cast).append((end - begin, key))
+ return ([max(cast)[1]] if cast else []) + ([min(wide)[1]] if wide else [])
+
+
+def _matches_source(cand: CastCandidate, reader: _Reader) -> Optional[str]:
+ """None when sampled live tensors equal the source cast bit for bit, else why not (a module changed in place)."""
+ import torch
+ from safetensors import safe_open
+
+ names = {n: d for n, d, _s in cand.names}
+ sampled = 0
+ for shard in cand.shards:
+ picks = _sample_keys(_shard_header(shard) or {}, names)
+ if not picks:
+ continue
+ with safe_open(shard, framework = "pt") as fh:
+ for key in picks:
+ reader.check()
+ live = reader.tensor(key)
+ src = fh.get_tensor(key)
+ if tuple(src.shape) != tuple(live.shape):
+ return f"{key} changed shape"
+ if not torch.equal(
+ src.to(live.dtype).view(-1).view(torch.uint8),
+ live.reshape(-1).view(torch.uint8),
+ ):
+ return f"{key} differs from its source cast"
+ del src, live
+ sampled += 1
+ return None if sampled else "no tensor to compare against the source"
+
+
+def _drop_pages(fd: int, upto: int) -> None:
+ try:
+ os.fdatasync(fd)
+ os.posix_fadvise(fd, 0, upto, os.POSIX_FADV_DONTNEED)
+ except (AttributeError, OSError):
+ pass
+
+
+def _write(path: str, cand: CastCandidate, metadata: dict, hasher: Any, reader: "_Reader") -> int:
+ """safetensors of ``cand.names`` read live one tensor at a time (``save_file``'s order, so mmap stays aligned)."""
+ import torch
+
+ order = sorted(
+ cand.names,
+ key = lambda e: (-torch.empty((), dtype = e[1]).element_size(), e[0]),
+ )
+ # transformers 4.x refuses a safetensors file whose metadata does not name its framework.
+ header: dict = {
+ "__metadata__": {"format": "pt", **{str(k): str(v) for k, v in metadata.items()}}
+ }
+ offset = 0
+ for name, dtype, shape in order:
+ tag = _cache._DTYPE_TAGS.get(str(dtype).replace("torch.", ""))
+ if tag is None:
+ raise ValueError(f"{name}: dtype {dtype} has no safetensors tag")
+ count = 1
+ for d in shape:
+ count *= int(d)
+ size = count * torch.empty((), dtype = dtype).element_size()
+ header[name] = {"dtype": tag, "shape": list(shape), "data_offsets": [offset, offset + size]}
+ offset += size
+ blob = json.dumps(header, separators = (",", ":")).encode("utf-8")
+ blob += b" " * ((8 - len(blob) % 8) % 8)
+ written = 0
+ dropped = 0
+ with open(path, "wb") as fh:
+ for chunk in (struct.pack("<Q", len(blob)), blob):
+ fh.write(chunk)
+ hasher.update(chunk)
+ written += len(chunk)
+ for name, dtype, shape in order:
+ reader.check()
+ tensor = reader.tensor(name)
+ if tensor.dtype != dtype or tuple(tensor.shape) != tuple(shape):
+ raise ValueError(f"{name} changed after the load")
+ if tensor.numel() == 0:
+ continue
+ data = memoryview(tensor.detach().contiguous().reshape(-1).view(torch.uint8).numpy())
+ fh.write(data)
+ hasher.update(data)
+ written += len(data)
+ del data, tensor
+ if written - dropped >= _DROP_EVERY_BYTES:
+ fh.flush()
+ _drop_pages(fh.fileno(), written)
+ dropped = written
+ fh.flush()
+ os.fsync(fh.fileno())
+ _drop_pages(fh.fileno(), written)
+ return 8 + len(blob) + offset
+
+
+def save(
+ cand: CastCandidate,
+ module: Any,
+ *,
+ logger: Any = None,
+ check: Optional[Any] = None,
+) -> Optional[str]:
+ """Persist ``module`` (the cast ``cand.component``); returns the artifact path or None (logged). Never raises."""
+ check = check or (lambda: None)
+ t0 = time.perf_counter()
+ root = _cache.cache_root()
+ try:
+ names = _state_names(module)
+ if names is None or [n for n, _d, _s in names] != [n for n, _d, _s in cand.names]:
+ _log(
+ logger, "info", f"{cand.component}: not cached (its tensors changed after the load)"
+ )
+ return None
+ if any(d != dt or s != st for (_n, d, s), (_m, dt, st) in zip(names, cand.names)):
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: not cached (a tensor changed dtype or shape after the load)",
+ )
+ return None
+ reader = _Reader(module, check)
+ why = _matches_source(cand, reader)
+ if why is not None:
+ _log(logger, "info", f"{cand.component}: not cached ({why})")
+ return None
+ import torch
+
+ nbytes = 0
+ for _n, dtype, shape in names:
+ count = 1
+ for d in shape:
+ count *= int(d)
+ nbytes += count * torch.empty((), dtype = dtype).element_size()
+ budget = _cache.max_cache_bytes()
+ if budget is not None and nbytes > budget:
+ _log(logger, "info", f"{cand.component}: not cached (larger than the cache budget)")
+ return None
+ with _cache._writer_lock(root, cand.key) as locked:
+ if not locked:
+ return None
+ existing = _cache._entry(cand.key)
+ if existing is not None:
+ with _cache._lock:
+ rejected = os.path.realpath(str(existing)) in _cache._REJECTED
+ if not rejected:
+ return None
+ _cache._reclaim(root / cand.key)
+ root.mkdir(parents = True, exist_ok = True)
+ need = nbytes + max(_cache._DISK_MARGIN_BYTES, nbytes // 20)
+ free = shutil.disk_usage(str(root)).free
+ if free < need:
+ _cache.evict(root = root, extra_free_bytes = need - free, keep = cand.key, logger = logger)
+ free = shutil.disk_usage(str(root)).free
+ if free < need:
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: not cached (disk short: {free / 2**30:.1f} GiB free, "
+ f"need {need / 2**30:.1f} GiB)",
+ )
+ return None
+ source_bytes = sum(os.path.getsize(os.path.realpath(s)) for s in cand.shards)
+ meta = {
+ "key": cand.key,
+ "component": cand.component,
+ "class": cand.class_name,
+ "dtype": cand.parts["dtype"],
+ "source_gib": round(source_bytes / 2**30, 2),
+ "torch_version": str(torch.__version__),
+ }
+ kdir = root / cand.key
+ kdir.mkdir(parents = True, exist_ok = True)
+ shutil.copyfile(cand.config_path, kdir / "config.json")
+ fd, tmp = tempfile.mkstemp(
+ dir = str(kdir), prefix = f".{_cache.FILE_NAME}.", suffix = _cache._TEMP_SUFFIX
+ )
+ os.close(fd)
+ hasher = hashlib.sha256()
+ try:
+ size = _write(
+ tmp,
+ cand,
+ {CAST_META_KEY: json.dumps(meta, sort_keys = True)},
+ hasher,
+ reader,
+ )
+ os.replace(tmp, kdir / _cache.FILE_NAME)
+ tmp = None
+ finally:
+ if tmp is not None:
+ try:
+ os.unlink(tmp)
+ except OSError:
+ pass
+ sha = hasher.hexdigest()
+ _cache._record_verified(kdir, sha)
+ manifest = {
+ "key": cand.key,
+ "bytes": size,
+ "sha256": sha,
+ "created": time.time(),
+ "parts": cand.parts,
+ }
+ _cache._atomic_bytes(
+ kdir / _cache.MANIFEST_NAME,
+ json.dumps(manifest, indent = 1, default = str).encode("utf-8"),
+ )
+ _cache._touch(kdir)
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: saved the {cand.parts['dtype']} cast ({cand.key}, {size / 2**30:.2f} GiB from "
+ f"{source_bytes / 2**30:.2f} GiB of source shards, in {time.perf_counter() - t0:.1f} s)",
+ )
+ _cache.evict(logger = logger, keep = cand.key)
+ return str(kdir / _cache.FILE_NAME)
+ except Exception as exc: # noqa: BLE001 - a cache write never fails a load
+ _log(
+ logger,
+ "warning",
+ f"{cand.component}: save failed: {type(exc).__name__}: {str(exc)[:200]}",
+ )
+ return None
+
+
+class CastWrites:
+ """Background writes of one load's cast components; registered with the prequant cache's background set."""
+
+ def __init__(
+ self,
+ pipe: Any,
+ cands: list,
+ logger: Any = None,
+ ) -> None:
+ import weakref
+
+ self._pipe = weakref.ref(pipe)
+ self._cands = list(cands)
+ self.key = ",".join(c.key for c in self._cands)
+ self._logger = logger
+ self._cancel = threading.Event()
+ self._finished = threading.Event()
+ self._thread: Optional[threading.Thread] = None
+ self.results: dict = {}
+
+ def _check(self) -> None:
+ if self._cancel.is_set():
+ raise RuntimeError("cast cache write cancelled")
+
+ def start(self) -> bool:
+ with _cache._lock:
+ if self._thread is not None or self._cancel.is_set():
+ return False
+ self._thread = threading.Thread(target = self._run, name = "cast-cache-write", daemon = True)
+ _cache._BACKGROUND.add(self)
+ try:
+ self._thread.start()
+ except Exception: # noqa: BLE001
+ with _cache._lock:
+ self._thread = None
+ self._close()
+ return False
+ return True
+
+ def _run(self) -> None:
+ try:
+ for cand in self._cands:
+ self._check()
+ pipe = self._pipe()
+ module = getattr(pipe, cand.component, None) if pipe is not None else None
+ pipe = None
+ if module is None:
+ continue
+ self.results[cand.component] = save(
+ cand, module, logger = self._logger, check = self._check
+ )
+ module = None
+ except Exception as exc: # noqa: BLE001
+ _log(self._logger, "info", f"background write stopped ({str(exc)[:200]})")
+ finally:
+ self._close()
+
+ def _close(self) -> None:
+ self._cands = []
+ with _cache._lock:
+ _cache._BACKGROUND.discard(self)
+ self._finished.set()
+
+ def discard(self) -> None:
+ with _cache._lock:
+ started = self._thread is not None
+ self._cancel.set()
+ if not started:
+ self._close()
+
+ def cancel(self) -> None:
+ self.discard()
+
+ def done(self) -> bool:
+ return self._finished.is_set()
+
+ def wait(self, timeout: Optional[float] = None) -> bool:
+ return self._finished.wait(timeout)
+
+
+def defer_writes(
+ pipe: Any,
+ cands: list,
+ *,
+ logger: Any = None,
+) -> Optional[CastWrites]:
+ """Capture the missed candidates of a just-loaded ``pipe``; ``start()`` the result after publish. Never raises."""
+ try:
+ if not enabled() or not cands:
+ return None
+ captured = capture(pipe, cands, logger = logger)
+ return CastWrites(pipe, captured, logger) if captured else None
+ except Exception as exc: # noqa: BLE001
+ _log(logger, "warning", f"write skipped ({type(exc).__name__}: {exc})")
+ return None
+
+
+def finish(writes: Any, timeout: float = 60.0) -> None:
+ """Teardown: let a running write finish for ``timeout`` s, then cancel it. Never raises."""
+ if writes is None:
+ return
+ try:
+ if not writes.wait(timeout):
+ writes.cancel()
+ writes.wait(30.0)
+ except Exception: # noqa: BLE001
+ pass
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index a5d9d388385..3e5e5528a02 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3152,6 +3152,58 @@ def _save_runtime_quant_to_cache(
logger.warning("diffusion.prequant_cache: skipped (%s)", exc)
+def _read_cast_cache(
+ pipe_kwargs: dict,
+ *,
+ source: Any,
+ revision: Optional[str],
+ base: Optional[str],
+ cancel_check: Any = None,
+) -> list:
+ """Inject cached float16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""
+ try:
+ from .diffusion_cast_cache import candidates, read_all
+
+ cands = candidates(
+ source,
+ torch_dtype = pipe_kwargs.get("torch_dtype"),
+ cache_dir = None
+ if source and os.path.isdir(str(source))
+ else pipe_kwargs.get("cache_dir"),
+ revision = revision,
+ skip = tuple(pipe_kwargs),
+ base_repo = base,
+ logger = logger,
+ )
+ if not cands:
+ return []
+ hits = read_all(cands, logger = logger, cancel_check = cancel_check)
+ pipe_kwargs.update(hits)
+ if hits:
+ logger.info(
+ "video.cast_cache: read %s from the local float16 cast cache",
+ ", ".join(sorted(hits)),
+ )
+ return [c for c in cands if c.component not in hits]
+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss
+ if cancel_check is not None:
+ cancel_check()
+ logger.warning("video.cast_cache: skipped (%s)", exc)
+ return []
+
+
+def _finish_cast_writes(pipe: Any) -> None:
+ writes = getattr(pipe, "_unsloth_cast_writes", None)
+ if writes is None:
+ return
+ try:
+ from .diffusion_cast_cache import finish
+ finish(writes)
+ pipe._unsloth_cast_writes = None
+ except Exception: # noqa: BLE001 - teardown is best effort
+ pass
+
+
def _video_native_cache_spec(
fam: Any,
*,
@@ -9144,13 +9196,26 @@ class VideoBackend:
# The encoder runs in the worker; the enhancer, processor and duration head are never used.
pipe_kwargs.update({name: None for name in LTX25_NONE_COMPONENTS})
+ cast_writes = None
if kind == "pipeline":
+ cast_missed = _read_cast_cache(
+ pipe_kwargs,
+ source = _base_local_dir or fetch_base,
+ revision = None if _base_local_dir else _base_fetch_revision,
+ base = base,
+ cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
+ )
# The pre-downloaded snapshot dir keeps from_pretrained off the hub; hub id when pre-download was skipped.
pipe = pipeline_cls.from_pretrained(
_base_local_dir or fetch_base,
**_pinned_base_kwargs(),
**pipe_kwargs,
)
+ if cast_missed:
+ from .diffusion_cast_cache import defer_writes
+
+ # Names only: the write reads the live tensors once the load is published.
+ cast_writes = defer_writes(pipe, cast_missed, logger = logger)
else:
transformer_cls = getattr(diffusers, fam.transformer_class)
# checkpoint_path was resolved (and downloaded) by the memory-planning branch above.
@@ -10390,6 +10455,9 @@ class VideoBackend:
transformer_quant_engaged or "off",
)
start_background_pins(pipe)
+ if cast_writes is not None:
+ pipe._unsloth_cast_writes = cast_writes
+ cast_writes.start()
return self.status()
@staticmethod
@@ -13939,6 +14007,8 @@ class VideoBackend:
close_encoder(getattr(state, "pipe", None))
# A pinner still copying would hold its chunks past release_pinned_host_memory().
stop_background_pins(getattr(state, "pipe", None))
+ # A cast cache write still reads the components the teardown frees.
+ _finish_cast_writes(getattr(state, "pipe", None))
uninstall_cfg_parallel(getattr(state, "pipe", None))
# Before clear_gpu_cache(), or the graph pool stays reserved.
diffusion_cuda_graph.uninstall_all(
diff --git a/studio/backend/tests/test_diffusion_cast_cache.py b/studio/backend/tests/test_diffusion_cast_cache.py
new file mode 100644
index 00000000000..6bed8fd7ba2
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_cast_cache.py
@@ -0,0 +1,294 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Local float16 cast cache of fp32-shard components: candidates, bit-identical round trip, refusals, video wiring."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+transformers = pytest.importorskip("transformers")
+pytest.importorskip("safetensors")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion_cast_cache as CC # noqa: E402
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+
+FP16_DEFAULT = {"vae": torch.float32, "default": torch.float16}
+
+
+def _tiny_wan():
+ torch.manual_seed(0)
+ return diffusers.WanTransformer3DModel(
+ patch_size = (1, 2, 2),
+ num_attention_heads = 2,
+ attention_head_dim = 16,
+ in_channels = 4,
+ out_channels = 4,
+ text_dim = 32,
+ freq_dim = 16,
+ ffn_dim = 64,
+ num_layers = 2,
+ cross_attn_norm = True,
+ qk_norm = "rms_norm_across_heads",
+ eps = 1e-6,
+ rope_max_seq_len = 32,
+ ).to(torch.float32)
+
+
+def _tiny_umt5():
+ torch.manual_seed(1)
+ config = transformers.UMT5Config(
+ vocab_size = 64,
+ d_model = 32,
+ d_kv = 8,
+ d_ff = 64,
+ num_layers = 2,
+ num_heads = 4,
+ relative_attention_num_buckets = 8,
+ )
+ return transformers.UMT5EncoderModel(config).to(torch.float32)
+
+
[email protected](autouse = True)
+def cache_root(tmp_path, monkeypatch):
+ root = tmp_path / "cache"
+ monkeypatch.setenv(C.ENV_DIR, str(root))
+ monkeypatch.delenv(CC.CAST_CACHE_ENV, raising = False)
+ monkeypatch.delenv(C.ENV_MODE, raising = False)
+ # No Hub probe from a local directory source.
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ with C._lock:
+ C._REJECTED.clear()
+ return root
+
+
+def pipe_dir(tmp_path):
+ root = tmp_path / "Wan2.1-T2V-tiny-Diffusers"
+ _tiny_wan().save_pretrained(root / "transformer", max_shard_size = "60KB")
+ _tiny_umt5().save_pretrained(root / "text_encoder")
+ (root / "vae").mkdir()
+ (root / "model_index.json").write_text(
+ json.dumps(
+ {
+ "_class_name": "WanPipeline",
+ "transformer": ["diffusers", "WanTransformer3DModel"],
+ "text_encoder": ["transformers", "UMT5EncoderModel"],
+ "vae": ["diffusers", "AutoencoderKLWan"],
+ "tokenizer": ["transformers", "T5TokenizerFast"],
+ "scheduler": ["diffusers", "UniPCMultistepScheduler"],
+ }
+ )
+ )
+ return root
+
+
+def _dense(cand):
+ kwargs = CC._load_kwargs(cand)
+ kwargs.pop("output_loading_info")
+ return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)
+
+
+class _Pipe:
+ pass
+
+
+def _save(cand, module):
+ pipe = _Pipe()
+ setattr(pipe, cand.component, module)
+ writes = CC.defer_writes(pipe, [cand])
+ assert writes is not None and writes.start()
+ assert writes.wait(60)
+ return writes.results.get(cand.component)
+
+
+def _bits(t):
+ return t.reshape(-1).view(torch.uint8) if t.numel() else t
+
+
+def _identical(a, b):
+ sa, sb = a.state_dict(), b.state_dict()
+ assert sa.keys() == sb.keys()
+ for k in sa:
+ assert sa[k].dtype == sb[k].dtype, k
+ assert torch.equal(_bits(sa[k]), _bits(sb[k])), k
+ ba, bb = dict(a.named_buffers()), dict(b.named_buffers())
+ assert ba.keys() == bb.keys()
+ for k in ba:
+ assert ba[k].dtype == bb[k].dtype and torch.equal(ba[k], bb[k]), k
+
+
+def _by_name(cands):
+ return {c.component: c for c in cands}
+
+
+def test_candidates_are_fp32_shards_loaded_at_fp16(pipe_dir):
+ cands = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))
+ assert set(cands) == {
+ "transformer",
+ "text_encoder",
+ } # the vae stays fp32; the rest hold no weights
+ assert len(cands["transformer"].shards) > 1
+ assert cands["text_encoder"].parts["keep_in_fp32"] == ["wo"]
+ assert cands["transformer"].key != cands["text_encoder"].key
+
+
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_no_candidates_outside_float16(pipe_dir, dtype):
+ assert CC.candidates(pipe_dir, torch_dtype = dtype) == []
+
+
+def test_kill_switches(pipe_dir, monkeypatch):
+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "0")
+ assert CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT) == []
+ monkeypatch.delenv(CC.CAST_CACHE_ENV)
+ monkeypatch.setenv(C.ENV_MODE, "0")
+ assert CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT) == []
+
+
+def test_skip_and_narrow_shards(pipe_dir):
+ assert set(
+ _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT, skip = ("text_encoder",)))
+ ) == {"transformer"}
+ _tiny_umt5().to(torch.float16).save_pretrained(pipe_dir / "text_encoder")
+ assert set(_by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))) == {"transformer"}
+
+
[email protected]("component", ["transformer", "text_encoder"])
+def test_round_trip_is_bit_identical(pipe_dir, component):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))[component]
+ assert CC.read(cand) is None # cold: a miss
+ dense = _dense(cand)
+ path = _save(cand, dense)
+ assert path is not None and C.path_in_cache(path)
+ stored = CC._shard_header(path)
+ assert {e["dtype"] for e in stored.values()} == {"F16", "F32"} # fp32 keep modules stay fp32
+ fresh = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))[component]
+ hit = CC.read(fresh)
+ assert hit is not None
+ _identical(dense, hit)
+ _identical(_dense(fresh), hit)
+ if component == "text_encoder":
+ # the tied embedding is written once and re-tied by from_pretrained
+ assert "encoder.embed_tokens.weight" not in stored
+ assert hit.shared.weight.data_ptr() == hit.encoder.embed_tokens.weight.data_ptr()
+
+
+def test_module_changed_in_place_is_not_cached(pipe_dir):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ dense = _dense(cand)
+ pipe = _Pipe()
+ pipe.transformer = dense
+ writes = CC.defer_writes(pipe, [cand])
+ with torch.no_grad():
+ for p in dense.parameters():
+ p.mul_(2)
+ writes.start()
+ writes.wait(60)
+ assert writes.results["transformer"] is None
+ assert C._entry(cand.key) is None
+
+
+def test_module_restructured_after_load_is_not_cached(pipe_dir):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ dense = _dense(cand)
+ pipe = _Pipe()
+ pipe.transformer = dense
+ writes = CC.defer_writes(pipe, [cand])
+ dense.proj_out = torch.nn.Linear(4, 4)
+ writes.start()
+ writes.wait(60)
+ assert writes.results["transformer"] is None
+
+
+def test_new_source_bytes_change_the_key(pipe_dir):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
+ assert _save(cand, _dense(cand)) is not None
+ model = _tiny_umt5()
+ with torch.no_grad():
+ model.shared.weight.add_(1)
+ model.save_pretrained(pipe_dir / "text_encoder")
+ fresh = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
+ assert fresh.key != cand.key
+ assert CC.read(fresh) is None
+
+
+def test_corrupt_entry_is_a_miss(pipe_dir):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ path = Path(_save(cand, _dense(cand)))
+ raw = bytearray(path.read_bytes())
+ raw[-1] ^= 0xFF
+ path.write_bytes(bytes(raw))
+ (path.parent / C.VERIFIED_NAME).unlink()
+ assert CC.read(cand) is None
+ with C._lock:
+ assert str(path.resolve()) in C._REJECTED
+
+
+def test_video_reader_injects_hits_and_returns_misses(pipe_dir):
+ from core.inference import video as V
+
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
+ _save(cand, _dense(cand))
+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
+ missed = V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None)
+ assert [c.component for c in missed] == ["transformer"]
+ assert isinstance(pipe_kwargs["text_encoder"], transformers.UMT5EncoderModel)
+ assert "transformer" not in pipe_kwargs
+ # an injected component is never a candidate
+ again = V._read_cast_cache(dict(pipe_kwargs), source = str(pipe_dir), revision = None, base = None)
+ assert [c.component for c in again] == ["transformer"]
+
+
+def test_teardown_waits_then_clears(pipe_dir):
+ from core.inference import video as V
+
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ pipe = _Pipe()
+ pipe.transformer = _dense(cand)
+ writes = CC.defer_writes(pipe, [cand])
+ pipe._unsloth_cast_writes = writes
+ writes.start()
+ V._finish_cast_writes(pipe)
+ assert pipe._unsloth_cast_writes is None
+ assert writes.done()
+ assert C._entry(cand.key) is not None
+
+
+def test_sample_keys_tolerate_one_kind_per_shard():
+ names = {"a": torch.float16, "b": torch.float16, "w": torch.float32}
+ entry = lambda n: {"dtype": "F32", "data_offsets": [0, n]} # noqa: E731
+ assert CC._sample_keys({"a": entry(8), "b": entry(16)}, names) == ["b"]
+ assert CC._sample_keys({"w": entry(8)}, names) == ["w"]
+ assert CC._sample_keys({"a": entry(CC._SAMPLE_MAX_BYTES + 1)}, names) == []
+
+
+def test_save_when_kept_fp32_tensors_exceed_the_sample_size(pipe_dir, monkeypatch):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
+ dense = _dense(cand)
+ wide = min(t.numel() * 4 for n, t in dense.state_dict().items() if t.dtype == torch.float32)
+ monkeypatch.setattr(CC, "_SAMPLE_MAX_BYTES", wide - 1)
+ assert _save(cand, dense) is not None
+ _identical(dense, CC.read(cand))
+
+
[email protected](not torch.cuda.is_available(), reason = "needs a CUDA device")
+def test_resident_device_module_is_read_back_bit_identical(pipe_dir):
+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
+ dense = _dense(cand)
+ reference = {k: v.clone() for k, v in dense.state_dict().items()}
+ dense.to("cuda")
+ assert _save(cand, dense) is not None
+ hit = CC.read(cand)
+ for k, v in hit.state_dict().items():
+ assert v.dtype == reference[k].dtype and torch.equal(_bits(v), _bits(reference[k])), k