Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cache fp32-shard video components cast to float16 for later loads (#15303)
Daniel Hand4be24722242
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.pynew file mode 100644index 00000000000..15046aebdf5--- /dev/null+++ b/studio/backend/core/inference/diffusion_cast_cache.py@@ -0,0 +1,814 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Local float16 cache of components loaded from fp32 shards (e.g. Wan), written from the live tensors after publish.++Later loads read the half-size file at the same dtype (a no-op cast, bit-identical); a sampled source comparison+refuses a module changed in place. Shares ``diffusion_prequant_cache``'s root, locks, manifest and LRU.+Kill switch: ``UNSLOTH_DIFFUSION_CAST_CACHE=0`` (``UNSLOTH_DIFFUSION_PREQUANT_CACHE=0`` disables it too).+"""++from __future__ import annotations++import hashlib+import json+import os+import shutil+import struct+import tempfile+import threading+import time+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Iterable, Optional++from . import diffusion_prequant_cache as _cache++CAST_CACHE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE"+CAST_LAYOUT_VERSION = 1+CAST_META_KEY = "unsloth_cast_component"+_LIBRARIES = ("diffusers", "transformers")+_STEMS = {"diffusers": "diffusion_pytorch_model", "transformers": "model"}+_WIDE_TAGS = ("F32", "F64")+_FLOAT_TAGS = ("F64", "F32", "F16", "BF16", "F8_E4M3", "F8_E5M2", "F8_E8M0")+# Per-shard source sample cap; a bigger read would grow the page cache.+_SAMPLE_MAX_BYTES = 64 << 20+_DROP_EVERY_BYTES = 256 << 20+# Per-component wait for group-offloaded tensors to return to host before reading the device copy.+_ONLOAD_WAIT_SECONDS = 10.0+++def enabled() -> bool:+ raw = (os.environ.get(CAST_CACHE_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no", "none") and _cache.enabled()+++def _is_fp16(dtype: Any) -> bool:+ return str(dtype) in ("torch.float16", "float16")+++def _component_dtype(torch_dtype: Any, component: str) -> Any:+ if isinstance(torch_dtype, dict):+ return torch_dtype.get(component, torch_dtype.get("default"))+ return torch_dtype+++def _minor(module_name: str) -> Optional[str]:+ try:+ import importlib.metadata as md+ return _cache._minor(md.version(module_name))+ except Exception: # noqa: BLE001+ return None+++@dataclass+class CastCandidate:+ component: str+ library: str+ class_name: str+ cls: Any+ dtype: Any+ source: str+ shards: list+ config_path: str+ key: str+ parts: dict+ names: list = field(default_factory = list)+++def _read_json(path: Optional[str]) -> Optional[dict]:+ if not path:+ return None+ try:+ with open(path, "r", encoding = "utf-8") as fh:+ data = json.load(fh)+ return data if isinstance(data, dict) else None+ except Exception: # noqa: BLE001+ return None+++def _shards_of(+ source: str, component: str, stem: str, cache_dir: Optional[str], revision: Optional[str]+) -> Optional[list]:+ """The shard files ``from_pretrained(source/component)`` reads, or None when unsure (both layouts, or any missing)."""+ index = _cache._local_file(+ source, f"{component}/{stem}.safetensors.index.json", cache_dir, revision+ )+ single = _cache._local_file(source, f"{component}/{stem}.safetensors", cache_dir, revision)+ if index is not None and single is not None:+ return None+ if single is not None:+ return [str(single)]+ weight_map = (_read_json(index) or {}).get("weight_map") or {}+ names = sorted(set(str(v) for v in weight_map.values()))+ if not names:+ return None+ paths = [_cache._local_file(source, f"{component}/{n}", cache_dir, revision) for n in names]+ return [str(p) for p in paths] if all(paths) else None+++def _shard_header(path: str) -> Optional[dict]:+ raw = _cache.read_header_bytes(path)+ if raw is None:+ return None+ try:+ header = json.loads(raw[8:].decode("utf-8"))+ except Exception: # noqa: BLE001+ return None+ header.pop("__metadata__", None)+ return header+++def _all_wide(shards: list) -> bool:+ """Every floating tensor of every shard is wider than float16 (the only case the cast halves)."""+ seen = False+ for shard in shards:+ header = _shard_header(shard)+ if header is None:+ return False+ for entry in header.values():+ tag = entry.get("dtype") if isinstance(entry, dict) else None+ if tag in _FLOAT_TAGS:+ if tag not in _WIDE_TAGS:+ return False+ seen = True+ return seen+++def _load_class(library: str, class_name: str) -> Any:+ try:+ if library == "diffusers":+ import diffusers as module+ else:+ import transformers as module+ return getattr(module, class_name, None)+ except Exception: # noqa: BLE001+ return None+++def candidates(+ source: Any,+ *,+ torch_dtype: Any,+ cache_dir: Optional[str] = None,+ revision: Optional[str] = None,+ skip: Iterable[str] = (),+ base_repo: Optional[str] = None,+ logger: Any = None,+) -> list:+ """Components at ``source`` that load fp32 shards at float16, with cache keys. Local reads only; never raises."""+ if not enabled() or not source:+ return []+ out: list = []+ try:+ import torch++ source = str(source)+ index = _read_json(_cache._local_file(source, "model_index.json", cache_dir, revision))+ if index is None:+ return []+ skip = set(skip)+ for component, spec in sorted(index.items()):+ if component.startswith("_") or component in skip:+ continue+ if not isinstance(spec, (list, tuple)) or len(spec) != 2:+ continue+ library, class_name = spec+ if library not in _LIBRARIES or not isinstance(class_name, str):+ continue+ dtype = _component_dtype(torch_dtype, component)+ if not _is_fp16(dtype):+ continue+ cls = _load_class(library, class_name)+ if cls is None or not isinstance(cls, type) or not issubclass(cls, torch.nn.Module):+ continue+ shards = _shards_of(source, component, _STEMS[library], cache_dir, revision)+ if not shards or not _all_wide(shards):+ continue+ config_path = _cache._local_file(+ source, f"{component}/config.json", cache_dir, revision+ )+ if config_path is None:+ continue+ fp = _cache.source_fingerprint(shards)+ if fp is None:+ continue+ parts = {+ "kind": "cast",+ "layout": CAST_LAYOUT_VERSION,+ "policy": _cache.POLICY_VERSION,+ "component": component,+ "library": library,+ "class": class_name,+ "dtype": str(dtype).replace("torch.", ""),+ "keep_in_fp32": sorted(+ str(m) for m in (getattr(cls, "_keep_in_fp32_modules", None) or ())+ ),+ "source": fp,+ "config": hashlib.sha256(Path(config_path).read_bytes()).hexdigest(),+ "base": _cache.base_identity(base_repo or source),+ "torch": _cache._minor(torch.__version__),+ library: _minor(library),+ }+ out.append(+ CastCandidate(+ component = component,+ library = library,+ class_name = class_name,+ cls = cls,+ dtype = dtype,+ source = source,+ shards = shards,+ config_path = str(config_path),+ key = _cache.cache_key(parts),+ parts = parts,+ )+ )+ except Exception as exc: # noqa: BLE001+ _log(logger, "warning", f"candidates skipped ({type(exc).__name__}: {str(exc)[:200]})")+ return []+ return out+++def _log(logger: Any, level: str, msg: str) -> None:+ if logger is not None:+ try:+ getattr(logger, level)("diffusion.cast_cache: %s", msg)+ except Exception: # noqa: BLE001+ pass+++def _header_metadata(path: Any) -> Optional[dict]:+ raw = _cache.read_header_bytes(path)+ if raw is None:+ return None+ try:+ meta = json.loads(raw[8:].decode("utf-8")).get("__metadata__") or {}+ return json.loads(meta[CAST_META_KEY])+ except Exception: # noqa: BLE001+ return None+++def _weights_link(kdir: Path, library: str) -> None:+ """``from_pretrained`` reads ``<stem>.safetensors``; the entry's artifact is ``model.safetensors``."""+ name = f"{_STEMS[library]}.safetensors"+ if name == _cache.FILE_NAME:+ return+ link = kdir / name+ if link.exists():+ return+ try:+ os.symlink(_cache.FILE_NAME, link)+ except OSError:+ os.link(kdir / _cache.FILE_NAME, link) # no symlink rights (Windows): a hard link+++def _load_kwargs(cand: CastCandidate) -> dict:+ kwargs: dict = {"low_cpu_mem_usage": True, "use_safetensors": True, "output_loading_info": True}+ use_dtype_kw = False+ if cand.library == "transformers":+ try:+ import transformers+ from packaging import version+ use_dtype_kw = version.parse(transformers.__version__) >= version.parse("4.56.0")+ except Exception: # noqa: BLE001+ use_dtype_kw = False+ kwargs["dtype" if use_dtype_kw else "torch_dtype"] = cand.dtype+ return kwargs+++def read(+ cand: CastCandidate,+ *,+ logger: Any = None,+ cancel_check: Optional[Any] = None,+) -> Any:+ """The cached cast ``cand.component``, or None (a miss: the dense load runs). Never raises."""+ if not enabled():+ return None+ try:+ root = _cache.cache_root()+ if not (root / cand.key / _cache.MANIFEST_NAME).is_file():+ return None+ if _cache._hub_source_stale(cand.shards, logger):+ return None+ with _cache._cache_lock():+ artifact = _cache._entry(cand.key)+ if artifact is None:+ return None+ with _cache._lock:+ if os.path.realpath(str(artifact)) in _cache._REJECTED:+ return None+ _cache._touch(artifact.parent)+ meta = _header_metadata(artifact)+ if not isinstance(meta, dict) or meta.get("key") != cand.key:+ _cache.note_rejected(artifact, logger)+ return None+ if not _cache.verify_entry(artifact, logger):+ _cache.note_rejected(artifact, logger)+ return None+ kdir = artifact.parent+ if not (kdir / "config.json").is_file():+ _cache.note_rejected(artifact, logger)+ return None+ _weights_link(kdir, cand.library)+ t0 = time.perf_counter()+ model, info = cand.cls.from_pretrained(str(kdir), **_load_kwargs(cand))+ bad = {+ k: list(v)+ for k, v in (info or {}).items()+ if k in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs") and v+ }+ if bad:+ _log(+ logger,+ "warning",+ f"{cand.component}: entry did not load cleanly ({str(bad)[:200]}); not used",+ )+ _cache.note_rejected(artifact, logger)+ return None+ if any(t.device.type == "meta" for t in (*model.parameters(), *model.buffers())):+ _cache.note_rejected(artifact, logger)+ return None+ model.eval()+ _log(+ logger,+ "info",+ f"{cand.component}: read the cached {meta.get('dtype')} cast ({artifact.stat().st_size / 2**30:.2f} GiB) "+ f"in {time.perf_counter() - t0:.1f} s instead of the {meta.get('source_gib', '?')} GiB source shards",+ )+ _cache.note_hit(artifact, logger)+ return model+ except Exception as exc: # noqa: BLE001 - an unusable entry is a miss+ _log(+ logger,+ "warning",+ f"{cand.component}: read failed: {type(exc).__name__}: {str(exc)[:200]}",+ )+ return None+++def read_all(+ cands: list,+ *,+ logger: Any = None,+ cancel_check: Optional[Any] = None,+) -> dict:+ """``{component: module}`` of every cached candidate. Never raises."""+ out: dict = {}+ for cand in cands or ():+ if cancel_check is not None:+ cancel_check()+ module = read(cand, logger = logger, cancel_check = cancel_check)+ if module is not None:+ out[cand.component] = module+ return out+++def _tied_keys(module: Any) -> set:+ keys: set = set()+ for attr in ("_tied_weights_keys", "all_tied_weights_keys"):+ value = getattr(module, attr, None)+ if isinstance(value, dict):+ keys.update(str(k) for k in value)+ elif isinstance(value, (list, tuple, set)):+ keys.update(str(k) for k in value)+ return keys+++def _resolve(module: Any, name: str) -> Any:+ parent_name, _, leaf = name.rpartition(".")+ parent = module.get_submodule(parent_name) if parent_name else module+ tensor = (+ parent._parameters.get(leaf) if leaf in parent._parameters else parent._buffers.get(leaf)+ )+ if tensor is None:+ raise KeyError(name)+ return tensor.data if hasattr(tensor, "data") else tensor+++def _state_names(module: Any) -> Optional[list]:+ """``[(name, dtype, shape)]`` to write: the state dict minus tied aliases ``from_pretrained`` restores."""+ import torch++ tied = _tied_keys(module)+ seen: dict = {}+ out = []+ for name, tensor in module.state_dict(keep_vars = True).items():+ if type(tensor) not in (torch.Tensor, torch.nn.Parameter) or tensor.device.type == "meta":+ return None+ ident = (tensor.data_ptr(), tensor.dtype, tuple(tensor.shape), tensor.device)+ if tensor.numel() and ident in seen and name in tied:+ continue+ seen.setdefault(ident, name)+ out.append((name, tensor.dtype, tuple(tensor.shape)))+ return out+++def capture(+ pipe: Any,+ cands: list,+ *,+ logger: Any = None,+) -> list:+ """Right after ``from_pretrained``: record what each missed candidate's module holds (names only, no tensors)."""+ out = []+ for cand in cands or ():+ try:+ module = getattr(pipe, cand.component, None)+ if module is None or type(module).__name__ != cand.class_name:+ continue+ names = _state_names(module)+ if not names:+ continue+ cand.names = names+ out.append(cand)+ except Exception as exc: # noqa: BLE001+ _log(logger, "info", f"{cand.component}: not cached ({type(exc).__name__}: {exc})")+ return out+++def _hooked(module: Any) -> bool:+ return any(+ getattr(m, "_diffusers_hook", None) is not None or getattr(m, "_hf_hook", None) is not None+ for m in module.modules()+ )+++class _Reader:+ """Host copies of one component's live tensors; device copies are read on a side stream after one device+ synchronise (an upload may be queued on another stream), never behind the render on the default stream."""++ def __init__(self, module: Any, check: Any) -> None:+ self.module = module+ self.check = check+ self.wait = _ONLOAD_WAIT_SECONDS if _hooked(module) else 0.0+ self.synced = False+ self.stream = None++ def tensor(self, name: str) -> Any:+ while True:+ tensor = _resolve(self.module, name)+ if tensor.device.type == "cpu":+ return tensor+ if self.wait <= 0:+ break+ self.check()+ time.sleep(0.25)+ self.wait -= 0.25+ import torch++ if tensor.device.type != "cuda":+ return tensor.to("cpu")+ if not self.synced:+ torch.cuda.synchronize(tensor.device)+ self.synced = True+ self.stream = torch.cuda.Stream(device = tensor.device)+ with torch.cuda.stream(self.stream):+ host = tensor.to("cpu", non_blocking = False)+ self.stream.synchronize()+ return host+++def _sample_keys(header: dict, names: dict) -> list:+ """The largest cast tensor up to ``_SAMPLE_MAX_BYTES`` and the smallest tensor kept wide, of one shard."""+ import torch++ cast, wide = [], []+ for key, entry in header.items():+ if key not in names or entry.get("dtype") not in _WIDE_TAGS:+ continue+ begin, end = (int(v) for v in entry["data_offsets"])+ if end - begin > _SAMPLE_MAX_BYTES:+ continue+ (wide if names[key] in (torch.float32, torch.float64) else cast).append((end - begin, key))+ return ([max(cast)[1]] if cast else []) + ([min(wide)[1]] if wide else [])+++def _matches_source(cand: CastCandidate, reader: _Reader) -> Optional[str]:+ """None when sampled live tensors equal the source cast bit for bit, else why not (a module changed in place)."""+ import torch+ from safetensors import safe_open++ names = {n: d for n, d, _s in cand.names}+ sampled = 0+ for shard in cand.shards:+ picks = _sample_keys(_shard_header(shard) or {}, names)+ if not picks:+ continue+ with safe_open(shard, framework = "pt") as fh:+ for key in picks:+ reader.check()+ live = reader.tensor(key)+ src = fh.get_tensor(key)+ if tuple(src.shape) != tuple(live.shape):+ return f"{key} changed shape"+ if not torch.equal(+ src.to(live.dtype).view(-1).view(torch.uint8),+ live.reshape(-1).view(torch.uint8),+ ):+ return f"{key} differs from its source cast"+ del src, live+ sampled += 1+ return None if sampled else "no tensor to compare against the source"+++def _drop_pages(fd: int, upto: int) -> None:+ try:+ os.fdatasync(fd)+ os.posix_fadvise(fd, 0, upto, os.POSIX_FADV_DONTNEED)+ except (AttributeError, OSError):+ pass+++def _write(path: str, cand: CastCandidate, metadata: dict, hasher: Any, reader: "_Reader") -> int:+ """safetensors of ``cand.names`` read live one tensor at a time (``save_file``'s order, so mmap stays aligned)."""+ import torch++ order = sorted(+ cand.names,+ key = lambda e: (-torch.empty((), dtype = e[1]).element_size(), e[0]),+ )+ # transformers 4.x refuses a safetensors file whose metadata does not name its framework.+ header: dict = {+ "__metadata__": {"format": "pt", **{str(k): str(v) for k, v in metadata.items()}}+ }+ offset = 0+ for name, dtype, shape in order:+ tag = _cache._DTYPE_TAGS.get(str(dtype).replace("torch.", ""))+ if tag is None:+ raise ValueError(f"{name}: dtype {dtype} has no safetensors tag")+ count = 1+ for d in shape:+ count *= int(d)+ size = count * torch.empty((), dtype = dtype).element_size()+ header[name] = {"dtype": tag, "shape": list(shape), "data_offsets": [offset, offset + size]}+ offset += size+ blob = json.dumps(header, separators = (",", ":")).encode("utf-8")+ blob += b" " * ((8 - len(blob) % 8) % 8)+ written = 0+ dropped = 0+ with open(path, "wb") as fh:+ for chunk in (struct.pack("<Q", len(blob)), blob):+ fh.write(chunk)+ hasher.update(chunk)+ written += len(chunk)+ for name, dtype, shape in order:+ reader.check()+ tensor = reader.tensor(name)+ if tensor.dtype != dtype or tuple(tensor.shape) != tuple(shape):+ raise ValueError(f"{name} changed after the load")+ if tensor.numel() == 0:+ continue+ data = memoryview(tensor.detach().contiguous().reshape(-1).view(torch.uint8).numpy())+ fh.write(data)+ hasher.update(data)+ written += len(data)+ del data, tensor+ if written - dropped >= _DROP_EVERY_BYTES:+ fh.flush()+ _drop_pages(fh.fileno(), written)+ dropped = written+ fh.flush()+ os.fsync(fh.fileno())+ _drop_pages(fh.fileno(), written)+ return 8 + len(blob) + offset+++def save(+ cand: CastCandidate,+ module: Any,+ *,+ logger: Any = None,+ check: Optional[Any] = None,+) -> Optional[str]:+ """Persist ``module`` (the cast ``cand.component``); returns the artifact path or None (logged). Never raises."""+ check = check or (lambda: None)+ t0 = time.perf_counter()+ root = _cache.cache_root()+ try:+ names = _state_names(module)+ if names is None or [n for n, _d, _s in names] != [n for n, _d, _s in cand.names]:+ _log(+ logger, "info", f"{cand.component}: not cached (its tensors changed after the load)"+ )+ return None+ if any(d != dt or s != st for (_n, d, s), (_m, dt, st) in zip(names, cand.names)):+ _log(+ logger,+ "info",+ f"{cand.component}: not cached (a tensor changed dtype or shape after the load)",+ )+ return None+ reader = _Reader(module, check)+ why = _matches_source(cand, reader)+ if why is not None:+ _log(logger, "info", f"{cand.component}: not cached ({why})")+ return None+ import torch++ nbytes = 0+ for _n, dtype, shape in names:+ count = 1+ for d in shape:+ count *= int(d)+ nbytes += count * torch.empty((), dtype = dtype).element_size()+ budget = _cache.max_cache_bytes()+ if budget is not None and nbytes > budget:+ _log(logger, "info", f"{cand.component}: not cached (larger than the cache budget)")+ return None+ with _cache._writer_lock(root, cand.key) as locked:+ if not locked:+ return None+ existing = _cache._entry(cand.key)+ if existing is not None:+ with _cache._lock:+ rejected = os.path.realpath(str(existing)) in _cache._REJECTED+ if not rejected:+ return None+ _cache._reclaim(root / cand.key)+ root.mkdir(parents = True, exist_ok = True)+ need = nbytes + max(_cache._DISK_MARGIN_BYTES, nbytes // 20)+ free = shutil.disk_usage(str(root)).free+ if free < need:+ _cache.evict(root = root, extra_free_bytes = need - free, keep = cand.key, logger = logger)+ free = shutil.disk_usage(str(root)).free+ if free < need:+ _log(+ logger,+ "info",+ f"{cand.component}: not cached (disk short: {free / 2**30:.1f} GiB free, "+ f"need {need / 2**30:.1f} GiB)",+ )+ return None+ source_bytes = sum(os.path.getsize(os.path.realpath(s)) for s in cand.shards)+ meta = {+ "key": cand.key,+ "component": cand.component,+ "class": cand.class_name,+ "dtype": cand.parts["dtype"],+ "source_gib": round(source_bytes / 2**30, 2),+ "torch_version": str(torch.__version__),+ }+ kdir = root / cand.key+ kdir.mkdir(parents = True, exist_ok = True)+ shutil.copyfile(cand.config_path, kdir / "config.json")+ fd, tmp = tempfile.mkstemp(+ dir = str(kdir), prefix = f".{_cache.FILE_NAME}.", suffix = _cache._TEMP_SUFFIX+ )+ os.close(fd)+ hasher = hashlib.sha256()+ try:+ size = _write(+ tmp,+ cand,+ {CAST_META_KEY: json.dumps(meta, sort_keys = True)},+ hasher,+ reader,+ )+ os.replace(tmp, kdir / _cache.FILE_NAME)+ tmp = None+ finally:+ if tmp is not None:+ try:+ os.unlink(tmp)+ except OSError:+ pass+ sha = hasher.hexdigest()+ _cache._record_verified(kdir, sha)+ manifest = {+ "key": cand.key,+ "bytes": size,+ "sha256": sha,+ "created": time.time(),+ "parts": cand.parts,+ }+ _cache._atomic_bytes(+ kdir / _cache.MANIFEST_NAME,+ json.dumps(manifest, indent = 1, default = str).encode("utf-8"),+ )+ _cache._touch(kdir)+ _log(+ logger,+ "info",+ f"{cand.component}: saved the {cand.parts['dtype']} cast ({cand.key}, {size / 2**30:.2f} GiB from "+ f"{source_bytes / 2**30:.2f} GiB of source shards, in {time.perf_counter() - t0:.1f} s)",+ )+ _cache.evict(logger = logger, keep = cand.key)+ return str(kdir / _cache.FILE_NAME)+ except Exception as exc: # noqa: BLE001 - a cache write never fails a load+ _log(+ logger,+ "warning",+ f"{cand.component}: save failed: {type(exc).__name__}: {str(exc)[:200]}",+ )+ return None+++class CastWrites:+ """Background writes of one load's cast components; registered with the prequant cache's background set."""++ def __init__(+ self,+ pipe: Any,+ cands: list,+ logger: Any = None,+ ) -> None:+ import weakref++ self._pipe = weakref.ref(pipe)+ self._cands = list(cands)+ self.key = ",".join(c.key for c in self._cands)+ self._logger = logger+ self._cancel = threading.Event()+ self._finished = threading.Event()+ self._thread: Optional[threading.Thread] = None+ self.results: dict = {}++ def _check(self) -> None:+ if self._cancel.is_set():+ raise RuntimeError("cast cache write cancelled")++ def start(self) -> bool:+ with _cache._lock:+ if self._thread is not None or self._cancel.is_set():+ return False+ self._thread = threading.Thread(target = self._run, name = "cast-cache-write", daemon = True)+ _cache._BACKGROUND.add(self)+ try:+ self._thread.start()+ except Exception: # noqa: BLE001+ with _cache._lock:+ self._thread = None+ self._close()+ return False+ return True++ def _run(self) -> None:+ try:+ for cand in self._cands:+ self._check()+ pipe = self._pipe()+ module = getattr(pipe, cand.component, None) if pipe is not None else None+ pipe = None+ if module is None:+ continue+ self.results[cand.component] = save(+ cand, module, logger = self._logger, check = self._check+ )+ module = None+ except Exception as exc: # noqa: BLE001+ _log(self._logger, "info", f"background write stopped ({str(exc)[:200]})")+ finally:+ self._close()++ def _close(self) -> None:+ self._cands = []+ with _cache._lock:+ _cache._BACKGROUND.discard(self)+ self._finished.set()++ def discard(self) -> None:+ with _cache._lock:+ started = self._thread is not None+ self._cancel.set()+ if not started:+ self._close()++ def cancel(self) -> None:+ self.discard()++ def done(self) -> bool:+ return self._finished.is_set()++ def wait(self, timeout: Optional[float] = None) -> bool:+ return self._finished.wait(timeout)+++def defer_writes(+ pipe: Any,+ cands: list,+ *,+ logger: Any = None,+) -> Optional[CastWrites]:+ """Capture the missed candidates of a just-loaded ``pipe``; ``start()`` the result after publish. Never raises."""+ try:+ if not enabled() or not cands:+ return None+ captured = capture(pipe, cands, logger = logger)+ return CastWrites(pipe, captured, logger) if captured else None+ except Exception as exc: # noqa: BLE001+ _log(logger, "warning", f"write skipped ({type(exc).__name__}: {exc})")+ return None+++def finish(writes: Any, timeout: float = 60.0) -> None:+ """Teardown: let a running write finish for ``timeout`` s, then cancel it. Never raises."""+ if writes is None:+ return+ try:+ if not writes.wait(timeout):+ writes.cancel()+ writes.wait(30.0)+ except Exception: # noqa: BLE001+ passdiff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex a5d9d388385..3e5e5528a02 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3152,6 +3152,58 @@ def _save_runtime_quant_to_cache(logger.warning("diffusion.prequant_cache: skipped (%s)", exc)+def _read_cast_cache(+ pipe_kwargs: dict,+ *,+ source: Any,+ revision: Optional[str],+ base: Optional[str],+ cancel_check: Any = None,+) -> list:+ """Inject cached float16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""+ try:+ from .diffusion_cast_cache import candidates, read_all++ cands = candidates(+ source,+ torch_dtype = pipe_kwargs.get("torch_dtype"),+ cache_dir = None+ if source and os.path.isdir(str(source))+ else pipe_kwargs.get("cache_dir"),+ revision = revision,+ skip = tuple(pipe_kwargs),+ base_repo = base,+ logger = logger,+ )+ if not cands:+ return []+ hits = read_all(cands, logger = logger, cancel_check = cancel_check)+ pipe_kwargs.update(hits)+ if hits:+ logger.info(+ "video.cast_cache: read %s from the local float16 cast cache",+ ", ".join(sorted(hits)),+ )+ return [c for c in cands if c.component not in hits]+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss+ if cancel_check is not None:+ cancel_check()+ logger.warning("video.cast_cache: skipped (%s)", exc)+ return []+++def _finish_cast_writes(pipe: Any) -> None:+ writes = getattr(pipe, "_unsloth_cast_writes", None)+ if writes is None:+ return+ try:+ from .diffusion_cast_cache import finish+ finish(writes)+ pipe._unsloth_cast_writes = None+ except Exception: # noqa: BLE001 - teardown is best effort+ pass++def _video_native_cache_spec(fam: Any,*,@@ -9144,13 +9196,26 @@ class VideoBackend:# The encoder runs in the worker; the enhancer, processor and duration head are never used.pipe_kwargs.update({name: None for name in LTX25_NONE_COMPONENTS})+ cast_writes = Noneif kind == "pipeline":+ cast_missed = _read_cast_cache(+ pipe_kwargs,+ source = _base_local_dir or fetch_base,+ revision = None if _base_local_dir else _base_fetch_revision,+ base = base,+ cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),+ )# The pre-downloaded snapshot dir keeps from_pretrained off the hub; hub id when pre-download was skipped.pipe = pipeline_cls.from_pretrained(_base_local_dir or fetch_base,**_pinned_base_kwargs(),**pipe_kwargs,)+ if cast_missed:+ from .diffusion_cast_cache import defer_writes++ # Names only: the write reads the live tensors once the load is published.+ cast_writes = defer_writes(pipe, cast_missed, logger = logger)else:transformer_cls = getattr(diffusers, fam.transformer_class)# checkpoint_path was resolved (and downloaded) by the memory-planning branch above.@@ -10390,6 +10455,9 @@ class VideoBackend:transformer_quant_engaged or "off",)start_background_pins(pipe)+ if cast_writes is not None:+ pipe._unsloth_cast_writes = cast_writes+ cast_writes.start()return self.status()@staticmethod@@ -13939,6 +14007,8 @@ class VideoBackend:close_encoder(getattr(state, "pipe", None))# A pinner still copying would hold its chunks past release_pinned_host_memory().stop_background_pins(getattr(state, "pipe", None))+ # A cast cache write still reads the components the teardown frees.+ _finish_cast_writes(getattr(state, "pipe", None))uninstall_cfg_parallel(getattr(state, "pipe", None))# Before clear_gpu_cache(), or the graph pool stays reserved.diffusion_cuda_graph.uninstall_all(diff --git a/studio/backend/tests/test_diffusion_cast_cache.py b/studio/backend/tests/test_diffusion_cast_cache.pynew file mode 100644index 00000000000..6bed8fd7ba2--- /dev/null+++ b/studio/backend/tests/test_diffusion_cast_cache.py@@ -0,0 +1,294 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Local float16 cast cache of fp32-shard components: candidates, bit-identical round trip, refusals, video wiring."""++from __future__ import annotations++import json+import sys+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+transformers = pytest.importorskip("transformers")+pytest.importorskip("safetensors")++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion_cast_cache as CC # noqa: E402+from core.inference import diffusion_prequant_cache as C # noqa: E402++FP16_DEFAULT = {"vae": torch.float32, "default": torch.float16}+++def _tiny_wan():+ torch.manual_seed(0)+ return diffusers.WanTransformer3DModel(+ patch_size = (1, 2, 2),+ num_attention_heads = 2,+ attention_head_dim = 16,+ in_channels = 4,+ out_channels = 4,+ text_dim = 32,+ freq_dim = 16,+ ffn_dim = 64,+ num_layers = 2,+ cross_attn_norm = True,+ qk_norm = "rms_norm_across_heads",+ eps = 1e-6,+ rope_max_seq_len = 32,+ ).to(torch.float32)+++def _tiny_umt5():+ torch.manual_seed(1)+ config = transformers.UMT5Config(+ vocab_size = 64,+ d_model = 32,+ d_kv = 8,+ d_ff = 64,+ num_layers = 2,+ num_heads = 4,+ relative_attention_num_buckets = 8,+ )+ return transformers.UMT5EncoderModel(config).to(torch.float32)++[email protected](autouse = True)+def cache_root(tmp_path, monkeypatch):+ root = tmp_path / "cache"+ monkeypatch.setenv(C.ENV_DIR, str(root))+ monkeypatch.delenv(CC.CAST_CACHE_ENV, raising = False)+ monkeypatch.delenv(C.ENV_MODE, raising = False)+ # No Hub probe from a local directory source.+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)+ with C._lock:+ C._REJECTED.clear()+ return root+++def pipe_dir(tmp_path):+ root = tmp_path / "Wan2.1-T2V-tiny-Diffusers"+ _tiny_wan().save_pretrained(root / "transformer", max_shard_size = "60KB")+ _tiny_umt5().save_pretrained(root / "text_encoder")+ (root / "vae").mkdir()+ (root / "model_index.json").write_text(+ json.dumps(+ {+ "_class_name": "WanPipeline",+ "transformer": ["diffusers", "WanTransformer3DModel"],+ "text_encoder": ["transformers", "UMT5EncoderModel"],+ "vae": ["diffusers", "AutoencoderKLWan"],+ "tokenizer": ["transformers", "T5TokenizerFast"],+ "scheduler": ["diffusers", "UniPCMultistepScheduler"],+ }+ )+ )+ return root+++def _dense(cand):+ kwargs = CC._load_kwargs(cand)+ kwargs.pop("output_loading_info")+ return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)+++class _Pipe:+ pass+++def _save(cand, module):+ pipe = _Pipe()+ setattr(pipe, cand.component, module)+ writes = CC.defer_writes(pipe, [cand])+ assert writes is not None and writes.start()+ assert writes.wait(60)+ return writes.results.get(cand.component)+++def _bits(t):+ return t.reshape(-1).view(torch.uint8) if t.numel() else t+++def _identical(a, b):+ sa, sb = a.state_dict(), b.state_dict()+ assert sa.keys() == sb.keys()+ for k in sa:+ assert sa[k].dtype == sb[k].dtype, k+ assert torch.equal(_bits(sa[k]), _bits(sb[k])), k+ ba, bb = dict(a.named_buffers()), dict(b.named_buffers())+ assert ba.keys() == bb.keys()+ for k in ba:+ assert ba[k].dtype == bb[k].dtype and torch.equal(ba[k], bb[k]), k+++def _by_name(cands):+ return {c.component: c for c in cands}+++def test_candidates_are_fp32_shards_loaded_at_fp16(pipe_dir):+ cands = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))+ assert set(cands) == {+ "transformer",+ "text_encoder",+ } # the vae stays fp32; the rest hold no weights+ assert len(cands["transformer"].shards) > 1+ assert cands["text_encoder"].parts["keep_in_fp32"] == ["wo"]+ assert cands["transformer"].key != cands["text_encoder"].key++[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_no_candidates_outside_float16(pipe_dir, dtype):+ assert CC.candidates(pipe_dir, torch_dtype = dtype) == []+++def test_kill_switches(pipe_dir, monkeypatch):+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "0")+ assert CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT) == []+ monkeypatch.delenv(CC.CAST_CACHE_ENV)+ monkeypatch.setenv(C.ENV_MODE, "0")+ assert CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT) == []+++def test_skip_and_narrow_shards(pipe_dir):+ assert set(+ _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT, skip = ("text_encoder",)))+ ) == {"transformer"}+ _tiny_umt5().to(torch.float16).save_pretrained(pipe_dir / "text_encoder")+ assert set(_by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))) == {"transformer"}++[email protected]("component", ["transformer", "text_encoder"])+def test_round_trip_is_bit_identical(pipe_dir, component):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))[component]+ assert CC.read(cand) is None # cold: a miss+ dense = _dense(cand)+ path = _save(cand, dense)+ assert path is not None and C.path_in_cache(path)+ stored = CC._shard_header(path)+ assert {e["dtype"] for e in stored.values()} == {"F16", "F32"} # fp32 keep modules stay fp32+ fresh = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))[component]+ hit = CC.read(fresh)+ assert hit is not None+ _identical(dense, hit)+ _identical(_dense(fresh), hit)+ if component == "text_encoder":+ # the tied embedding is written once and re-tied by from_pretrained+ assert "encoder.embed_tokens.weight" not in stored+ assert hit.shared.weight.data_ptr() == hit.encoder.embed_tokens.weight.data_ptr()+++def test_module_changed_in_place_is_not_cached(pipe_dir):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ dense = _dense(cand)+ pipe = _Pipe()+ pipe.transformer = dense+ writes = CC.defer_writes(pipe, [cand])+ with torch.no_grad():+ for p in dense.parameters():+ p.mul_(2)+ writes.start()+ writes.wait(60)+ assert writes.results["transformer"] is None+ assert C._entry(cand.key) is None+++def test_module_restructured_after_load_is_not_cached(pipe_dir):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ dense = _dense(cand)+ pipe = _Pipe()+ pipe.transformer = dense+ writes = CC.defer_writes(pipe, [cand])+ dense.proj_out = torch.nn.Linear(4, 4)+ writes.start()+ writes.wait(60)+ assert writes.results["transformer"] is None+++def test_new_source_bytes_change_the_key(pipe_dir):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]+ assert _save(cand, _dense(cand)) is not None+ model = _tiny_umt5()+ with torch.no_grad():+ model.shared.weight.add_(1)+ model.save_pretrained(pipe_dir / "text_encoder")+ fresh = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]+ assert fresh.key != cand.key+ assert CC.read(fresh) is None+++def test_corrupt_entry_is_a_miss(pipe_dir):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ path = Path(_save(cand, _dense(cand)))+ raw = bytearray(path.read_bytes())+ raw[-1] ^= 0xFF+ path.write_bytes(bytes(raw))+ (path.parent / C.VERIFIED_NAME).unlink()+ assert CC.read(cand) is None+ with C._lock:+ assert str(path.resolve()) in C._REJECTED+++def test_video_reader_injects_hits_and_returns_misses(pipe_dir):+ from core.inference import video as V++ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]+ _save(cand, _dense(cand))+ pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}+ missed = V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None)+ assert [c.component for c in missed] == ["transformer"]+ assert isinstance(pipe_kwargs["text_encoder"], transformers.UMT5EncoderModel)+ assert "transformer" not in pipe_kwargs+ # an injected component is never a candidate+ again = V._read_cast_cache(dict(pipe_kwargs), source = str(pipe_dir), revision = None, base = None)+ assert [c.component for c in again] == ["transformer"]+++def test_teardown_waits_then_clears(pipe_dir):+ from core.inference import video as V++ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ pipe = _Pipe()+ pipe.transformer = _dense(cand)+ writes = CC.defer_writes(pipe, [cand])+ pipe._unsloth_cast_writes = writes+ writes.start()+ V._finish_cast_writes(pipe)+ assert pipe._unsloth_cast_writes is None+ assert writes.done()+ assert C._entry(cand.key) is not None+++def test_sample_keys_tolerate_one_kind_per_shard():+ names = {"a": torch.float16, "b": torch.float16, "w": torch.float32}+ entry = lambda n: {"dtype": "F32", "data_offsets": [0, n]} # noqa: E731+ assert CC._sample_keys({"a": entry(8), "b": entry(16)}, names) == ["b"]+ assert CC._sample_keys({"w": entry(8)}, names) == ["w"]+ assert CC._sample_keys({"a": entry(CC._SAMPLE_MAX_BYTES + 1)}, names) == []+++def test_save_when_kept_fp32_tensors_exceed_the_sample_size(pipe_dir, monkeypatch):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]+ dense = _dense(cand)+ wide = min(t.numel() * 4 for n, t in dense.state_dict().items() if t.dtype == torch.float32)+ monkeypatch.setattr(CC, "_SAMPLE_MAX_BYTES", wide - 1)+ assert _save(cand, dense) is not None+ _identical(dense, CC.read(cand))++[email protected](not torch.cuda.is_available(), reason = "needs a CUDA device")+def test_resident_device_module_is_read_back_bit_identical(pipe_dir):+ cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]+ dense = _dense(cand)+ reference = {k: v.clone() for k, v in dense.state_dict().items()}+ dense.to("cuda")+ assert _save(cand, dense) is not None+ hit = CC.read(cand)+ for k, v in hit.state_dict().items():+ assert v.dtype == reference[k].dtype and torch.equal(_bits(v), _bits(reference[k])), k