Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: cast cache for fp32-shard components at bf16 too, and on the image path (Z-Image-Turbo reads 11.5 GiB, not 22.9) (#15321)
Daniel Hancad94d3747d3
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex e22ec3eaa76..428f1ec8932 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -4034,6 +4034,46 @@ def _comfy_keep_fp32_pricing(fam: Any, target: Any) -> dict:return {"keep_fp32": keep, "keep_fp32_bytes": 4 if fp16 else None}+def _image_cast_candidates(+ fam: Any,+ source: Any,+ *,+ dtype: Any,+ base: Optional[str],+ skip: tuple = (),+) -> list:+ """Generic diffusers assembly only (per-component assemblers are not wired). Never raises."""+ try:+ from .diffusion_cast_cache import candidates, image_enabled++ if not image_enabled() or not source:+ return []+ if is_flux3_family(fam) or getattr(fam, "name", None) in (+ MOSS_IMAGE_FAMILY_NAME,+ KREA2_FAMILY_NAME,+ IDEOGRAM4_FAMILY_NAME,+ ):+ return []+ return candidates(+ source,+ torch_dtype = dtype,+ cache_dir = None if os.path.isdir(str(source)) else hub_cache_dir(),+ skip = skip,+ base_repo = base,+ logger = logger,+ )+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss+ logger.warning("diffusion.cast_cache: skipped (%s)", exc)+ return []+++def _image_cast_cached(cands: list, variant: Optional[str]) -> list:+ if not cands or variant is not None:+ return []+ from .diffusion_cast_cache import cached_components+ return cached_components(cands)++def _converted_shards_pread(dtype: Any, small_host: Any) -> Any:"""Windows: transformers components converted to ``dtype`` read through ``pread`` (no copy-on-write commit for thestored bytes, #14513); components the small-host route loads at their stored dtype stay memory-mapped."""@@ -9402,6 +9442,7 @@ class DiffusionBackend:load_bg_compile = Nonestate = Nonepending_cache_write = None+ pending_cast_write = Nonenative_streamed = Falsetry:self._raise_if_load_cancelled(_load_token)@@ -9567,6 +9608,13 @@ class DiffusionBackend:if _fp16_variant_eligibleelse None)+ _cast_cands = _image_cast_candidates(+ fam,+ _base_local_dir or fetch_base,+ dtype = dtype,+ base = base,+ skip = tuple(_te_replaced or ()),+ )# Stopped once the pipeline is built.self._stop_load_prefetch()self._load_prefetch = start_load_prefetch(@@ -9575,7 +9623,8 @@ class DiffusionBackend:prequant_scheme = pipeline_seed_scheme,prequant_path_override = transformer_prequant_path,prequant_base_repo = base,- text_encoders_replaced = _te_replaced,+ text_encoders_replaced = tuple(_te_replaced or ())+ + tuple(_image_cast_cached(_cast_cands, _fp16_variant)),cache_dir = hub_cache_dir(),logger = logger,variant = _fp16_variant,@@ -9848,6 +9897,24 @@ class DiffusionBackend:_fp16_variant,str(dtype).replace("torch.", ""),)+ _cast_missed = []+ if (+ _cast_cands+ and _fp16_variant is None+ and not (small_host is not None and small_host.engaged)+ ):+ from .diffusion_cast_cache import inject as _cast_inject++ # Twins (fp16 variant) and the small-host dtype map read other bytes.+ _cast_missed = _cast_inject(+ pipe_kwargs,+ _cast_cands,+ logger = logger,+ cancel_check = lambda: self._raise_if_load_cancelled(+ _load_token+ ),+ label = "diffusion",+ )# The prefetched snapshot dir keeps from_pretrained off the hub (24 GB per FLUX.1# otherwise)try:@@ -9857,6 +9924,13 @@ class DiffusionBackend:)finally:self._stop_load_prefetch()+ if _cast_missed:+ from .diffusion_cast_cache import defer_writes++ # Names only: the write reads the live tensors once the load is published.+ pending_cast_write = defer_writes(+ pipe, _cast_missed, logger = logger+ )if fam.single_file_is_pipeline:# A v-pred SDXL repo can ship an epsilon scheduler config beside the root# single file that declares v_pred / ztsnr (NoobAI-XL V-Pred 1.0).@@ -11320,6 +11394,15 @@ class DiffusionBackend:except Exception: # noqa: BLE001 - a cache write never fails a loadpasspending_cache_write = None+ if pending_cast_write is not None:+ try:+ if state_committed:+ pending_cast_write.start()+ else:+ pending_cast_write.discard()+ except Exception: # noqa: BLE001 - a cache write never fails a load+ pass+ pending_cast_write = None# Pre-commit failure: roll back the process-wide mutations (symmetric with _unload_locked).if not state_committed:# Never published, so _unload_locked cannot reach them: the pinners would keep the encoders.diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.pyindex 15046aebdf5..49e69ee0aba 100644--- a/studio/backend/core/inference/diffusion_cast_cache.py+++ b/studio/backend/core/inference/diffusion_cast_cache.py@@ -1,11 +1,14 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Local float16 cache of components loaded from fp32 shards (e.g. Wan), written from the live tensors after publish.+"""Local float16 / bfloat16 cache of components loaded from fp32 shards (Wan, Z-Image, SDXL, Lumina, ...), written+from the live tensors after publish.Later loads read the half-size file at the same dtype (a no-op cast, bit-identical); a sampled source comparisonrefuses a module changed in place. Shares ``diffusion_prequant_cache``'s root, locks, manifest and LRU.-Kill switch: ``UNSLOTH_DIFFUSION_CAST_CACHE=0`` (``UNSLOTH_DIFFUSION_PREQUANT_CACHE=0`` disables it too).+Kill switch: ``UNSLOTH_DIFFUSION_CAST_CACHE=0`` (``UNSLOTH_DIFFUSION_PREQUANT_CACHE=0`` disables it too);+``UNSLOTH_DIFFUSION_CAST_CACHE=fp16`` / ``=bf16`` keeps one target dtype only; ``UNSLOTH_DIFFUSION_CAST_CACHE_IMAGE=0``+turns the image pipeline path off."""from __future__ import annotations@@ -25,6 +28,9 @@ from typing import Any, Iterable, Optionalfrom . import diffusion_prequant_cache as _cacheCAST_CACHE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE"+CAST_CACHE_IMAGE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE_IMAGE"+_OFF = ("0", "off", "false", "no", "none")+_HALF = {"float16": "float16", "fp16": "float16", "bfloat16": "bfloat16", "bf16": "bfloat16"}CAST_LAYOUT_VERSION = 1CAST_META_KEY = "unsloth_cast_component"_LIBRARIES = ("diffusers", "transformers")@@ -40,11 +46,21 @@ _ONLOAD_WAIT_SECONDS = 10.0def enabled() -> bool:raw = (os.environ.get(CAST_CACHE_ENV) or "").strip().lower()- return raw not in ("0", "off", "false", "no", "none") and _cache.enabled()+ return raw not in _OFF and _cache.enabled()-def _is_fp16(dtype: Any) -> bool:- return str(dtype) in ("torch.float16", "float16")+def image_enabled() -> bool:+ raw = (os.environ.get(CAST_CACHE_IMAGE_ENV) or "").strip().lower()+ return raw not in _OFF and enabled()+++def _target_dtypes() -> tuple:+ raw = (os.environ.get(CAST_CACHE_ENV) or "").strip().lower()+ return (_HALF[raw],) if raw in _HALF else ("float16", "bfloat16")+++def _is_half(dtype: Any) -> bool:+ return str(dtype).replace("torch.", "") in _target_dtypes()def _component_dtype(torch_dtype: Any, component: str) -> Any:@@ -74,6 +90,7 @@ class CastCandidate:key: strparts: dictnames: list = field(default_factory = list)+ nbytes: int = 0def _read_json(path: Optional[str]) -> Optional[dict]:@@ -119,20 +136,54 @@ def _shard_header(path: str) -> Optional[dict]:return header-def _all_wide(shards: list) -> bool:- """Every floating tensor of every shard is wider than float16 (the only case the cast halves)."""+def _cast_bytes(shards: list) -> Optional[int]:+ """16-bit bytes if every floating tensor is fp32 / fp64, else None. An estimate: fp32-kept modules count at 16 bits."""seen = False+ total = 0for shard in shards:header = _shard_header(shard)if header is None:- return False+ return Nonefor entry in header.values():- tag = entry.get("dtype") if isinstance(entry, dict) else None+ if not isinstance(entry, dict):+ continue+ tag = entry.get("dtype")+ try:+ begin, end = (int(v) for v in entry.get("data_offsets") or (0, 0))+ except (TypeError, ValueError):+ return Noneif tag in _FLOAT_TAGS:if tag not in _WIDE_TAGS:- return False+ return Noneseen = True- return seen+ total += (end - begin) // (4 if tag == "F32" else 8) * 2+ else:+ total += end - begin+ return total if seen else None+++def _all_wide(shards: list) -> bool:+ return _cast_bytes(shards) is not None+++def _fit_budget(cands: list, logger: Any = None) -> list:+ """A set over the LRU budget would evict its own entries every load. Greedy largest-first, so it is deterministic."""+ budget = _cache.max_cache_bytes()+ if budget is None:+ return cands+ keep, used = set(), 0+ for cand in sorted(cands, key = lambda c: (-c.nbytes, c.component)):+ if used + cand.nbytes > budget:+ _log(+ logger,+ "info",+ f"{cand.component}: not cached ({cand.nbytes / 2**30:.1f} GiB would not fit the "+ f"{budget / 2**30:.0f} GiB cache budget beside this load's other components)",+ )+ continue+ used += cand.nbytes+ keep.add(cand.component)+ return [c for c in cands if c.component in keep]def _load_class(library: str, class_name: str) -> Any:@@ -156,7 +207,8 @@ def candidates(base_repo: Optional[str] = None,logger: Any = None,) -> list:- """Components at ``source`` that load fp32 shards at float16, with cache keys. Local reads only; never raises."""+ """Components at ``source`` that load fp32 shards at float16 / bfloat16, with cache keys. Local reads only; never+ raises."""if not enabled() or not source:return []out: list = []@@ -177,13 +229,14 @@ def candidates(if library not in _LIBRARIES or not isinstance(class_name, str):continuedtype = _component_dtype(torch_dtype, component)- if not _is_fp16(dtype):+ if not _is_half(dtype):continuecls = _load_class(library, class_name)if cls is None or not isinstance(cls, type) or not issubclass(cls, torch.nn.Module):continueshards = _shards_of(source, component, _STEMS[library], cache_dir, revision)- if not shards or not _all_wide(shards):+ nbytes = _cast_bytes(shards) if shards else None+ if nbytes is None:continueconfig_path = _cache._local_file(source, f"{component}/config.json", cache_dir, revision@@ -222,12 +275,48 @@ def candidates(config_path = str(config_path),key = _cache.cache_key(parts),parts = parts,+ nbytes = nbytes,))except Exception as exc: # noqa: BLE001_log(logger, "warning", f"candidates skipped ({type(exc).__name__}: {str(exc)[:200]})")return []- return out+ return _fit_budget(out, logger)+++def cached_components(cands: list) -> list:+ try:+ root = _cache.cache_root()+ return [c.component for c in cands or () if (root / c.key / _cache.MANIFEST_NAME).is_file()]+ except Exception: # noqa: BLE001+ return []+++def inject(+ pipe_kwargs: dict,+ cands: list,+ *,+ logger: Any = None,+ cancel_check: Optional[Any] = None,+ label: str = "diffusion",+) -> list:+ """Returns the misses. Never raises, except a cancel from ``cancel_check``."""+ try:+ cands = [c for c in cands or () if c.component not in pipe_kwargs]+ if not cands:+ return []+ hits = read_all(cands, logger = logger, cancel_check = cancel_check)+ pipe_kwargs.update(hits)+ if hits and logger is not None:+ logger.info(+ "%s.cast_cache: read %s from the local cast cache", label, ", ".join(sorted(hits))+ )+ return [c for c in cands if c.component not in hits]+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss+ if cancel_check is not None:+ cancel_check()+ _log(logger, "warning", f"inject skipped ({type(exc).__name__}: {str(exc)[:200]})")+ return []def _log(logger: Any, level: str, msg: str) -> None:@@ -588,18 +677,22 @@ def save(root = _cache.cache_root()try:names = _state_names(module)- if names is None or [n for n, _d, _s in names] != [n for n, _d, _s in cand.names]:+ live = {n: (d, s) for n, d, s in names or ()}+ # Tensors added after the load (fuse_qkv's to_qkv) are left out: the next load re-adds them from the same+ # weights. A renamed / removed / re-typed one (LoRA wrap, quantize, layerwise fp8 storage) refuses the write.+ if names is None or any(n not in live for n, _d, _s in cand.names):_log(logger, "info", f"{cand.component}: not cached (its tensors changed after the load)")return None- if any(d != dt or s != st for (_n, d, s), (_m, dt, st) in zip(names, cand.names)):+ if any(live[n] != (d, s) for n, d, s in cand.names):_log(logger,"info",f"{cand.component}: not cached (a tensor changed dtype or shape after the load)",)return None+ names = list(cand.names)reader = _Reader(module, check)why = _matches_source(cand, reader)if why is not None:@@ -751,6 +844,8 @@ class CastWrites:pipe = self._pipe()module = getattr(pipe, cand.component, None) if pipe is not None else Nonepipe = None+ # torch.compile(module) wraps it; the weights (and their names) are the original's.+ module = getattr(module, "_orig_mod", module)if module is None:continueself.results[cand.component] = save(diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 3e5e5528a02..a71064319dd 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3160,7 +3160,7 @@ def _read_cast_cache(base: Optional[str],cancel_check: Any = None,) -> list:- """Inject cached float16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""+ """Inject cached float16 / bfloat16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""try:from .diffusion_cast_cache import candidates, read_all@@ -3181,7 +3181,7 @@ def _read_cast_cache(pipe_kwargs.update(hits)if hits:logger.info(- "video.cast_cache: read %s from the local float16 cast cache",+ "video.cast_cache: read %s from the local cast cache",", ".join(sorted(hits)),)return [c for c in cands if c.component not in hits]diff --git a/studio/backend/tests/test_diffusion_cast_cache.py b/studio/backend/tests/test_diffusion_cast_cache.pyindex 6bed8fd7ba2..43efb39fa5f 100644--- a/studio/backend/tests/test_diffusion_cast_cache.py+++ b/studio/backend/tests/test_diffusion_cast_cache.py@@ -143,9 +143,8 @@ def test_candidates_are_fp32_shards_loaded_at_fp16(pipe_dir):assert cands["transformer"].key != cands["text_encoder"].key[email protected]("dtype", [torch.bfloat16, torch.float32])-def test_no_candidates_outside_float16(pipe_dir, dtype):- assert CC.candidates(pipe_dir, torch_dtype = dtype) == []+def test_no_candidates_at_float32(pipe_dir):+ assert CC.candidates(pipe_dir, torch_dtype = torch.float32) == []def test_kill_switches(pipe_dir, monkeypatch):diff --git a/studio/backend/tests/test_diffusion_cast_cache_bf16_image.py b/studio/backend/tests/test_diffusion_cast_cache_bf16_image.pynew file mode 100644index 00000000000..cb8fa947667--- /dev/null+++ b/studio/backend/tests/test_diffusion_cast_cache_bf16_image.py@@ -0,0 +1,273 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Cast cache beyond video float16: bfloat16 targets, the image pipeline wiring, the per-load budget, live-module+tolerances (a compiled wrapper, tensors added after the load) and the per-dtype / per-path switches."""++from __future__ import annotations++import json+import sys+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+transformers = pytest.importorskip("transformers")+pytest.importorskip("safetensors")++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion_cast_cache as CC # noqa: E402+from core.inference import diffusion_prequant_cache as C # noqa: E402++BF16 = torch.bfloat16+HALF = (torch.float16, torch.bfloat16)+IMAGE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE_IMAGE"+++def _tiny_zimage():+ torch.manual_seed(0)+ return diffusers.ZImageTransformer2DModel(+ all_patch_size = (2,),+ all_f_patch_size = (1,),+ in_channels = 4,+ dim = 32,+ n_layers = 2,+ n_refiner_layers = 1,+ n_heads = 2,+ n_kv_heads = 2,+ cap_feat_dim = 16,+ axes_dims = [4, 6, 6],+ axes_lens = [64, 32, 32],+ ).to(torch.float32)+++def _tiny_vae():+ torch.manual_seed(1)+ return diffusers.AutoencoderKL(+ in_channels = 3,+ out_channels = 3,+ down_block_types = ("DownEncoderBlock2D",),+ up_block_types = ("UpDecoderBlock2D",),+ block_out_channels = (8,),+ latent_channels = 4,+ norm_num_groups = 4,+ ).to(torch.float32)+++def _tiny_umt5():+ torch.manual_seed(2)+ config = transformers.UMT5Config(+ vocab_size = 64,+ d_model = 32,+ d_kv = 8,+ d_ff = 64,+ num_layers = 2,+ num_heads = 4,+ relative_attention_num_buckets = 8,+ )+ return transformers.UMT5EncoderModel(config).to(torch.float32)++[email protected](autouse = True)+def cache_root(tmp_path, monkeypatch):+ root = tmp_path / "cache"+ monkeypatch.setenv(C.ENV_DIR, str(root))+ for env in (CC.CAST_CACHE_ENV, IMAGE_ENV, C.ENV_MODE, C.ENV_MAX_GB):+ monkeypatch.delenv(env, raising = False)+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)+ with C._lock:+ C._REJECTED.clear()+ return root+++def image_dir(tmp_path):+ root = tmp_path / "Z-Image-tiny"+ _tiny_zimage().save_pretrained(root / "transformer", max_shard_size = "40KB")+ _tiny_vae().save_pretrained(root / "vae")+ _tiny_umt5().save_pretrained(root / "text_encoder")+ (root / "model_index.json").write_text(+ json.dumps(+ {+ "_class_name": "ZImagePipeline",+ "transformer": ["diffusers", "ZImageTransformer2DModel"],+ "vae": ["diffusers", "AutoencoderKL"],+ "text_encoder": ["transformers", "UMT5EncoderModel"],+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],+ }+ )+ )+ return root+++def _by_name(cands):+ return {c.component: c for c in cands}+++def _dense(cand):+ kwargs = CC._load_kwargs(cand)+ kwargs.pop("output_loading_info")+ return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)+++class _Pipe:+ pass+++def _write(pipe, cands):+ writes = CC.defer_writes(pipe, cands)+ assert writes is not None and writes.start()+ assert writes.wait(60)+ return writes.results+++def _bits(t):+ return t.reshape(-1).view(torch.uint8) if t.numel() else t+++def _identical(a, b):+ sa, sb = a.state_dict(), b.state_dict()+ assert sa.keys() == sb.keys()+ for k in sa:+ assert sa[k].dtype == sb[k].dtype, k+ assert torch.equal(_bits(sa[k]), _bits(sb[k])), k++[email protected]("dtype", HALF)+def test_every_fp32_component_is_a_candidate_at_both_half_dtypes(image_dir, dtype):+ cands = _by_name(CC.candidates(image_dir, torch_dtype = dtype))+ assert set(cands) == {"transformer", "vae", "text_encoder"}+ assert {c.parts["dtype"] for c in cands.values()} == {str(dtype).replace("torch.", "")}+ assert all(c.nbytes > 0 for c in cands.values())+++def test_fp16_and_bf16_entries_never_share_a_key(image_dir):+ a = _by_name(CC.candidates(image_dir, torch_dtype = torch.float16))+ b = _by_name(CC.candidates(image_dir, torch_dtype = BF16))+ assert all(a[k].key != b[k].key for k in a)++[email protected]("dtype", HALF)[email protected]("component", ["transformer", "vae", "text_encoder"])+def test_round_trip_is_bit_identical(image_dir, dtype, component):+ cand = _by_name(CC.candidates(image_dir, torch_dtype = dtype))[component]+ assert CC.read(cand) is None+ dense = _dense(cand)+ pipe = _Pipe()+ setattr(pipe, component, dense)+ path = _write(pipe, [cand])[component]+ assert path is not None and C.path_in_cache(path)+ tag = {torch.float16: "F16", BF16: "BF16"}[dtype]+ assert tag in {e["dtype"] for e in CC._shard_header(path).values()}+ fresh = _by_name(CC.candidates(image_dir, torch_dtype = dtype))[component]+ hit = CC.read(fresh)+ assert hit is not None+ _identical(dense, hit)+++def test_dtype_switch_keeps_one_target(image_dir, monkeypatch):+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "fp16")+ assert CC.candidates(image_dir, torch_dtype = BF16) == []+ assert CC.candidates(image_dir, torch_dtype = torch.float16)+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "bf16")+ assert CC.candidates(image_dir, torch_dtype = torch.float16) == []+ assert CC.candidates(image_dir, torch_dtype = BF16)+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "0")+ assert CC.candidates(image_dir, torch_dtype = BF16) == []+++def test_load_set_fits_the_budget(image_dir, monkeypatch):+ sizes = {c.component: c.nbytes for c in CC.candidates(image_dir, torch_dtype = BF16)}+ # room for the transformer alone: the others never get an entry they would evict on every load+ monkeypatch.setenv(C.ENV_MAX_GB, str((sizes["transformer"] + 1) / 2**30))+ names = [c.component for c in CC.candidates(image_dir, torch_dtype = BF16)]+ assert "transformer" in names and sum(sizes[n] for n in names) <= sizes["transformer"] + 1+++def test_compiled_wrapper_and_added_tensors_still_write_the_load(image_dir):+ cand = _by_name(CC.candidates(image_dir, torch_dtype = BF16))["transformer"]+ dense = _dense(cand)+ reference = {k: v.clone() for k, v in dense.state_dict().items()}+ pipe = _Pipe()+ pipe.transformer = dense+ writes = CC.defer_writes(pipe, [cand])+ dense.register_parameter("extra_fused", torch.nn.Parameter(torch.zeros(3, dtype = BF16)))+ pipe.transformer = torch.compile(dense)+ assert writes.start() and writes.wait(60)+ assert writes.results["transformer"] is not None+ hit = CC.read(cand)+ assert set(hit.state_dict()) == set(reference)+ for k, v in hit.state_dict().items():+ assert torch.equal(_bits(v), _bits(reference[k])), k+++def test_renamed_or_retyped_tensor_still_refuses(image_dir):+ cand = _by_name(CC.candidates(image_dir, torch_dtype = BF16))["vae"]+ dense = _dense(cand)+ pipe = _Pipe()+ pipe.vae = dense+ writes = CC.defer_writes(pipe, [cand])+ dense.to(+ torch.float32+ ) # an upcast after the load (or fp8 layerwise storage) is not what the load produced+ assert writes.start() and writes.wait(60)+ assert writes.results["vae"] is None+++def test_image_helpers_inject_hits_and_skip_the_prefetch(image_dir, monkeypatch):+ from core.inference import diffusion as D++ fams = {f.name: f for f in D._IMAGE_REGISTRY}+ fam = fams["z-image"]+ cands = D._image_cast_candidates(fam, str(image_dir), dtype = BF16, base = None)+ assert {c.component for c in cands} == {"transformer", "vae", "text_encoder"}+ assert D._image_cast_cached(cands, None) == []+ tr = _by_name(cands)["transformer"]+ pipe = _Pipe()+ pipe.transformer = _dense(tr)+ assert _write(pipe, [tr])["transformer"] is not None+ cands = D._image_cast_candidates(fam, str(image_dir), dtype = BF16, base = None)+ assert D._image_cast_cached(cands, None) == ["transformer"]+ assert D._image_cast_cached(cands, "fp16") == [] # twins read: the prefetch must read them+ pipe_kwargs = {"torch_dtype": BF16, "local_files_only": True}+ missed = CC.inject(pipe_kwargs, cands)+ assert sorted(c.component for c in missed) == ["text_encoder", "vae"]+ _identical(pipe_kwargs["transformer"], pipe.transformer)+ # the per-path switch, and families assembled per component, are never wired+ monkeypatch.setenv(IMAGE_ENV, "0")+ assert D._image_cast_candidates(fam, str(image_dir), dtype = BF16, base = None) == []+ monkeypatch.delenv(IMAGE_ENV)+ for name in (D.KREA2_FAMILY_NAME, D.MOSS_IMAGE_FAMILY_NAME, "flux.3-klein"):+ assert D._image_cast_candidates(fams[name], str(image_dir), dtype = BF16, base = None) == []+++def test_video_reader_serves_bf16(image_dir):+ from core.inference import video as V++ cand = _by_name(CC.candidates(image_dir, torch_dtype = BF16))["text_encoder"]+ pipe = _Pipe()+ pipe.text_encoder = _dense(cand)+ assert _write(pipe, [cand])["text_encoder"] is not None+ pipe_kwargs = {"torch_dtype": {"vae": torch.float32, "default": BF16}, "local_files_only": True}+ missed = V._read_cast_cache(pipe_kwargs, source = str(image_dir), revision = None, base = None)+ assert sorted(c.component for c in missed) == ["transformer"]+ _identical(pipe_kwargs["text_encoder"], pipe.text_encoder)+++def test_pipeline_from_injected_components_matches_the_dense_load(image_dir):+ """The whole assembly: a pipeline loaded with the hits equals the dense bf16 pipeline load bit for bit."""+ cands = CC.candidates(image_dir, torch_dtype = BF16)+ dense = {c.component: _dense(c) for c in cands}+ pipe = _Pipe()+ for name, module in dense.items():+ setattr(pipe, name, module)+ assert all(_write(pipe, cands).values())+ kwargs = {}+ assert CC.inject(kwargs, CC.candidates(image_dir, torch_dtype = BF16)) == []+ for name, module in dense.items():+ _identical(module, kwargs[name])