Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: cast cache for fp32-shard components at bf16 too, and on the image path (Z-Image-Turbo reads 11.5 GiB, not 22.9) (#15321)

Daniel Hancad94d3747d3
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index e22ec3eaa76..428f1ec8932 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -4034,6 +4034,46 @@ def _comfy_keep_fp32_pricing(fam: Any, target: Any) -> dict:
return {"keep_fp32": keep, "keep_fp32_bytes": 4 if fp16 else None}
+def _image_cast_candidates(
+ fam: Any,
+ source: Any,
+ *,
+ dtype: Any,
+ base: Optional[str],
+ skip: tuple = (),
+) -> list:
+ """Generic diffusers assembly only (per-component assemblers are not wired). Never raises."""
+ try:
+ from .diffusion_cast_cache import candidates, image_enabled
+
+ if not image_enabled() or not source:
+ return []
+ if is_flux3_family(fam) or getattr(fam, "name", None) in (
+ MOSS_IMAGE_FAMILY_NAME,
+ KREA2_FAMILY_NAME,
+ IDEOGRAM4_FAMILY_NAME,
+ ):
+ return []
+ return candidates(
+ source,
+ torch_dtype = dtype,
+ cache_dir = None if os.path.isdir(str(source)) else hub_cache_dir(),
+ skip = skip,
+ base_repo = base,
+ logger = logger,
+ )
+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss
+ logger.warning("diffusion.cast_cache: skipped (%s)", exc)
+ return []
+
+
+def _image_cast_cached(cands: list, variant: Optional[str]) -> list:
+ if not cands or variant is not None:
+ return []
+ from .diffusion_cast_cache import cached_components
+ return cached_components(cands)
+
+
def _converted_shards_pread(dtype: Any, small_host: Any) -> Any:
"""Windows: transformers components converted to ``dtype`` read through ``pread`` (no copy-on-write commit for the
stored bytes, #14513); components the small-host route loads at their stored dtype stay memory-mapped."""
@@ -9402,6 +9442,7 @@ class DiffusionBackend:
load_bg_compile = None
state = None
pending_cache_write = None
+ pending_cast_write = None
native_streamed = False
try:
self._raise_if_load_cancelled(_load_token)
@@ -9567,6 +9608,13 @@ class DiffusionBackend:
if _fp16_variant_eligible
else None
)
+ _cast_cands = _image_cast_candidates(
+ fam,
+ _base_local_dir or fetch_base,
+ dtype = dtype,
+ base = base,
+ skip = tuple(_te_replaced or ()),
+ )
# Stopped once the pipeline is built.
self._stop_load_prefetch()
self._load_prefetch = start_load_prefetch(
@@ -9575,7 +9623,8 @@ class DiffusionBackend:
prequant_scheme = pipeline_seed_scheme,
prequant_path_override = transformer_prequant_path,
prequant_base_repo = base,
- text_encoders_replaced = _te_replaced,
+ text_encoders_replaced = tuple(_te_replaced or ())
+ + tuple(_image_cast_cached(_cast_cands, _fp16_variant)),
cache_dir = hub_cache_dir(),
logger = logger,
variant = _fp16_variant,
@@ -9848,6 +9897,24 @@ class DiffusionBackend:
_fp16_variant,
str(dtype).replace("torch.", ""),
)
+ _cast_missed = []
+ if (
+ _cast_cands
+ and _fp16_variant is None
+ and not (small_host is not None and small_host.engaged)
+ ):
+ from .diffusion_cast_cache import inject as _cast_inject
+
+ # Twins (fp16 variant) and the small-host dtype map read other bytes.
+ _cast_missed = _cast_inject(
+ pipe_kwargs,
+ _cast_cands,
+ logger = logger,
+ cancel_check = lambda: self._raise_if_load_cancelled(
+ _load_token
+ ),
+ label = "diffusion",
+ )
# The prefetched snapshot dir keeps from_pretrained off the hub (24 GB per FLUX.1
# otherwise)
try:
@@ -9857,6 +9924,13 @@ class DiffusionBackend:
)
finally:
self._stop_load_prefetch()
+ if _cast_missed:
+ from .diffusion_cast_cache import defer_writes
+
+ # Names only: the write reads the live tensors once the load is published.
+ pending_cast_write = defer_writes(
+ pipe, _cast_missed, logger = logger
+ )
if fam.single_file_is_pipeline:
# A v-pred SDXL repo can ship an epsilon scheduler config beside the root
# single file that declares v_pred / ztsnr (NoobAI-XL V-Pred 1.0).
@@ -11320,6 +11394,15 @@ class DiffusionBackend:
except Exception: # noqa: BLE001 - a cache write never fails a load
pass
pending_cache_write = None
+ if pending_cast_write is not None:
+ try:
+ if state_committed:
+ pending_cast_write.start()
+ else:
+ pending_cast_write.discard()
+ except Exception: # noqa: BLE001 - a cache write never fails a load
+ pass
+ pending_cast_write = None
# Pre-commit failure: roll back the process-wide mutations (symmetric with _unload_locked).
if not state_committed:
# Never published, so _unload_locked cannot reach them: the pinners would keep the encoders.
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.py
index 15046aebdf5..49e69ee0aba 100644
--- a/studio/backend/core/inference/diffusion_cast_cache.py
+++ b/studio/backend/core/inference/diffusion_cast_cache.py
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Local float16 cache of components loaded from fp32 shards (e.g. Wan), written from the live tensors after publish.
+"""Local float16 / bfloat16 cache of components loaded from fp32 shards (Wan, Z-Image, SDXL, Lumina, ...), written
+from the live tensors after publish.
Later loads read the half-size file at the same dtype (a no-op cast, bit-identical); a sampled source comparison
refuses a module changed in place. Shares ``diffusion_prequant_cache``'s root, locks, manifest and LRU.
-Kill switch: ``UNSLOTH_DIFFUSION_CAST_CACHE=0`` (``UNSLOTH_DIFFUSION_PREQUANT_CACHE=0`` disables it too).
+Kill switch: ``UNSLOTH_DIFFUSION_CAST_CACHE=0`` (``UNSLOTH_DIFFUSION_PREQUANT_CACHE=0`` disables it too);
+``UNSLOTH_DIFFUSION_CAST_CACHE=fp16`` / ``=bf16`` keeps one target dtype only; ``UNSLOTH_DIFFUSION_CAST_CACHE_IMAGE=0``
+turns the image pipeline path off.
"""
from __future__ import annotations
@@ -25,6 +28,9 @@ from typing import Any, Iterable, Optional
from . import diffusion_prequant_cache as _cache
CAST_CACHE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE"
+CAST_CACHE_IMAGE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE_IMAGE"
+_OFF = ("0", "off", "false", "no", "none")
+_HALF = {"float16": "float16", "fp16": "float16", "bfloat16": "bfloat16", "bf16": "bfloat16"}
CAST_LAYOUT_VERSION = 1
CAST_META_KEY = "unsloth_cast_component"
_LIBRARIES = ("diffusers", "transformers")
@@ -40,11 +46,21 @@ _ONLOAD_WAIT_SECONDS = 10.0
def enabled() -> bool:
raw = (os.environ.get(CAST_CACHE_ENV) or "").strip().lower()
- return raw not in ("0", "off", "false", "no", "none") and _cache.enabled()
+ return raw not in _OFF and _cache.enabled()
-def _is_fp16(dtype: Any) -> bool:
- return str(dtype) in ("torch.float16", "float16")
+def image_enabled() -> bool:
+ raw = (os.environ.get(CAST_CACHE_IMAGE_ENV) or "").strip().lower()
+ return raw not in _OFF and enabled()
+
+
+def _target_dtypes() -> tuple:
+ raw = (os.environ.get(CAST_CACHE_ENV) or "").strip().lower()
+ return (_HALF[raw],) if raw in _HALF else ("float16", "bfloat16")
+
+
+def _is_half(dtype: Any) -> bool:
+ return str(dtype).replace("torch.", "") in _target_dtypes()
def _component_dtype(torch_dtype: Any, component: str) -> Any:
@@ -74,6 +90,7 @@ class CastCandidate:
key: str
parts: dict
names: list = field(default_factory = list)
+ nbytes: int = 0
def _read_json(path: Optional[str]) -> Optional[dict]:
@@ -119,20 +136,54 @@ def _shard_header(path: str) -> Optional[dict]:
return header
-def _all_wide(shards: list) -> bool:
- """Every floating tensor of every shard is wider than float16 (the only case the cast halves)."""
+def _cast_bytes(shards: list) -> Optional[int]:
+ """16-bit bytes if every floating tensor is fp32 / fp64, else None. An estimate: fp32-kept modules count at 16 bits."""
seen = False
+ total = 0
for shard in shards:
header = _shard_header(shard)
if header is None:
- return False
+ return None
for entry in header.values():
- tag = entry.get("dtype") if isinstance(entry, dict) else None
+ if not isinstance(entry, dict):
+ continue
+ tag = entry.get("dtype")
+ try:
+ begin, end = (int(v) for v in entry.get("data_offsets") or (0, 0))
+ except (TypeError, ValueError):
+ return None
if tag in _FLOAT_TAGS:
if tag not in _WIDE_TAGS:
- return False
+ return None
seen = True
- return seen
+ total += (end - begin) // (4 if tag == "F32" else 8) * 2
+ else:
+ total += end - begin
+ return total if seen else None
+
+
+def _all_wide(shards: list) -> bool:
+ return _cast_bytes(shards) is not None
+
+
+def _fit_budget(cands: list, logger: Any = None) -> list:
+ """A set over the LRU budget would evict its own entries every load. Greedy largest-first, so it is deterministic."""
+ budget = _cache.max_cache_bytes()
+ if budget is None:
+ return cands
+ keep, used = set(), 0
+ for cand in sorted(cands, key = lambda c: (-c.nbytes, c.component)):
+ if used + cand.nbytes > budget:
+ _log(
+ logger,
+ "info",
+ f"{cand.component}: not cached ({cand.nbytes / 2**30:.1f} GiB would not fit the "
+ f"{budget / 2**30:.0f} GiB cache budget beside this load's other components)",
+ )
+ continue
+ used += cand.nbytes
+ keep.add(cand.component)
+ return [c for c in cands if c.component in keep]
def _load_class(library: str, class_name: str) -> Any:
@@ -156,7 +207,8 @@ def candidates(
base_repo: Optional[str] = None,
logger: Any = None,
) -> list:
- """Components at ``source`` that load fp32 shards at float16, with cache keys. Local reads only; never raises."""
+ """Components at ``source`` that load fp32 shards at float16 / bfloat16, with cache keys. Local reads only; never
+ raises."""
if not enabled() or not source:
return []
out: list = []
@@ -177,13 +229,14 @@ def candidates(
if library not in _LIBRARIES or not isinstance(class_name, str):
continue
dtype = _component_dtype(torch_dtype, component)
- if not _is_fp16(dtype):
+ if not _is_half(dtype):
continue
cls = _load_class(library, class_name)
if cls is None or not isinstance(cls, type) or not issubclass(cls, torch.nn.Module):
continue
shards = _shards_of(source, component, _STEMS[library], cache_dir, revision)
- if not shards or not _all_wide(shards):
+ nbytes = _cast_bytes(shards) if shards else None
+ if nbytes is None:
continue
config_path = _cache._local_file(
source, f"{component}/config.json", cache_dir, revision
@@ -222,12 +275,48 @@ def candidates(
config_path = str(config_path),
key = _cache.cache_key(parts),
parts = parts,
+ nbytes = nbytes,
)
)
except Exception as exc: # noqa: BLE001
_log(logger, "warning", f"candidates skipped ({type(exc).__name__}: {str(exc)[:200]})")
return []
- return out
+ return _fit_budget(out, logger)
+
+
+def cached_components(cands: list) -> list:
+ try:
+ root = _cache.cache_root()
+ return [c.component for c in cands or () if (root / c.key / _cache.MANIFEST_NAME).is_file()]
+ except Exception: # noqa: BLE001
+ return []
+
+
+def inject(
+ pipe_kwargs: dict,
+ cands: list,
+ *,
+ logger: Any = None,
+ cancel_check: Optional[Any] = None,
+ label: str = "diffusion",
+) -> list:
+ """Returns the misses. Never raises, except a cancel from ``cancel_check``."""
+ try:
+ cands = [c for c in cands or () if c.component not in pipe_kwargs]
+ if not cands:
+ return []
+ hits = read_all(cands, logger = logger, cancel_check = cancel_check)
+ pipe_kwargs.update(hits)
+ if hits and logger is not None:
+ logger.info(
+ "%s.cast_cache: read %s from the local cast cache", label, ", ".join(sorted(hits))
+ )
+ return [c for c in cands if c.component not in hits]
+ except Exception as exc: # noqa: BLE001 - a cache problem is a miss
+ if cancel_check is not None:
+ cancel_check()
+ _log(logger, "warning", f"inject skipped ({type(exc).__name__}: {str(exc)[:200]})")
+ return []
def _log(logger: Any, level: str, msg: str) -> None:
@@ -588,18 +677,22 @@ def save(
root = _cache.cache_root()
try:
names = _state_names(module)
- if names is None or [n for n, _d, _s in names] != [n for n, _d, _s in cand.names]:
+ live = {n: (d, s) for n, d, s in names or ()}
+ # Tensors added after the load (fuse_qkv's to_qkv) are left out: the next load re-adds them from the same
+ # weights. A renamed / removed / re-typed one (LoRA wrap, quantize, layerwise fp8 storage) refuses the write.
+ if names is None or any(n not in live for n, _d, _s in cand.names):
_log(
logger, "info", f"{cand.component}: not cached (its tensors changed after the load)"
)
return None
- if any(d != dt or s != st for (_n, d, s), (_m, dt, st) in zip(names, cand.names)):
+ if any(live[n] != (d, s) for n, d, s in cand.names):
_log(
logger,
"info",
f"{cand.component}: not cached (a tensor changed dtype or shape after the load)",
)
return None
+ names = list(cand.names)
reader = _Reader(module, check)
why = _matches_source(cand, reader)
if why is not None:
@@ -751,6 +844,8 @@ class CastWrites:
pipe = self._pipe()
module = getattr(pipe, cand.component, None) if pipe is not None else None
pipe = None
+ # torch.compile(module) wraps it; the weights (and their names) are the original's.
+ module = getattr(module, "_orig_mod", module)
if module is None:
continue
self.results[cand.component] = save(
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 3e5e5528a02..a71064319dd 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3160,7 +3160,7 @@ def _read_cast_cache(
base: Optional[str],
cancel_check: Any = None,
) -> list:
- """Inject cached float16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""
+ """Inject cached float16 / bfloat16 casts into ``pipe_kwargs``; returns the unserved candidates. Never raises."""
try:
from .diffusion_cast_cache import candidates, read_all
@@ -3181,7 +3181,7 @@ def _read_cast_cache(
pipe_kwargs.update(hits)
if hits:
logger.info(
- "video.cast_cache: read %s from the local float16 cast cache",
+ "video.cast_cache: read %s from the local cast cache",
", ".join(sorted(hits)),
)
return [c for c in cands if c.component not in hits]
diff --git a/studio/backend/tests/test_diffusion_cast_cache.py b/studio/backend/tests/test_diffusion_cast_cache.py
index 6bed8fd7ba2..43efb39fa5f 100644
--- a/studio/backend/tests/test_diffusion_cast_cache.py
+++ b/studio/backend/tests/test_diffusion_cast_cache.py
@@ -143,9 +143,8 @@ def test_candidates_are_fp32_shards_loaded_at_fp16(pipe_dir):
assert cands["transformer"].key != cands["text_encoder"].key
[email protected]("dtype", [torch.bfloat16, torch.float32])
-def test_no_candidates_outside_float16(pipe_dir, dtype):
- assert CC.candidates(pipe_dir, torch_dtype = dtype) == []
+def test_no_candidates_at_float32(pipe_dir):
+ assert CC.candidates(pipe_dir, torch_dtype = torch.float32) == []
def test_kill_switches(pipe_dir, monkeypatch):
diff --git a/studio/backend/tests/test_diffusion_cast_cache_bf16_image.py b/studio/backend/tests/test_diffusion_cast_cache_bf16_image.py
new file mode 100644
index 00000000000..cb8fa947667
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_cast_cache_bf16_image.py
@@ -0,0 +1,273 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Cast cache beyond video float16: bfloat16 targets, the image pipeline wiring, the per-load budget, live-module
+tolerances (a compiled wrapper, tensors added after the load) and the per-dtype / per-path switches."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+transformers = pytest.importorskip("transformers")
+pytest.importorskip("safetensors")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion_cast_cache as CC # noqa: E402
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+
+BF16 = torch.bfloat16
+HALF = (torch.float16, torch.bfloat16)
+IMAGE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE_IMAGE"
+
+
+def _tiny_zimage():
+ torch.manual_seed(0)
+ return diffusers.ZImageTransformer2DModel(
+ all_patch_size = (2,),
+ all_f_patch_size = (1,),
+ in_channels = 4,
+ dim = 32,
+ n_layers = 2,
+ n_refiner_layers = 1,
+ n_heads = 2,
+ n_kv_heads = 2,
+ cap_feat_dim = 16,
+ axes_dims = [4, 6, 6],
+ axes_lens = [64, 32, 32],
+ ).to(torch.float32)
+
+
+def _tiny_vae():
+ torch.manual_seed(1)
+ return diffusers.AutoencoderKL(
+ in_channels = 3,
+ out_channels = 3,
+ down_block_types = ("DownEncoderBlock2D",),
+ up_block_types = ("UpDecoderBlock2D",),
+ block_out_channels = (8,),
+ latent_channels = 4,
+ norm_num_groups = 4,
+ ).to(torch.float32)
+
+
+def _tiny_umt5():
+ torch.manual_seed(2)
+ config = transformers.UMT5Config(
+ vocab_size = 64,
+ d_model = 32,
+ d_kv = 8,
+ d_ff = 64,
+ num_layers = 2,
+ num_heads = 4,
+ relative_attention_num_buckets = 8,
+ )
+ return transformers.UMT5EncoderModel(config).to(torch.float32)
+
+
[email protected](autouse = True)
+def cache_root(tmp_path, monkeypatch):
+ root = tmp_path / "cache"
+ monkeypatch.setenv(C.ENV_DIR, str(root))
+ for env in (CC.CAST_CACHE_ENV, IMAGE_ENV, C.ENV_MODE, C.ENV_MAX_GB):
+ monkeypatch.delenv(env, raising = False)
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ with C._lock:
+ C._REJECTED.clear()
+ return root
+
+
+def image_dir(tmp_path):
+ root = tmp_path / "Z-Image-tiny"
+ _tiny_zimage().save_pretrained(root / "transformer", max_shard_size = "40KB")
+ _tiny_vae().save_pretrained(root / "vae")
+ _tiny_umt5().save_pretrained(root / "text_encoder")
+ (root / "model_index.json").write_text(
+ json.dumps(
+ {
+ "_class_name": "ZImagePipeline",
+ "transformer": ["diffusers", "ZImageTransformer2DModel"],
+ "vae": ["diffusers", "AutoencoderKL"],
+ "text_encoder": ["transformers", "UMT5EncoderModel"],
+ "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler"],
+ }
+ )
+ )
+ return root
+
+
+def _by_name(cands):
+ return {c.component: c for c in cands}
+
+
+def _dense(cand):
+ kwargs = CC._load_kwargs(cand)
+ kwargs.pop("output_loading_info")
+ return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)
+
+
+class _Pipe:
+ pass
+
+
+def _write(pipe, cands):
+ writes = CC.defer_writes(pipe, cands)
+ assert writes is not None and writes.start()
+ assert writes.wait(60)
+ return writes.results
+
+
+def _bits(t):
+ return t.reshape(-1).view(torch.uint8) if t.numel() else t
+
+
+def _identical(a, b):
+ sa, sb = a.state_dict(), b.state_dict()
+ assert sa.keys() == sb.keys()
+ for k in sa:
+ assert sa[k].dtype == sb[k].dtype, k
+ assert torch.equal(_bits(sa[k]), _bits(sb[k])), k
+
+
[email protected]("dtype", HALF)
+def test_every_fp32_component_is_a_candidate_at_both_half_dtypes(image_dir, dtype):
+ cands = _by_name(CC.candidates(image_dir, torch_dtype = dtype))
+ assert set(cands) == {"transformer", "vae", "text_encoder"}
+ assert {c.parts["dtype"] for c in cands.values()} == {str(dtype).replace("torch.", "")}
+ assert all(c.nbytes > 0 for c in cands.values())
+
+
+def test_fp16_and_bf16_entries_never_share_a_key(image_dir):
+ a = _by_name(CC.candidates(image_dir, torch_dtype = torch.float16))
+ b = _by_name(CC.candidates(image_dir, torch_dtype = BF16))
+ assert all(a[k].key != b[k].key for k in a)
+
+
[email protected]("dtype", HALF)
[email protected]("component", ["transformer", "vae", "text_encoder"])
+def test_round_trip_is_bit_identical(image_dir, dtype, component):
+ cand = _by_name(CC.candidates(image_dir, torch_dtype = dtype))[component]
+ assert CC.read(cand) is None
+ dense = _dense(cand)
+ pipe = _Pipe()
+ setattr(pipe, component, dense)
+ path = _write(pipe, [cand])[component]
+ assert path is not None and C.path_in_cache(path)
+ tag = {torch.float16: "F16", BF16: "BF16"}[dtype]
+ assert tag in {e["dtype"] for e in CC._shard_header(path).values()}
+ fresh = _by_name(CC.candidates(image_dir, torch_dtype = dtype))[component]
+ hit = CC.read(fresh)
+ assert hit is not None
+ _identical(dense, hit)
+
+
+def test_dtype_switch_keeps_one_target(image_dir, monkeypatch):
+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "fp16")
+ assert CC.candidates(image_dir, torch_dtype = BF16) == []
+ assert CC.candidates(image_dir, torch_dtype = torch.float16)
+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "bf16")
+ assert CC.candidates(image_dir, torch_dtype = torch.float16) == []
+ assert CC.candidates(image_dir, torch_dtype = BF16)
+ monkeypatch.setenv(CC.CAST_CACHE_ENV, "0")
+ assert CC.candidates(image_dir, torch_dtype = BF16) == []
+
+
+def test_load_set_fits_the_budget(image_dir, monkeypatch):
+ sizes = {c.component: c.nbytes for c in CC.candidates(image_dir, torch_dtype = BF16)}
+ # room for the transformer alone: the others never get an entry they would evict on every load
+ monkeypatch.setenv(C.ENV_MAX_GB, str((sizes["transformer"] + 1) / 2**30))
+ names = [c.component for c in CC.candidates(image_dir, torch_dtype = BF16)]
+ assert "transformer" in names and sum(sizes[n] for n in names) <= sizes["transformer"] + 1
+
+
+def test_compiled_wrapper_and_added_tensors_still_write_the_load(image_dir):
+ cand = _by_name(CC.candidates(image_dir, torch_dtype = BF16))["transformer"]
+ dense = _dense(cand)
+ reference = {k: v.clone() for k, v in dense.state_dict().items()}
+ pipe = _Pipe()
+ pipe.transformer = dense
+ writes = CC.defer_writes(pipe, [cand])
+ dense.register_parameter("extra_fused", torch.nn.Parameter(torch.zeros(3, dtype = BF16)))
+ pipe.transformer = torch.compile(dense)
+ assert writes.start() and writes.wait(60)
+ assert writes.results["transformer"] is not None
+ hit = CC.read(cand)
+ assert set(hit.state_dict()) == set(reference)
+ for k, v in hit.state_dict().items():
+ assert torch.equal(_bits(v), _bits(reference[k])), k
+
+
+def test_renamed_or_retyped_tensor_still_refuses(image_dir):
+ cand = _by_name(CC.candidates(image_dir, torch_dtype = BF16))["vae"]
+ dense = _dense(cand)
+ pipe = _Pipe()
+ pipe.vae = dense
+ writes = CC.defer_writes(pipe, [cand])
+ dense.to(
+ torch.float32
+ ) # an upcast after the load (or fp8 layerwise storage) is not what the load produced
+ assert writes.start() and writes.wait(60)
+ assert writes.results["vae"] is None
+
+
+def test_image_helpers_inject_hits_and_skip_the_prefetch(image_dir, monkeypatch):
+ from core.inference import diffusion as D
+
+ fams = {f.name: f for f in D._IMAGE_REGISTRY}
+ fam = fams["z-image"]
+ cands = D._image_cast_candidates(fam, str(image_dir), dtype = BF16, base = None)
+ assert {c.component for c in cands} == {"transformer", "vae", "text_encoder"}
+ assert D._image_cast_cached(cands, None) == []
+ tr = _by_name(cands)["transformer"]
+ pipe = _Pipe()
+ pipe.transformer = _dense(tr)
+ assert _write(pipe, [tr])["transformer"] is not None
+ cands = D._image_cast_candidates(fam, str(image_dir), dtype = BF16, base = None)
+ assert D._image_cast_cached(cands, None) == ["transformer"]
+ assert D._image_cast_cached(cands, "fp16") == [] # twins read: the prefetch must read them
+ pipe_kwargs = {"torch_dtype": BF16, "local_files_only": True}
+ missed = CC.inject(pipe_kwargs, cands)
+ assert sorted(c.component for c in missed) == ["text_encoder", "vae"]
+ _identical(pipe_kwargs["transformer"], pipe.transformer)
+ # the per-path switch, and families assembled per component, are never wired
+ monkeypatch.setenv(IMAGE_ENV, "0")
+ assert D._image_cast_candidates(fam, str(image_dir), dtype = BF16, base = None) == []
+ monkeypatch.delenv(IMAGE_ENV)
+ for name in (D.KREA2_FAMILY_NAME, D.MOSS_IMAGE_FAMILY_NAME, "flux.3-klein"):
+ assert D._image_cast_candidates(fams[name], str(image_dir), dtype = BF16, base = None) == []
+
+
+def test_video_reader_serves_bf16(image_dir):
+ from core.inference import video as V
+
+ cand = _by_name(CC.candidates(image_dir, torch_dtype = BF16))["text_encoder"]
+ pipe = _Pipe()
+ pipe.text_encoder = _dense(cand)
+ assert _write(pipe, [cand])["text_encoder"] is not None
+ pipe_kwargs = {"torch_dtype": {"vae": torch.float32, "default": BF16}, "local_files_only": True}
+ missed = V._read_cast_cache(pipe_kwargs, source = str(image_dir), revision = None, base = None)
+ assert sorted(c.component for c in missed) == ["transformer"]
+ _identical(pipe_kwargs["text_encoder"], pipe.text_encoder)
+
+
+def test_pipeline_from_injected_components_matches_the_dense_load(image_dir):
+ """The whole assembly: a pipeline loaded with the hits equals the dense bf16 pipeline load bit for bit."""
+ cands = CC.candidates(image_dir, torch_dtype = BF16)
+ dense = {c.component: _dense(c) for c in cands}
+ pipe = _Pipe()
+ for name, module in dense.items():
+ setattr(pipe, name, module)
+ assert all(_write(pipe, cands).values())
+ kwargs = {}
+ assert CC.inject(kwargs, CC.candidates(image_dir, torch_dtype = BF16)) == []
+ for name, module in dense.items():
+ _identical(module, kwargs[name])