Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: cast-cache writes read block-streamed tensors from their host copy (#15357)

Daniel Hana2db3077fb86
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.py
index 49e69ee0aba..c85ec9fa540 100644
--- a/studio/backend/core/inference/diffusion_cast_cache.py
+++ b/studio/backend/core/inference/diffusion_cast_cache.py
@@ -31,7 +31,8 @@ CAST_CACHE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE"
CAST_CACHE_IMAGE_ENV = "UNSLOTH_DIFFUSION_CAST_CACHE_IMAGE"
_OFF = ("0", "off", "false", "no", "none")
_HALF = {"float16": "float16", "fp16": "float16", "bfloat16": "bfloat16", "bf16": "bfloat16"}
-CAST_LAYOUT_VERSION = 1
+# 2: layout-1 entries written under block streaming may hold tensors read before their upload landed.
+CAST_LAYOUT_VERSION = 2
CAST_META_KEY = "unsloth_cast_component"
_LIBRARIES = ("diffusers", "transformers")
_STEMS = {"diffusers": "diffusion_pytorch_model", "transformers": "model"}
@@ -465,7 +466,8 @@ def _tied_keys(module: Any) -> set:
return keys
-def _resolve(module: Any, name: str) -> Any:
+def _resolve_ref(module: Any, name: str) -> Any:
+ """The Parameter / buffer itself (offload groups key host copies by it)."""
parent_name, _, leaf = name.rpartition(".")
parent = module.get_submodule(parent_name) if parent_name else module
tensor = (
@@ -473,9 +475,31 @@ def _resolve(module: Any, name: str) -> Any:
)
if tensor is None:
raise KeyError(name)
+ return tensor
+
+
+def _resolve(module: Any, name: str) -> Any:
+ tensor = _resolve_ref(module, name)
return tensor.data if hasattr(tensor, "data") else tensor
+def _offload_groups(module: Any) -> list:
+ out: list = []
+ seen: set = set()
+ for m in module.modules():
+ hooks = getattr(getattr(m, "_diffusers_hook", None), "hooks", None)
+ if not isinstance(hooks, dict):
+ continue
+ for hook in list(hooks.values()):
+ group = getattr(hook, "group", None)
+ if group is None or id(group) in seen:
+ continue
+ seen.add(id(group))
+ if isinstance(getattr(group, "cpu_param_dict", None), dict):
+ out.append(group)
+ return out
+
+
def _state_names(module: Any) -> Optional[list]:
"""``[(name, dtype, shape)]`` to write: the state dict minus tied aliases ``from_pretrained`` restores."""
import torch
@@ -525,19 +549,40 @@ def _hooked(module: Any) -> bool:
class _Reader:
- """Host copies of one component's live tensors; device copies are read on a side stream after one device
- synchronise (an upload may be queued on another stream), never behind the render on the default stream."""
+ """Host copies of one component's live tensors, read while the pipeline may be rendering.
+
+ Streamed tensors come from their offload group's host copy: device ``.data`` is re-pointed before the prefetch
+ lands, so reading it races the copy stream (NaN entries). Other device tensors are read on a side stream after
+ a per-read device synchronise, never behind the render on the default stream."""
def __init__(self, module: Any, check: Any) -> None:
self.module = module
self.check = check
+ self.groups = _offload_groups(module)
self.wait = _ONLOAD_WAIT_SECONDS if _hooked(module) else 0.0
- self.synced = False
self.stream = None
+ def _host_copy(self, ref: Any) -> Any:
+ for group in self.groups:
+ try:
+ src = group.cpu_param_dict.get(ref)
+ except Exception: # noqa: BLE001
+ src = None
+ if (
+ src is not None
+ and getattr(src, "device", None) is not None
+ and src.device.type == "cpu"
+ ):
+ return src
+ return None
+
def tensor(self, name: str) -> Any:
while True:
- tensor = _resolve(self.module, name)
+ ref = _resolve_ref(self.module, name)
+ host = self._host_copy(ref)
+ if host is not None:
+ return host.data if hasattr(host, "data") else host
+ tensor = ref.data if hasattr(ref, "data") else ref
if tensor.device.type == "cpu":
return tensor
if self.wait <= 0:
@@ -549,9 +594,8 @@ class _Reader:
if tensor.device.type != "cuda":
return tensor.to("cpu")
- if not self.synced:
- torch.cuda.synchronize(tensor.device)
- self.synced = True
+ torch.cuda.synchronize(tensor.device)
+ if self.stream is None:
self.stream = torch.cuda.Stream(device = tensor.device)
with torch.cuda.stream(self.stream):
host = tensor.to("cpu", non_blocking = False)
diff --git a/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py b/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py
new file mode 100644
index 00000000000..51050324a63
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py
@@ -0,0 +1,206 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A cast-cache write that runs while the published pipeline renders reads block-streamed tensors from their offload
+group's host copy, never from a device destination whose upload has not landed yet."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+pytest.importorskip("transformers")
+pytest.importorskip("safetensors")
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion_cast_cache as CC # noqa: E402
+from core.inference import diffusion_prequant_cache as C # noqa: E402
+
+FP16_DEFAULT = {"vae": torch.float32, "default": torch.float16}
+
+
[email protected](autouse = True)
+def cache_root(tmp_path, monkeypatch):
+ root = tmp_path / "cache"
+ monkeypatch.setenv(C.ENV_DIR, str(root))
+ monkeypatch.delenv(CC.CAST_CACHE_ENV, raising = False)
+ monkeypatch.delenv(C.ENV_MODE, raising = False)
+ monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
+ with C._lock:
+ C._REJECTED.clear()
+ return root
+
+
+def pipe_dir(tmp_path):
+ torch.manual_seed(0)
+ model = diffusers.WanTransformer3DModel(
+ patch_size = (1, 2, 2),
+ num_attention_heads = 2,
+ attention_head_dim = 16,
+ in_channels = 4,
+ out_channels = 4,
+ text_dim = 32,
+ freq_dim = 16,
+ ffn_dim = 64,
+ num_layers = 2,
+ cross_attn_norm = True,
+ qk_norm = "rms_norm_across_heads",
+ eps = 1e-6,
+ rope_max_seq_len = 32,
+ ).to(torch.float32)
+ root = tmp_path / "Wan2.1-T2V-tiny-Diffusers"
+ model.save_pretrained(root / "transformer", max_shard_size = "60KB")
+ (root / "model_index.json").write_text(
+ json.dumps(
+ {"_class_name": "WanPipeline", "transformer": ["diffusers", "WanTransformer3DModel"]}
+ )
+ )
+ return root
+
+
+def _cand(pipe_dir):
+ return {c.component: c for c in CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT)}[
+ "transformer"
+ ]
+
+
+def _dense(cand):
+ kwargs = CC._load_kwargs(cand)
+ kwargs.pop("output_loading_info")
+ return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)
+
+
+def _bits(t):
+ return t.reshape(-1).view(torch.uint8) if t.numel() else t
+
+
+def _assert_hit_equals(cand, reference: dict):
+ hit = CC.read(cand)
+ assert hit is not None
+ for k, v in hit.state_dict().items():
+ assert v.dtype == reference[k].dtype and torch.equal(_bits(v), _bits(reference[k])), k
+
+
+def _unsampled_cast_key(cand) -> str:
+ """A float16 tensor the pre-write source comparison does not sample, so only the write itself can catch it."""
+ names = {n: d for n, d, _s in cand.names}
+ sampled = set()
+ for shard in cand.shards:
+ sampled.update(CC._sample_keys(CC._shard_header(shard) or {}, names))
+ keys = sorted(
+ n for n, d in names.items() if d == torch.float16 and n not in sampled and "weight" in n
+ )
+ assert keys
+ return keys[0]
+
+
+class _Group:
+ def __init__(self, cpu_param_dict):
+ self.cpu_param_dict = cpu_param_dict
+
+
+class _Hook:
+ def __init__(self, group):
+ self.group = group
+
+
+class _Registry:
+ def __init__(self, hook):
+ self.hooks = {"group_offloading": hook}
+
+
+def test_streamed_tensor_is_written_from_its_host_copy(pipe_dir):
+ cand = _cand(pipe_dir)
+ dense = _dense(cand)
+ reference = {k: v.clone() for k, v in dense.state_dict().items()}
+ host = {t: t.data.clone() for t in [*dense.parameters(), *dense.buffers()]}
+ dense._diffusers_hook = _Registry(_Hook(_Group(host)))
+ CC.capture(type("P", (), {"transformer": dense})(), [cand])
+ key = _unsampled_cast_key(cand)
+ live = dense.get_parameter(key)
+ live.data = torch.full_like(live.data, float("nan"))
+ assert CC.save(cand, dense) is not None
+ _assert_hit_equals(cand, reference)
+
+
+def test_unstreamed_module_still_reads_its_live_tensors(pipe_dir):
+ cand = _cand(pipe_dir)
+ dense = _dense(cand)
+ reference = {k: v.clone() for k, v in dense.state_dict().items()}
+ CC.capture(type("P", (), {"transformer": dense})(), [cand])
+ assert CC.save(cand, dense) is not None
+ _assert_hit_equals(cand, reference)
+
+
+def test_layout_version_retires_entries_written_by_the_racy_reader(pipe_dir):
+ assert CC.CAST_LAYOUT_VERSION >= 2
+ assert _cand(pipe_dir).parts["layout"] == CC.CAST_LAYOUT_VERSION
+
+
[email protected](not torch.cuda.is_available(), reason = "needs a CUDA device")
+def test_block_streamed_write_during_a_refill_is_bit_identical(pipe_dir, monkeypatch):
+ """Before every read each group is re-pointed to fresh NaN device tensors whose upload has not landed."""
+ from diffusers.hooks import apply_group_offloading
+
+ monkeypatch.setattr(CC, "_ONLOAD_WAIT_SECONDS", 0.0)
+ cand = _cand(pipe_dir)
+ dense = _dense(cand)
+ reference = {k: v.clone() for k, v in dense.state_dict().items()}
+ apply_group_offloading(
+ dense,
+ onload_device = torch.device("cuda"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ )
+ CC.capture(type("P", (), {"transformer": dense})(), [cand])
+ groups = []
+ for m in dense.modules():
+ for hook in getattr(getattr(m, "_diffusers_hook", None), "hooks", {}).values():
+ g = getattr(hook, "group", None)
+ if g is not None and all(g is not x for x in groups):
+ groups.append(g)
+ # a group holding no tensors (e.g. the unmatched-modules group) keeps an empty host copy
+ groups = [g for g in groups if g.cpu_param_dict]
+ assert groups
+
+ def onload():
+ for g in groups:
+ with torch.cuda.stream(g.stream):
+ for t, src in g.cpu_param_dict.items():
+ t.data = src.to("cuda", non_blocking = True)
+ torch.cuda.synchronize()
+
+ names = {n: d for n, d, _s in cand.names}
+ sampled = sum(len(CC._sample_keys(CC._shard_header(sh) or {}, names)) for sh in cand.shards)
+ state = {"calls": 0}
+
+ def prefetch():
+ # After the sampled reads (so the write itself is what goes wrong), as the prefetcher does: `.data`
+ # re-pointed now, its upload queued behind work that has not finished.
+ state["calls"] += 1
+ if state["calls"] <= sampled:
+ return
+ moved = {t: torch.full_like(t.data, float("nan")) for g in groups for t in g.cpu_param_dict}
+ torch.cuda.synchronize()
+ for g in groups:
+ for t in g.cpu_param_dict:
+ t.data = moved[t]
+ with torch.cuda.stream(g.stream):
+ torch.cuda._sleep(50_000_000)
+ for t, src in g.cpu_param_dict.items():
+ t.data.copy_(src, non_blocking = True)
+
+ onload()
+ assert CC.save(cand, dense, check = prefetch) is not None
+ torch.cuda.synchronize()
+ _assert_hit_equals(cand, reference)