Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cast-cache writer reads the host copy offload already keeps instead of waiting out an onload (#15698)

Daniel Hand4e56bc0faa2
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.py
index 60dfb32f0e1..46e8e15180f 100644
--- a/studio/backend/core/inference/diffusion_cast_cache.py
+++ b/studio/backend/core/inference/diffusion_cast_cache.py
@@ -575,10 +575,41 @@ class _Reader:
return src
return None
+ def _kept_copy(self, ref: Any, name: str) -> Any:
+ """While ``ref`` is onloaded: the host copy its device copy was uploaded from (top-level group or keep-cpu offload)."""
+ live = ref.data if hasattr(ref, "data") else ref
+ device = getattr(live, "device", None)
+ if device is None or device.type == "cpu":
+ return None
+ src = None
+ try:
+ from .diffusion_memory import TOP_GROUP_HOST_ATTR, offload_host_copy
+ for group in self.groups:
+ top = getattr(group, TOP_GROUP_HOST_ATTR, None)
+ if isinstance(top, dict) and ref in top:
+ src = top.get(ref)
+ break
+ if src is None:
+ src = offload_host_copy(self.module, name, ref)
+ except Exception: # noqa: BLE001 - no kept copy: wait for the host as before
+ return None
+ src = src.data if hasattr(src, "data") else src
+ if (
+ src is None
+ or getattr(src, "device", None) is None
+ or src.device.type != "cpu"
+ or src.dtype != live.dtype
+ or tuple(src.shape) != tuple(live.shape)
+ ):
+ return None
+ return src
+
def tensor(self, name: str) -> Any:
while True:
ref = _resolve_ref(self.module, name)
host = self._host_copy(ref)
+ if host is None:
+ host = self._kept_copy(ref, name)
if host is not None:
return host.data if hasattr(host, "data") else host
tensor = ref.data if hasattr(ref, "data") else ref
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index c3543b4fd05..671c74f19c2 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1014,6 +1014,43 @@ def _wrap_cpu_offload_hook(
setattr(hook, _KEEP_ATTR, True)
+def offload_host_copy(module: Any, name: str, tensor: Any) -> Any:
+ """The keep-cpu host tensor for ``name`` while ``tensor`` is on the device unchanged since its onload (``_init_hook``'s
+ re-point rule: same object, version and storage), else None."""
+ state = getattr(module, "__dict__", {}).get(_KEEP_ATTR)
+ if (
+ not isinstance(state, dict)
+ or getattr(getattr(tensor, "device", None), "type", "cpu") == "cpu"
+ ):
+ return None
+ try:
+ kept = (state.get("host") or {}).get(name)
+ if kept is not None:
+ seen = (state.get("version") or {}).get(name)
+ if (
+ (state.get("owner") or {}).get(name) is tensor
+ and seen is not None
+ and seen[0] is tensor
+ and seen[1] == tensor._version
+ and seen[2] == _tensor_ptr(tensor)
+ ):
+ return kept
+ return None
+ kept = (state.get("buffer_host") or {}).get(name)
+ seen = (state.get("buffer_version") or {}).get(name)
+ if (
+ kept is not None
+ and seen is not None
+ and seen[0] is tensor
+ and seen[1] == _buffer_version(tensor)
+ and seen[2] == tensor.data_ptr()
+ ):
+ return kept
+ except Exception: # noqa: BLE001 - a state mid-update: the caller reads the device copy
+ return None
+ return None
+
+
def reclaim_offload_host_memory(offload_policy: str, logger: Any = None) -> bool:
"""Return unused allocator pages after whole-model CPU offload, without touching live
tensors, Python GC, or device caches. Unsupported allocators and failures are non-fatal."""
@@ -7094,6 +7131,8 @@ def install_group_offload_buffer_restore() -> bool:
PIN_TOP_GROUP_ENV = "UNSLOTH_DIFFUSION_PIN_TOP_GROUP"
+# read-only: diffusers leaves the top-level group's cpu_param_dict empty
+TOP_GROUP_HOST_ATTR = "_unsloth_top_host"
def _pin_top_level_group(
@@ -7190,6 +7229,7 @@ def _pin_top_level_group(
group.onload_ = onload_
group.offload_ = offload_
group._unsloth_pinned_top = True
+ setattr(group, TOP_GROUP_HOST_ATTR, host)
if pinned_mib is not None:
pinned_mib[0] = already + need_mib
if logger is not None:
@@ -7270,6 +7310,7 @@ def _skip_top_level_copy_back(module: Any, logger: Any = None) -> bool:
group.onload_ = onload_
group.offload_ = offload_
group._unsloth_top_no_copy_back = True
+ setattr(group, TOP_GROUP_HOST_ATTR, host)
if logger is not None:
logger.info(
"diffusion.memory: %s top-level weights (%d MiB) not pinnable here; offload re-points to the host "
diff --git a/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py b/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py
index 51050324a63..91e97ce1745 100644
--- a/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py
+++ b/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py
@@ -204,3 +204,145 @@ def test_block_streamed_write_during_a_refill_is_bit_identical(pipe_dir, monkeyp
assert CC.save(cand, dense, check = prefetch) is not None
torch.cuda.synchronize()
_assert_hit_equals(cand, reference)
+
+
+needs_swap = pytest.mark.skipif(
+ not hasattr(torch.utils, "swap_tensors"), reason = "needs torch.utils.swap_tensors (torch >= 2.3)"
+)
+
+
+class _TopGroup:
+ """diffusers' unmatched top-level group: no stream, so its cpu_param_dict stays empty."""
+
+ def __init__(self, module):
+ self.cpu_param_dict = {}
+ self.stream = None
+ self.offload_to_disk_path = None
+ self.onload_device = torch.device("cuda")
+ self.modules = [module]
+ self.parameters = []
+ self.buffers = []
+
+
+class _TopRegistry:
+ def __init__(self, group):
+ self.hooks = {"group_offloading": _Hook(group)}
+
+ def get_hook(self, name):
+ return self.hooks["group_offloading"]
+
+
+def _onload(module):
+ """Onload keeping each Parameter / buffer object; meta stands in for a device copy the writer must not read."""
+ twins = []
+ for t in [*module.parameters(), *module.buffers()]:
+ twin = torch.empty_like(t, device = "meta")
+ if isinstance(t, torch.nn.Parameter):
+ twin = torch.nn.Parameter(twin, requires_grad = t.requires_grad)
+ torch.utils.swap_tensors(t, twin)
+ twins.append((t, twin))
+ return twins
+
+
+def _offload(twins):
+ for t, twin in twins:
+ torch.utils.swap_tensors(t, twin)
+
+
+class _CpuOffloadHook:
+ """accelerate's CpuOffload without a device: pre_forward onloads, init_hook offloads."""
+
+ twins: list = []
+
+ def init_hook(self, mod):
+ _offload(self.twins)
+ self.twins = []
+ return mod
+
+ def pre_forward(self, mod, *args, **kwargs):
+ self.twins = _onload(mod)
+ return args, kwargs
+
+
+def _written(cand, module, path):
+ import hashlib
+ CC._write(str(path), cand, {"k": "v"}, hashlib.sha256(), CC._Reader(module, lambda: None))
+ return path.read_bytes()
+
+
+def sleeps(monkeypatch):
+ """Counted, not slept."""
+ calls = []
+
+ class _Time:
+ def __getattr__(self, name):
+ return getattr(__import__("time"), name)
+
+ def sleep(self, s):
+ calls.append(s)
+
+ monkeypatch.setattr(CC, "time", _Time())
+ return calls
+
+
+@needs_swap
+def test_onloaded_top_level_group_is_written_from_its_kept_host_copy(pipe_dir, tmp_path, sleeps):
+ from core.inference import diffusion_memory as dm
+
+ cand = _cand(pipe_dir)
+ dense = _dense(cand)
+ CC.capture(type("P", (), {"transformer": dense})(), [cand])
+ want = _written(cand, dense, tmp_path / "plain.safetensors")
+ group = _TopGroup(dense)
+ dense._diffusers_hook = _TopRegistry(group)
+ assert dm._skip_top_level_copy_back(dense)
+ assert group.cpu_param_dict == {}
+ twins = _onload(dense)
+ assert _written(cand, dense, tmp_path / "onloaded.safetensors") == want
+ assert sleeps == []
+ # replaced on the host while offloaded: the live tensor is read
+ _offload(twins)
+ live = dense.get_parameter(_unsampled_cast_key(cand))
+ live.data = live.data + 1
+ assert _written(cand, dense, tmp_path / "replaced.safetensors") != want
+ assert sleeps == []
+
+
+@needs_swap
+def test_onloaded_model_offload_is_written_from_its_kept_host_copy(pipe_dir, tmp_path, sleeps):
+ from core.inference import diffusion_memory as dm
+
+ cand = _cand(pipe_dir)
+ dense = _dense(cand)
+ CC.capture(type("P", (), {"transformer": dense})(), [cand])
+ want = _written(cand, dense, tmp_path / "plain.safetensors")
+ hook = _CpuOffloadHook()
+ dense._hf_hook = hook
+ dm._wrap_cpu_offload_hook(hook, dense)
+ hook.pre_forward(dense)
+ assert all(p.device.type == "meta" for p in dense.parameters())
+ assert _written(cand, dense, tmp_path / "onloaded.safetensors") == want
+ assert sleeps == []
+
+
+@needs_swap
+def test_a_weight_changed_on_the_device_is_never_read_from_the_host(pipe_dir, sleeps):
+ from core.inference import diffusion_memory as dm
+
+ cand = _cand(pipe_dir)
+ dense = _dense(cand)
+ CC.capture(type("P", (), {"transformer": dense})(), [cand])
+ hook = _CpuOffloadHook()
+ dense._hf_hook = hook
+ dm._wrap_cpu_offload_hook(hook, dense)
+ hook.pre_forward(dense)
+ key = _unsampled_cast_key(cand)
+ with torch.no_grad():
+ dense.get_parameter(key).add_(1) # the kept host copy is stale now
+ reader = CC._Reader(dense, lambda: None)
+ other = next(n for n, _d, _s in cand.names if n != key and "weight" in n)
+ assert reader.tensor(other).device.type == "cpu" and sleeps == []
+ with pytest.raises(NotImplementedError):
+ reader.tensor(key) # waits, then reads the device copy
+ assert sleeps