Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cast-cache writer reads the host copy offload already keeps instead of waiting out an onload (#15698)
Daniel Hand4e56bc0faa2
diff --git a/studio/backend/core/inference/diffusion_cast_cache.py b/studio/backend/core/inference/diffusion_cast_cache.pyindex 60dfb32f0e1..46e8e15180f 100644--- a/studio/backend/core/inference/diffusion_cast_cache.py+++ b/studio/backend/core/inference/diffusion_cast_cache.py@@ -575,10 +575,41 @@ class _Reader:return srcreturn None+ def _kept_copy(self, ref: Any, name: str) -> Any:+ """While ``ref`` is onloaded: the host copy its device copy was uploaded from (top-level group or keep-cpu offload)."""+ live = ref.data if hasattr(ref, "data") else ref+ device = getattr(live, "device", None)+ if device is None or device.type == "cpu":+ return None+ src = None+ try:+ from .diffusion_memory import TOP_GROUP_HOST_ATTR, offload_host_copy+ for group in self.groups:+ top = getattr(group, TOP_GROUP_HOST_ATTR, None)+ if isinstance(top, dict) and ref in top:+ src = top.get(ref)+ break+ if src is None:+ src = offload_host_copy(self.module, name, ref)+ except Exception: # noqa: BLE001 - no kept copy: wait for the host as before+ return None+ src = src.data if hasattr(src, "data") else src+ if (+ src is None+ or getattr(src, "device", None) is None+ or src.device.type != "cpu"+ or src.dtype != live.dtype+ or tuple(src.shape) != tuple(live.shape)+ ):+ return None+ return src+def tensor(self, name: str) -> Any:while True:ref = _resolve_ref(self.module, name)host = self._host_copy(ref)+ if host is None:+ host = self._kept_copy(ref, name)if host is not None:return host.data if hasattr(host, "data") else hosttensor = ref.data if hasattr(ref, "data") else refdiff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex c3543b4fd05..671c74f19c2 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1014,6 +1014,43 @@ def _wrap_cpu_offload_hook(setattr(hook, _KEEP_ATTR, True)+def offload_host_copy(module: Any, name: str, tensor: Any) -> Any:+ """The keep-cpu host tensor for ``name`` while ``tensor`` is on the device unchanged since its onload (``_init_hook``'s+ re-point rule: same object, version and storage), else None."""+ state = getattr(module, "__dict__", {}).get(_KEEP_ATTR)+ if (+ not isinstance(state, dict)+ or getattr(getattr(tensor, "device", None), "type", "cpu") == "cpu"+ ):+ return None+ try:+ kept = (state.get("host") or {}).get(name)+ if kept is not None:+ seen = (state.get("version") or {}).get(name)+ if (+ (state.get("owner") or {}).get(name) is tensor+ and seen is not None+ and seen[0] is tensor+ and seen[1] == tensor._version+ and seen[2] == _tensor_ptr(tensor)+ ):+ return kept+ return None+ kept = (state.get("buffer_host") or {}).get(name)+ seen = (state.get("buffer_version") or {}).get(name)+ if (+ kept is not None+ and seen is not None+ and seen[0] is tensor+ and seen[1] == _buffer_version(tensor)+ and seen[2] == tensor.data_ptr()+ ):+ return kept+ except Exception: # noqa: BLE001 - a state mid-update: the caller reads the device copy+ return None+ return None++def reclaim_offload_host_memory(offload_policy: str, logger: Any = None) -> bool:"""Return unused allocator pages after whole-model CPU offload, without touching livetensors, Python GC, or device caches. Unsupported allocators and failures are non-fatal."""@@ -7094,6 +7131,8 @@ def install_group_offload_buffer_restore() -> bool:PIN_TOP_GROUP_ENV = "UNSLOTH_DIFFUSION_PIN_TOP_GROUP"+# read-only: diffusers leaves the top-level group's cpu_param_dict empty+TOP_GROUP_HOST_ATTR = "_unsloth_top_host"def _pin_top_level_group(@@ -7190,6 +7229,7 @@ def _pin_top_level_group(group.onload_ = onload_group.offload_ = offload_group._unsloth_pinned_top = True+ setattr(group, TOP_GROUP_HOST_ATTR, host)if pinned_mib is not None:pinned_mib[0] = already + need_mibif logger is not None:@@ -7270,6 +7310,7 @@ def _skip_top_level_copy_back(module: Any, logger: Any = None) -> bool:group.onload_ = onload_group.offload_ = offload_group._unsloth_top_no_copy_back = True+ setattr(group, TOP_GROUP_HOST_ATTR, host)if logger is not None:logger.info("diffusion.memory: %s top-level weights (%d MiB) not pinnable here; offload re-points to the host "diff --git a/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py b/studio/backend/tests/test_diffusion_cast_cache_streamed_write.pyindex 51050324a63..91e97ce1745 100644--- a/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py+++ b/studio/backend/tests/test_diffusion_cast_cache_streamed_write.py@@ -204,3 +204,145 @@ def test_block_streamed_write_during_a_refill_is_bit_identical(pipe_dir, monkeypassert CC.save(cand, dense, check = prefetch) is not Nonetorch.cuda.synchronize()_assert_hit_equals(cand, reference)+++needs_swap = pytest.mark.skipif(+ not hasattr(torch.utils, "swap_tensors"), reason = "needs torch.utils.swap_tensors (torch >= 2.3)"+)+++class _TopGroup:+ """diffusers' unmatched top-level group: no stream, so its cpu_param_dict stays empty."""++ def __init__(self, module):+ self.cpu_param_dict = {}+ self.stream = None+ self.offload_to_disk_path = None+ self.onload_device = torch.device("cuda")+ self.modules = [module]+ self.parameters = []+ self.buffers = []+++class _TopRegistry:+ def __init__(self, group):+ self.hooks = {"group_offloading": _Hook(group)}++ def get_hook(self, name):+ return self.hooks["group_offloading"]+++def _onload(module):+ """Onload keeping each Parameter / buffer object; meta stands in for a device copy the writer must not read."""+ twins = []+ for t in [*module.parameters(), *module.buffers()]:+ twin = torch.empty_like(t, device = "meta")+ if isinstance(t, torch.nn.Parameter):+ twin = torch.nn.Parameter(twin, requires_grad = t.requires_grad)+ torch.utils.swap_tensors(t, twin)+ twins.append((t, twin))+ return twins+++def _offload(twins):+ for t, twin in twins:+ torch.utils.swap_tensors(t, twin)+++class _CpuOffloadHook:+ """accelerate's CpuOffload without a device: pre_forward onloads, init_hook offloads."""++ twins: list = []++ def init_hook(self, mod):+ _offload(self.twins)+ self.twins = []+ return mod++ def pre_forward(self, mod, *args, **kwargs):+ self.twins = _onload(mod)+ return args, kwargs+++def _written(cand, module, path):+ import hashlib+ CC._write(str(path), cand, {"k": "v"}, hashlib.sha256(), CC._Reader(module, lambda: None))+ return path.read_bytes()+++def sleeps(monkeypatch):+ """Counted, not slept."""+ calls = []++ class _Time:+ def __getattr__(self, name):+ return getattr(__import__("time"), name)++ def sleep(self, s):+ calls.append(s)++ monkeypatch.setattr(CC, "time", _Time())+ return calls+++@needs_swap+def test_onloaded_top_level_group_is_written_from_its_kept_host_copy(pipe_dir, tmp_path, sleeps):+ from core.inference import diffusion_memory as dm++ cand = _cand(pipe_dir)+ dense = _dense(cand)+ CC.capture(type("P", (), {"transformer": dense})(), [cand])+ want = _written(cand, dense, tmp_path / "plain.safetensors")+ group = _TopGroup(dense)+ dense._diffusers_hook = _TopRegistry(group)+ assert dm._skip_top_level_copy_back(dense)+ assert group.cpu_param_dict == {}+ twins = _onload(dense)+ assert _written(cand, dense, tmp_path / "onloaded.safetensors") == want+ assert sleeps == []+ # replaced on the host while offloaded: the live tensor is read+ _offload(twins)+ live = dense.get_parameter(_unsampled_cast_key(cand))+ live.data = live.data + 1+ assert _written(cand, dense, tmp_path / "replaced.safetensors") != want+ assert sleeps == []+++@needs_swap+def test_onloaded_model_offload_is_written_from_its_kept_host_copy(pipe_dir, tmp_path, sleeps):+ from core.inference import diffusion_memory as dm++ cand = _cand(pipe_dir)+ dense = _dense(cand)+ CC.capture(type("P", (), {"transformer": dense})(), [cand])+ want = _written(cand, dense, tmp_path / "plain.safetensors")+ hook = _CpuOffloadHook()+ dense._hf_hook = hook+ dm._wrap_cpu_offload_hook(hook, dense)+ hook.pre_forward(dense)+ assert all(p.device.type == "meta" for p in dense.parameters())+ assert _written(cand, dense, tmp_path / "onloaded.safetensors") == want+ assert sleeps == []+++@needs_swap+def test_a_weight_changed_on_the_device_is_never_read_from_the_host(pipe_dir, sleeps):+ from core.inference import diffusion_memory as dm++ cand = _cand(pipe_dir)+ dense = _dense(cand)+ CC.capture(type("P", (), {"transformer": dense})(), [cand])+ hook = _CpuOffloadHook()+ dense._hf_hook = hook+ dm._wrap_cpu_offload_hook(hook, dense)+ hook.pre_forward(dense)+ key = _unsampled_cast_key(cand)+ with torch.no_grad():+ dense.get_parameter(key).add_(1) # the kept host copy is stale now+ reader = CC._Reader(dense, lambda: None)+ other = next(n for n, _d, _s in cand.names if n != key and "weight" in n)+ assert reader.tensor(other).device.type == "cpu" and sleeps == []+ with pytest.raises(NotImplementedError):+ reader.tensor(key) # waits, then reads the device copy+ assert sleeps