Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: whole-model offload points weights back at their host tensors instead of copying them (#11764)

Daniel Han28d5e76de6d8
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index e5842da5e11..a510504e61c 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -114,6 +114,141 @@ def _resolve_host_memory_reclaimer() -> Optional[Callable[[], None]]:
return reclaim
+OFFLOAD_KEEP_CPU_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_KEEP_CPU"
+_KEEP_ATTR = "_unsloth_offload_keep_cpu"
+
+
+def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:
+ """Offload by re-pointing unmodified weights at their host tensors; enable_model_cpu_offload is wrapped since diffusers rebuilds hooks."""
+ if (os.environ.get(OFFLOAD_KEEP_CPU_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
+ return 0
+ try:
+ from accelerate.hooks import CpuOffload
+ except Exception: # noqa: BLE001 - no accelerate, no offload hooks to wrap
+ return 0
+ enable = getattr(pipe, "enable_model_cpu_offload", None)
+ if callable(enable) and not getattr(enable, _KEEP_ATTR, False):
+
+ def _enable(*args: Any, **kwargs: Any) -> Any:
+ out = enable(*args, **kwargs)
+ _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)
+ return out
+
+ setattr(_enable, _KEEP_ATTR, True)
+ try:
+ pipe.enable_model_cpu_offload = _enable
+ except Exception: # noqa: BLE001 - a pipeline refusing the attribute keeps the stock hooks
+ return 0
+ return _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)
+
+
+def _keepable(param: Any) -> bool:
+ import torch
+ return type(param.data) is torch.Tensor
+
+
+def _wrap_cpu_offload_hooks(
+ pipe: Any,
+ hook_cls: type,
+ logger: Any = None,
+) -> int:
+ wrapped = 0
+ components = getattr(pipe, "components", None) or {}
+ for module in components.values():
+ hook = getattr(module, "_hf_hook", None)
+ if not isinstance(hook, hook_cls) or getattr(hook, _KEEP_ATTR, False):
+ continue
+ try:
+ _wrap_cpu_offload_hook(hook, module)
+ wrapped += 1
+ except Exception as exc: # noqa: BLE001 - that module keeps the stock copy
+ if logger is not None:
+ logger.debug(
+ "diffusion.memory: keep-cpu offload not applied to %s: %s",
+ type(module).__name__,
+ exc,
+ )
+ return wrapped
+
+
+def _wrap_cpu_offload_hook(hook: Any, module: Any) -> None:
+ # Keyed by name: a move can hand the module new Parameter objects, and a stale key must miss, not alias.
+ state = module.__dict__.get(_KEEP_ATTR)
+ if state is None:
+ state = {"host": {}, "owner": {}, "version": {}}
+ module.__dict__[_KEEP_ATTR] = state
+ state.setdefault("owner", {})
+ host, owner, version = state["host"], state["owner"], state["version"]
+
+ def _capture(mod: Any) -> None:
+ host.clear()
+ owner.clear()
+ for name, p in mod.named_parameters():
+ if p.device.type == "cpu" and _keepable(p):
+ host[name] = p.data
+ owner[name] = p
+
+ _capture(module)
+ version.clear()
+ init_hook, pre_forward = hook.init_hook, hook.pre_forward
+
+ def _init_hook(mod: Any) -> Any:
+ for name, p in mod.named_parameters():
+ kept = host.get(name)
+ seen = version.get(name)
+ # Identity and data_ptr too: a replacement can match the version, and `p.data = ...` does not bump it.
+ if (
+ kept is None
+ or p.device.type == "cpu"
+ or seen is None
+ or seen[0] is not p
+ or seen[1] != p._version
+ or seen[2] != p.data_ptr()
+ ):
+ continue
+ try:
+ p.data = kept
+ except Exception: # noqa: BLE001 - incompatible tensor types: the stock copy below handles it
+ pass
+ out = init_hook(mod)
+ _capture(mod)
+ version.clear()
+ return out
+
+ def _pre_forward(mod: Any, *args: Any, **kwargs: Any) -> Any:
+ # `host` gate: an all-subclass module (GGUF, torchao) never fills `version`, so would rescan every forward.
+ onload = not version and bool(host)
+ if onload:
+ for name, p in mod.named_parameters():
+ kept = host.get(name)
+ if kept is not None and (
+ owner.get(name) is not p
+ or p.device.type != "cpu"
+ or p.data.data_ptr() != kept.data_ptr()
+ ):
+ host.pop(name, None)
+ owner.pop(name, None)
+ out = pre_forward(mod, *args, **kwargs)
+ if onload:
+ for name, p in mod.named_parameters():
+ if name in host and owner.get(name) is p and p.device.type != "cpu":
+ version[name] = (p, p._version, p.data_ptr())
+ return out
+
+ try:
+ import torch
+
+ # Same as accelerate's own pre_forward: a compiled forward must not trace the hook.
+ _init_hook, _pre_forward = (
+ torch.compiler.disable(_init_hook),
+ torch.compiler.disable(_pre_forward),
+ )
+ except Exception: # noqa: BLE001 - older torch: the hook runs outside any compiled region anyway
+ pass
+ hook.init_hook, hook.pre_forward = _init_hook, _pre_forward
+ setattr(hook, _KEEP_ATTR, True)
+
+
def reclaim_offload_host_memory(offload_policy: str, logger: Any = None) -> bool:
"""Return unused allocator pages after whole-model CPU offload, without touching live
tensors, Python GC, or device caches. Unsupported allocators and failures are non-fatal."""
@@ -1028,12 +1163,14 @@ def apply_memory_plan(
# case where the decode spike can OOM, so turn tiling on now.
nonlocal tiling_engaged
pipe.enable_model_cpu_offload(device = placement)
+ keep_cpu_weights_on_offload(pipe, logger)
if not tiling_engaged:
tiling_engaged = _enable_vae_saver(pipe, "enable_vae_tiling", "enable_tiling", logger)
policy = plan.offload_policy
if policy == OFFLOAD_MODEL:
pipe.enable_model_cpu_offload(device = placement)
+ keep_cpu_weights_on_offload(pipe, logger)
elif policy == OFFLOAD_GROUP:
# getattr, not attribute access: manually built / duck-typed plans predate this field.
if not _apply_group_offload(
diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
new file mode 100644
index 00000000000..57cc0e18454
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU tests onload to ``meta``: a kept weight returns to its host storage, a stock copy raises (no data)."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+hooks = pytest.importorskip("accelerate.hooks")
+
+from core.inference import diffusion_memory as dm # noqa: E402
+
+
+def _pipe(device, *names):
+ """Diffusers-shaped pipe whose enable_model_cpu_offload rebuilds chained CpuOffload hooks each call."""
+ torch.manual_seed(0)
+ comps = {name: torch.nn.Linear(8, 8) for name in names}
+ pipe = types.SimpleNamespace(components = comps, enabled = 0)
+
+ def enable_model_cpu_offload(device = device):
+ pipe.enabled += 1
+ prev = None
+ for module in comps.values():
+ hooks.remove_hook_from_module(module)
+ module.to("cpu")
+ hook = hooks.CpuOffload(execution_device = device, prev_module_hook = prev)
+ hooks.add_hook_to_module(module, hook)
+ prev = hooks.UserCpuOffloadHook(module, hook)
+
+ pipe.enable_model_cpu_offload = enable_model_cpu_offload
+ for name, module in comps.items():
+ setattr(pipe, name, module)
+ return pipe
+
+
+def _ptrs(module):
+ return [p.data_ptr() for p in module.parameters()]
+
+
+def _wrapped(module):
+ return bool(getattr(module._hf_hook, dm._KEEP_ATTR, False))
+
+
+def test_every_offload_hook_is_wrapped():
+ pipe = _pipe("meta", "text_encoder", "transformer")
+ pipe.enable_model_cpu_offload()
+ assert dm.keep_cpu_weights_on_offload(pipe) == 2
+ assert _wrapped(pipe.text_encoder) and _wrapped(pipe.transformer)
+ assert dm.keep_cpu_weights_on_offload(pipe) == 0
+
+
+def test_hooks_rebuilt_by_a_later_enable_are_wrapped_too():
+ pipe = _pipe("meta", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ for _ in range(3):
+ pipe.enable_model_cpu_offload()
+ assert _wrapped(pipe.transformer)
+
+
+def test_a_device_that_hands_back_new_parameters_takes_the_stock_path():
+ pipe = _pipe("meta", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ pipe.transformer(torch.ones(1, 8))
+ assert pipe.transformer.weight.device.type == "meta"
+ with pytest.raises(NotImplementedError):
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+
+
+def test_kill_switch_and_missing_hooks_leave_the_stock_path(monkeypatch):
+ pipe = _pipe("meta", "transformer")
+ pipe.enable_model_cpu_offload()
+ monkeypatch.setenv(dm.OFFLOAD_KEEP_CPU_ENV, "0")
+ assert dm.keep_cpu_weights_on_offload(pipe) == 0
+ assert not _wrapped(pipe.transformer)
+ monkeypatch.delenv(dm.OFFLOAD_KEEP_CPU_ENV)
+ assert dm.keep_cpu_weights_on_offload(types.SimpleNamespace(components = {})) == 0
+
+
+def test_subclass_weights_are_not_kept():
+ class Sub(torch.Tensor):
+ pass
+
+ lin = torch.nn.Linear(2, 2)
+ assert dm._keepable(lin.weight)
+ lin.weight = torch.nn.Parameter(lin.weight.detach().as_subclass(Sub))
+ assert not dm._keepable(lin.weight)
+
+
+def test_an_all_subclass_module_is_not_rescanned_every_forward():
+ class Sub(torch.Tensor):
+ pass
+
+ pipe = _pipe("meta", "transformer")
+ lin = pipe.transformer
+ lin.weight = torch.nn.Parameter(lin.weight.detach().as_subclass(Sub))
+ lin.bias = torch.nn.Parameter(lin.bias.detach().as_subclass(Sub))
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ scans = []
+ stock = type(lin).named_parameters
+
+ def named_parameters(self, *args, **kwargs):
+ # accelerate's own device probe passes recurse=; the wrapper's scans pass nothing.
+ if not args and not kwargs:
+ scans.append(1)
+ return stock(self, *args, **kwargs)
+
+ lin.named_parameters = types.MethodType(named_parameters, lin)
+ for _ in range(5):
+ lin._hf_hook.pre_forward(lin, torch.zeros(1, 8))
+ assert scans == []
+
+
+def test_model_offload_plan_wraps_the_pipeline():
+ pipe = _pipe("meta", "transformer")
+ plan = types.SimpleNamespace(
+ offload_policy = dm.OFFLOAD_MODEL, vae_tiling = False, vae_slicing = False
+ )
+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "meta")
+ assert policy == dm.OFFLOAD_MODEL
+ assert _wrapped(pipe.transformer)
+
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
+
+
+def _render(pipe, x):
+ out = pipe.transformer(pipe.text_encoder(x)).detach().cpu()
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ pipe.enable_model_cpu_offload()
+ return out
+
+
+@cuda
+def test_real_offload_keeps_host_storage_and_is_bit_identical():
+ x = torch.randn(4, 8)
+ stock = _pipe("cuda", "text_encoder", "transformer")
+ stock.enable_model_cpu_offload()
+ ref = _render(stock, x)
+ pipe = _pipe("cuda", "text_encoder", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ host = {n: _ptrs(m) for n, m in pipe.components.items()}
+ for _ in range(3):
+ assert torch.equal(_render(pipe, x), ref)
+ assert {n: _ptrs(m) for n, m in pipe.components.items()} == host
+
+
+@cuda
+def test_a_weight_written_on_the_device_is_copied_back():
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ before = pipe.transformer.weight.detach().clone()
+ pipe.transformer(torch.ones(1, 8))
+ with torch.no_grad():
+ pipe.transformer.weight.add_(1)
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ assert pipe.transformer.weight.device.type == "cpu"
+ assert torch.equal(pipe.transformer.weight.detach(), before + 1)
+
+
+@cuda
+def test_a_parameter_replaced_on_the_device_is_copied_back():
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ pipe.transformer(torch.ones(1, 8))
+ # Same name and version as the onloaded weight: only identity tells it apart.
+ old = pipe.transformer.weight
+ new = torch.nn.Parameter(torch.full_like(old, 3.0))
+ with torch.no_grad():
+ while new._version < old._version:
+ new.mul_(1)
+ assert new._version == old._version
+ pipe.transformer.weight = new
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ assert pipe.transformer.weight.device.type == "cpu"
+ assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 3.0))
+
+
+@cuda
[email protected]("how", ["parameter", "data"])
+def test_a_parameter_replaced_while_offloaded_is_not_restored_to_the_old_weight(how):
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ pipe.transformer(torch.ones(1, 8))
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ assert pipe.transformer.weight.device.type == "cpu"
+ fresh = torch.full((8, 8), 5.0)
+ if how == "parameter":
+ pipe.transformer.weight = torch.nn.Parameter(fresh)
+ else:
+ pipe.transformer.weight.data = fresh
+ out = pipe.transformer(torch.ones(1, 8))
+ assert torch.allclose(
+ out.cpu(), torch.full((1, 8), 40.0) + pipe.transformer.bias.detach().cpu()
+ )
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ assert pipe.transformer.weight.device.type == "cpu"
+ assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 5.0))
+
+
+@cuda
+def test_a_weight_reassigned_through_data_on_the_device_is_copied_back():
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ pipe.transformer(torch.ones(1, 8))
+ version = pipe.transformer.weight._version
+ pipe.transformer.weight.data = torch.full((8, 8), 7.0, device = "cuda")
+ assert pipe.transformer.weight._version == version
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ assert pipe.transformer.weight.device.type == "cpu"
+ assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 7.0))