Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: whole-model offload points weights back at their host tensors instead of copying them (#11764)
Daniel Han28d5e76de6d8
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex e5842da5e11..a510504e61c 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -114,6 +114,141 @@ def _resolve_host_memory_reclaimer() -> Optional[Callable[[], None]]:return reclaim+OFFLOAD_KEEP_CPU_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_KEEP_CPU"+_KEEP_ATTR = "_unsloth_offload_keep_cpu"+++def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:+ """Offload by re-pointing unmodified weights at their host tensors; enable_model_cpu_offload is wrapped since diffusers rebuilds hooks."""+ if (os.environ.get(OFFLOAD_KEEP_CPU_ENV) or "").strip().lower() in ("0", "off", "false", "no"):+ return 0+ try:+ from accelerate.hooks import CpuOffload+ except Exception: # noqa: BLE001 - no accelerate, no offload hooks to wrap+ return 0+ enable = getattr(pipe, "enable_model_cpu_offload", None)+ if callable(enable) and not getattr(enable, _KEEP_ATTR, False):++ def _enable(*args: Any, **kwargs: Any) -> Any:+ out = enable(*args, **kwargs)+ _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)+ return out++ setattr(_enable, _KEEP_ATTR, True)+ try:+ pipe.enable_model_cpu_offload = _enable+ except Exception: # noqa: BLE001 - a pipeline refusing the attribute keeps the stock hooks+ return 0+ return _wrap_cpu_offload_hooks(pipe, CpuOffload, logger)+++def _keepable(param: Any) -> bool:+ import torch+ return type(param.data) is torch.Tensor+++def _wrap_cpu_offload_hooks(+ pipe: Any,+ hook_cls: type,+ logger: Any = None,+) -> int:+ wrapped = 0+ components = getattr(pipe, "components", None) or {}+ for module in components.values():+ hook = getattr(module, "_hf_hook", None)+ if not isinstance(hook, hook_cls) or getattr(hook, _KEEP_ATTR, False):+ continue+ try:+ _wrap_cpu_offload_hook(hook, module)+ wrapped += 1+ except Exception as exc: # noqa: BLE001 - that module keeps the stock copy+ if logger is not None:+ logger.debug(+ "diffusion.memory: keep-cpu offload not applied to %s: %s",+ type(module).__name__,+ exc,+ )+ return wrapped+++def _wrap_cpu_offload_hook(hook: Any, module: Any) -> None:+ # Keyed by name: a move can hand the module new Parameter objects, and a stale key must miss, not alias.+ state = module.__dict__.get(_KEEP_ATTR)+ if state is None:+ state = {"host": {}, "owner": {}, "version": {}}+ module.__dict__[_KEEP_ATTR] = state+ state.setdefault("owner", {})+ host, owner, version = state["host"], state["owner"], state["version"]++ def _capture(mod: Any) -> None:+ host.clear()+ owner.clear()+ for name, p in mod.named_parameters():+ if p.device.type == "cpu" and _keepable(p):+ host[name] = p.data+ owner[name] = p++ _capture(module)+ version.clear()+ init_hook, pre_forward = hook.init_hook, hook.pre_forward++ def _init_hook(mod: Any) -> Any:+ for name, p in mod.named_parameters():+ kept = host.get(name)+ seen = version.get(name)+ # Identity and data_ptr too: a replacement can match the version, and `p.data = ...` does not bump it.+ if (+ kept is None+ or p.device.type == "cpu"+ or seen is None+ or seen[0] is not p+ or seen[1] != p._version+ or seen[2] != p.data_ptr()+ ):+ continue+ try:+ p.data = kept+ except Exception: # noqa: BLE001 - incompatible tensor types: the stock copy below handles it+ pass+ out = init_hook(mod)+ _capture(mod)+ version.clear()+ return out++ def _pre_forward(mod: Any, *args: Any, **kwargs: Any) -> Any:+ # `host` gate: an all-subclass module (GGUF, torchao) never fills `version`, so would rescan every forward.+ onload = not version and bool(host)+ if onload:+ for name, p in mod.named_parameters():+ kept = host.get(name)+ if kept is not None and (+ owner.get(name) is not p+ or p.device.type != "cpu"+ or p.data.data_ptr() != kept.data_ptr()+ ):+ host.pop(name, None)+ owner.pop(name, None)+ out = pre_forward(mod, *args, **kwargs)+ if onload:+ for name, p in mod.named_parameters():+ if name in host and owner.get(name) is p and p.device.type != "cpu":+ version[name] = (p, p._version, p.data_ptr())+ return out++ try:+ import torch++ # Same as accelerate's own pre_forward: a compiled forward must not trace the hook.+ _init_hook, _pre_forward = (+ torch.compiler.disable(_init_hook),+ torch.compiler.disable(_pre_forward),+ )+ except Exception: # noqa: BLE001 - older torch: the hook runs outside any compiled region anyway+ pass+ hook.init_hook, hook.pre_forward = _init_hook, _pre_forward+ setattr(hook, _KEEP_ATTR, True)++def reclaim_offload_host_memory(offload_policy: str, logger: Any = None) -> bool:"""Return unused allocator pages after whole-model CPU offload, without touching livetensors, Python GC, or device caches. Unsupported allocators and failures are non-fatal."""@@ -1028,12 +1163,14 @@ def apply_memory_plan(# case where the decode spike can OOM, so turn tiling on now.nonlocal tiling_engagedpipe.enable_model_cpu_offload(device = placement)+ keep_cpu_weights_on_offload(pipe, logger)if not tiling_engaged:tiling_engaged = _enable_vae_saver(pipe, "enable_vae_tiling", "enable_tiling", logger)policy = plan.offload_policyif policy == OFFLOAD_MODEL:pipe.enable_model_cpu_offload(device = placement)+ keep_cpu_weights_on_offload(pipe, logger)elif policy == OFFLOAD_GROUP:# getattr, not attribute access: manually built / duck-typed plans predate this field.if not _apply_group_offload(diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.pynew file mode 100644index 00000000000..57cc0e18454--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py@@ -0,0 +1,222 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU tests onload to ``meta``: a kept weight returns to its host storage, a stock copy raises (no data)."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")+hooks = pytest.importorskip("accelerate.hooks")++from core.inference import diffusion_memory as dm # noqa: E402+++def _pipe(device, *names):+ """Diffusers-shaped pipe whose enable_model_cpu_offload rebuilds chained CpuOffload hooks each call."""+ torch.manual_seed(0)+ comps = {name: torch.nn.Linear(8, 8) for name in names}+ pipe = types.SimpleNamespace(components = comps, enabled = 0)++ def enable_model_cpu_offload(device = device):+ pipe.enabled += 1+ prev = None+ for module in comps.values():+ hooks.remove_hook_from_module(module)+ module.to("cpu")+ hook = hooks.CpuOffload(execution_device = device, prev_module_hook = prev)+ hooks.add_hook_to_module(module, hook)+ prev = hooks.UserCpuOffloadHook(module, hook)++ pipe.enable_model_cpu_offload = enable_model_cpu_offload+ for name, module in comps.items():+ setattr(pipe, name, module)+ return pipe+++def _ptrs(module):+ return [p.data_ptr() for p in module.parameters()]+++def _wrapped(module):+ return bool(getattr(module._hf_hook, dm._KEEP_ATTR, False))+++def test_every_offload_hook_is_wrapped():+ pipe = _pipe("meta", "text_encoder", "transformer")+ pipe.enable_model_cpu_offload()+ assert dm.keep_cpu_weights_on_offload(pipe) == 2+ assert _wrapped(pipe.text_encoder) and _wrapped(pipe.transformer)+ assert dm.keep_cpu_weights_on_offload(pipe) == 0+++def test_hooks_rebuilt_by_a_later_enable_are_wrapped_too():+ pipe = _pipe("meta", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ for _ in range(3):+ pipe.enable_model_cpu_offload()+ assert _wrapped(pipe.transformer)+++def test_a_device_that_hands_back_new_parameters_takes_the_stock_path():+ pipe = _pipe("meta", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ pipe.transformer(torch.ones(1, 8))+ assert pipe.transformer.weight.device.type == "meta"+ with pytest.raises(NotImplementedError):+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+++def test_kill_switch_and_missing_hooks_leave_the_stock_path(monkeypatch):+ pipe = _pipe("meta", "transformer")+ pipe.enable_model_cpu_offload()+ monkeypatch.setenv(dm.OFFLOAD_KEEP_CPU_ENV, "0")+ assert dm.keep_cpu_weights_on_offload(pipe) == 0+ assert not _wrapped(pipe.transformer)+ monkeypatch.delenv(dm.OFFLOAD_KEEP_CPU_ENV)+ assert dm.keep_cpu_weights_on_offload(types.SimpleNamespace(components = {})) == 0+++def test_subclass_weights_are_not_kept():+ class Sub(torch.Tensor):+ pass++ lin = torch.nn.Linear(2, 2)+ assert dm._keepable(lin.weight)+ lin.weight = torch.nn.Parameter(lin.weight.detach().as_subclass(Sub))+ assert not dm._keepable(lin.weight)+++def test_an_all_subclass_module_is_not_rescanned_every_forward():+ class Sub(torch.Tensor):+ pass++ pipe = _pipe("meta", "transformer")+ lin = pipe.transformer+ lin.weight = torch.nn.Parameter(lin.weight.detach().as_subclass(Sub))+ lin.bias = torch.nn.Parameter(lin.bias.detach().as_subclass(Sub))+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ scans = []+ stock = type(lin).named_parameters++ def named_parameters(self, *args, **kwargs):+ # accelerate's own device probe passes recurse=; the wrapper's scans pass nothing.+ if not args and not kwargs:+ scans.append(1)+ return stock(self, *args, **kwargs)++ lin.named_parameters = types.MethodType(named_parameters, lin)+ for _ in range(5):+ lin._hf_hook.pre_forward(lin, torch.zeros(1, 8))+ assert scans == []+++def test_model_offload_plan_wraps_the_pipeline():+ pipe = _pipe("meta", "transformer")+ plan = types.SimpleNamespace(+ offload_policy = dm.OFFLOAD_MODEL, vae_tiling = False, vae_slicing = False+ )+ policy, _ = dm.apply_memory_plan(pipe, plan, device = "meta")+ assert policy == dm.OFFLOAD_MODEL+ assert _wrapped(pipe.transformer)+++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")+++def _render(pipe, x):+ out = pipe.transformer(pipe.text_encoder(x)).detach().cpu()+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ pipe.enable_model_cpu_offload()+ return out+++@cuda+def test_real_offload_keeps_host_storage_and_is_bit_identical():+ x = torch.randn(4, 8)+ stock = _pipe("cuda", "text_encoder", "transformer")+ stock.enable_model_cpu_offload()+ ref = _render(stock, x)+ pipe = _pipe("cuda", "text_encoder", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ host = {n: _ptrs(m) for n, m in pipe.components.items()}+ for _ in range(3):+ assert torch.equal(_render(pipe, x), ref)+ assert {n: _ptrs(m) for n, m in pipe.components.items()} == host+++@cuda+def test_a_weight_written_on_the_device_is_copied_back():+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ before = pipe.transformer.weight.detach().clone()+ pipe.transformer(torch.ones(1, 8))+ with torch.no_grad():+ pipe.transformer.weight.add_(1)+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ assert pipe.transformer.weight.device.type == "cpu"+ assert torch.equal(pipe.transformer.weight.detach(), before + 1)+++@cuda+def test_a_parameter_replaced_on_the_device_is_copied_back():+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ pipe.transformer(torch.ones(1, 8))+ # Same name and version as the onloaded weight: only identity tells it apart.+ old = pipe.transformer.weight+ new = torch.nn.Parameter(torch.full_like(old, 3.0))+ with torch.no_grad():+ while new._version < old._version:+ new.mul_(1)+ assert new._version == old._version+ pipe.transformer.weight = new+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ assert pipe.transformer.weight.device.type == "cpu"+ assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 3.0))+++@cuda[email protected]("how", ["parameter", "data"])+def test_a_parameter_replaced_while_offloaded_is_not_restored_to_the_old_weight(how):+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ pipe.transformer(torch.ones(1, 8))+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ assert pipe.transformer.weight.device.type == "cpu"+ fresh = torch.full((8, 8), 5.0)+ if how == "parameter":+ pipe.transformer.weight = torch.nn.Parameter(fresh)+ else:+ pipe.transformer.weight.data = fresh+ out = pipe.transformer(torch.ones(1, 8))+ assert torch.allclose(+ out.cpu(), torch.full((1, 8), 40.0) + pipe.transformer.bias.detach().cpu()+ )+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ assert pipe.transformer.weight.device.type == "cpu"+ assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 5.0))+++@cuda+def test_a_weight_reassigned_through_data_on_the_device_is_copied_back():+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ pipe.transformer(torch.ones(1, 8))+ version = pipe.transformer.weight._version+ pipe.transformer.weight.data = torch.full((8, 8), 7.0, device = "cuda")+ assert pipe.transformer.weight._version == version+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ assert pipe.transformer.weight.device.type == "cpu"+ assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 7.0))