Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: fast whole-component swap for low_vram offload (#14767)

Daniel Han4e2838ec5f0c
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index bebb55f8873..7d3e0aab92f 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -21,6 +21,7 @@ import contextvars
import functools
import os
import sys
+import threading
from dataclasses import dataclass, field, replace
from typing import Any, Callable, Optional
from .rocm_bf16 import is_rocm_torch
@@ -137,7 +138,8 @@ def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:
def _enable(*args: Any, **kwargs: Any) -> Any:
# The re-enable after every call starts with pipe.to("cpu"), which would copy back the module run last.
_offload_through_kept_hooks(pipe, CpuOffload, logger)
- out = enable(*args, **kwargs)
+ with _keep_device_cache(pipe):
+ out = enable(*args, **kwargs)
_wrap_cpu_offload_hooks(pipe, CpuOffload, logger)
return out
@@ -305,8 +307,10 @@ def _pin_host_weights(
host: dict,
logger: Any = None,
buffer_host: Optional[dict] = None,
+ layout: Optional[list] = None,
) -> int:
- """Pack kept host weights and buffers into page-locked chunks; returns bytes pinned. Each copy keeps its source strides."""
+ """Pack kept host weights and buffers into page-locked chunks; returns bytes pinned. Each copy keeps its source strides.
+ ``layout`` (if given) receives ``(chunk, [(name, start, view), ...])`` per chunk for ``_upload_pinned_chunks``."""
import torch
mode = _pin_mode()
@@ -381,7 +385,7 @@ def _pin_host_weights(
else view.as_strided(src.shape, src.stride())
)
view.copy_(src)
- placed.append((name, p, attr, view))
+ placed.append((name, p, attr, view, index, start))
except Exception as exc: # noqa: BLE001 - e.g. a WSL pinned-memory cap: keep the pageable weights
bufs.clear()
placed.clear()
@@ -394,8 +398,12 @@ def _pin_host_weights(
exc,
)
return 0
+ members: list = [[] for _ in bufs]
wrapped: dict = {}
- for name, p, attr, view in placed:
+ for name, p, attr, view, index, start in placed:
+ if attr is None:
+ # Wrapped (torchao) inner tensors stay with the stock module.to: the chunk upload re-points p.data only.
+ members[index].append((name, start, view))
if name.startswith("\0"):
name = name[1:]
_set_buffer(module, name, view)
@@ -409,9 +417,163 @@ def _pin_host_weights(
host[name] = view
for name, p in wrapped.items():
host[name] = p.data # the kept host weight now reads the pinned inner tensors
+ if layout is not None:
+ layout[:] = [(buf, entries) for buf, entries in zip(bufs, members) if entries]
return need
+OFFLOAD_FAST_UPLOAD_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_FAST_UPLOAD"
+
+
+def _on_device(module: Any) -> bool:
+ first = next(iter(module.parameters()), None)
+ return first is not None and first.device.type != "cpu"
+
+
+def _keep_device_cache(pipe: Any):
+ """Skip the empty_cache inside diffusers' enable_model_cpu_offload (re-run after every call) once the
+ components onload through pinned chunks: the chunks freed by the last offload are reused by the next onload,
+ while an emptied cache makes every onload cudaMalloc again and every offload cudaFree."""
+ module = sys.modules.get("diffusers.pipelines.pipeline_utils")
+ original = getattr(module, "empty_device_cache", None) if module is not None else None
+ components = getattr(pipe, "components", None) or {}
+ engaged = _fast_upload_enabled() and any(
+ bool((getattr(m, "__dict__", {}).get(_KEEP_ATTR) or {}).get("pin_layout"))
+ for m in components.values()
+ )
+ if not engaged or not callable(original):
+ yield
+ return
+ # Process-global patch: overlapping enables share one count, and the last one out restores the real function.
+ with _cache_patch_lock:
+ if not _cache_patch:
+ _cache_patch.update(module = module, original = original, depth = 0)
+ module.empty_device_cache = _skip_empty_device_cache
+ _cache_patch["depth"] += 1
+ try:
+ yield
+ finally:
+ with _cache_patch_lock:
+ _cache_patch["depth"] -= 1
+ if _cache_patch["depth"] <= 0:
+ if _cache_patch["module"].empty_device_cache is _skip_empty_device_cache:
+ _cache_patch["module"].empty_device_cache = _cache_patch["original"]
+ _cache_patch.clear()
+
+
+_cache_patch_lock = threading.Lock()
+_cache_patch: dict = {}
+
+
+def _skip_empty_device_cache(*args: Any, **kwargs: Any) -> None:
+ return None
+
+
+def _fast_upload_enabled() -> bool:
+ return (os.environ.get(OFFLOAD_FAST_UPLOAD_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _same_view(tensor: Any, view: Any) -> bool:
+ return (
+ tensor.device.type == "cpu"
+ and tensor.data_ptr() == view.data_ptr()
+ and tensor.dtype == view.dtype
+ and tensor.shape == view.shape
+ and tensor.stride() == view.stride()
+ )
+
+
+def _upload_pinned_chunks(
+ module: Any, layout: list, host: dict, owner: dict, buffer_host: dict, device: Any
+) -> int:
+ """Onload a module's pinned weights chunk by chunk: one device allocation and one non_blocking copy per pinned
+ chunk, then every weight is re-pointed at its slice. The stock ``module.to`` allocates every tensor on its own,
+ and with the device cache emptied between components each of those was a cudaMalloc (and a cudaFree on the way
+ out): under host load that ran at 57-120 MiB/s against a 50 GiB/s link. Chunks are near-uniform, so the next
+ component reuses the previous one's cached chunks. Same bytes, same strides, so outputs are bit-identical.
+ Weights no longer at their pinned host view (replaced, reassigned) are left for the stock copy. Returns bytes."""
+ import torch
+
+ params = dict(module.named_parameters())
+ buffers = dict(_named_buffers(module))
+ copied = 0
+ moved: list = []
+ try:
+ for buf, entries in layout:
+ live = []
+ for name, start, view in entries:
+ # By address, not identity: ``p.data`` and the re-captured host map hand out fresh tensor objects.
+ if name.startswith("\0"):
+ leaf = name[1:]
+ b = buffers.get(leaf)
+ if b is not None and buffer_host.get(leaf) is b and _same_view(b, view):
+ live.append((name, start, view))
+ continue
+ p = params.get(name)
+ kept = host.get(name)
+ if (
+ p is not None
+ and owner.get(name) is p
+ and kept is not None
+ and _same_view(kept, view)
+ and _same_view(p.data, view)
+ ):
+ live.append((name, start, view))
+ if not live:
+ continue
+ # Pack the movable weights: replaced weights and torchao wrapper tensors (left to the stock module.to)
+ # would otherwise leave holes that are allocated and uploaded for nothing.
+ used, slots = 0, []
+ for name, start, view in sorted(live, key = lambda e: e[1]):
+ used = -(-used // _PIN_ALIGN) * _PIN_ALIGN
+ slots.append((name, start, view, used))
+ used += view.nbytes
+ dense = all(start == at for _, start, _, at in slots)
+ # Same stream as the forward that follows, so no sync: kernels queue behind the copy.
+ dev_buf = torch.empty(used, dtype = torch.uint8, device = device)
+ if not torch._has_compatible_shallow_copy_type(buf, dev_buf):
+ # e.g. meta: the stock move hands back new Parameters there instead of re-pointing them.
+ raise TypeError("device tensors cannot back host parameters")
+ if dense:
+ dev_buf.copy_(buf[:used], non_blocking = True)
+ else:
+ for _, start, view, at in slots:
+ dev_buf[at : at + view.nbytes].copy_(
+ buf[start : start + view.nbytes], non_blocking = True
+ )
+ copied += used
+ for name, _, view, at in slots:
+ dev = dev_buf[at : at + view.nbytes].view(view.dtype)
+ dev = (
+ dev.view(view.shape)
+ if view.is_contiguous()
+ else dev.as_strided(view.shape, view.stride())
+ )
+ if name.startswith("\0"):
+ _set_buffer(module, name[1:], dev)
+ else:
+ params[name].data = dev
+ moved.append((name, view))
+ except Exception:
+ # Put every weight back on its host view; the stock per-tensor copy then runs (and raises a real OOM itself).
+ for name, view in moved:
+ try:
+ if name.startswith("\0"):
+ _set_buffer(module, name[1:], view)
+ else:
+ params[name].data = view
+ except Exception: # noqa: BLE001
+ pass
+ return 0
+ return copied
+
+
def _named_buffers(module: Any) -> list:
named = getattr(module, "named_buffers", None)
return list(named()) if callable(named) else []
@@ -530,17 +692,65 @@ def _wrap_cpu_offload_hook(
if onload and not state.get("pin_tried"):
# Once per module, on its first onload, so loading pays nothing.
state["pin_tried"] = True
+ layout: list = []
try:
- pinned = _pin_host_weights(mod, host, logger, buffer_host = buffer_host)
+ pinned = _pin_host_weights(
+ mod, host, logger, buffer_host = buffer_host, layout = layout
+ )
except Exception: # noqa: BLE001 - pinning is an optimisation, never a failure
pinned = 0
+ if pinned:
+ state["pin_layout"] = layout
if pinned and logger is not None:
logger.info(
"diffusion.memory: pinned %.1f GiB of %s host weights",
pinned / 2**30,
type(mod).__name__,
)
- out = pre_forward(mod, *args, **kwargs)
+ layout = state.get("pin_layout") if onload and _fast_upload_enabled() else None
+ if layout and "to" in mod.__dict__:
+ # An instance-level mover is already installed (stable arena, pinned swap): defer to it.
+ layout = None
+ if not layout:
+ out = pre_forward(mod, *args, **kwargs)
+ else:
+ # Offload the previous component through its kept hook here, so the stock pre_forward finds it on the
+ # host and skips its empty_cache: the freed chunks stay in the caching allocator for this onload.
+ prev = getattr(hook, "prev_module_hook", None)
+ prev_model = getattr(prev, "model", None)
+ if (
+ prev_model is not None
+ and getattr(getattr(prev, "hook", None), _KEEP_ATTR, False)
+ and callable(getattr(prev, "offload", None))
+ and _on_device(prev_model)
+ ):
+ prev.offload()
+
+ # The stock pre_forward offloads the previous component, then calls module.to(device) only if the module
+ # is not there yet. Shadow that one call so the pinned chunks go up first; module.to then moves whatever
+ # the chunks did not cover (unpinned or replaced weights) exactly as before.
+ def _to(*to_args: Any, **to_kwargs: Any) -> Any:
+ mod.__dict__.pop("to", None)
+ import torch
+
+ target = to_args[0] if to_args else to_kwargs.get("device")
+ try:
+ target = torch.device(target)
+ except Exception: # noqa: BLE001 - not a plain device move: stock only
+ target = None
+ if (
+ target is not None
+ and target.type != "cpu"
+ and len(to_args) + len(to_kwargs) == 1
+ ):
+ _upload_pinned_chunks(mod, layout, host, owner, buffer_host, target)
+ return mod.to(*to_args, **to_kwargs)
+
+ mod.__dict__["to"] = _to
+ try:
+ out = pre_forward(mod, *args, **kwargs)
+ finally:
+ mod.__dict__.pop("to", None)
if onload:
for name, p in mod.named_parameters():
if name in host and owner.get(name) is p and p.device.type != "cpu":
diff --git a/studio/backend/tests/test_diffusion_offload_fast_upload.py b/studio/backend/tests/test_diffusion_offload_fast_upload.py
new file mode 100644
index 00000000000..e719be70ade
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_fast_upload.py
@@ -0,0 +1,335 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Whole-component offload onloads pinned weights one chunk at a time, not one tensor at a time.
+
+The offload chain empties the device cache between components, so a per-tensor onload pays one cudaMalloc per
+weight (and one cudaFree on the way out). Under host load that ran at 57-120 MiB/s instead of the link's 50 GiB/s.
+CPU tests fake the pinning and onload to ``meta``: a per-chunk onload leaves every weight a view of one shared
+allocation, a per-tensor onload gives every weight its own.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+hooks = pytest.importorskip("accelerate.hooks")
+
+from core.inference import diffusion_memory as dm # noqa: E402
+
+N_LAYERS = 6
+
+
+def _stack():
+ torch.manual_seed(0)
+ return torch.nn.Sequential(*(torch.nn.Linear(16, 16) for _ in range(N_LAYERS)))
+
+
+def _pipe(device, *names):
+ comps = {name: _stack() for name in names}
+ pipe = types.SimpleNamespace(components = comps)
+
+ def enable_model_cpu_offload(device = device):
+ prev = None
+ for module in comps.values():
+ hooks.remove_hook_from_module(module)
+ module.to("cpu")
+ hook = hooks.CpuOffload(execution_device = device, prev_module_hook = prev)
+ hooks.add_hook_to_module(module, hook)
+ prev = hooks.UserCpuOffloadHook(module, hook)
+
+ pipe.enable_model_cpu_offload = enable_model_cpu_offload
+ for name, module in comps.items():
+ setattr(pipe, name, module)
+ return pipe
+
+
+def fake_pin(monkeypatch):
+ """Pinning without CUDA: plain host chunks, small enough that one module spans several."""
+ real_empty = torch.empty
+ device_allocs: list = []
+
+ def empty(*args, **kwargs):
+ kwargs.pop("pin_memory", None)
+ out = real_empty(*args, **kwargs)
+ if out.device.type != "cpu":
+ device_allocs.append(out.nbytes)
+ return out
+
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch, "empty", empty)
+ return device_allocs
+
+
+def _storages(module):
+ return {p.untyped_storage().nbytes() for p in module.parameters()}
+
+
+def _onload(pipe):
+ module = pipe.transformer
+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))
+ return module
+
+
+def test_pinning_records_the_chunk_layout(fake_pin):
+ module = _stack()
+ host = {n: p.data for n, p in module.named_parameters()}
+ layout: list = []
+ assert dm._pin_host_weights(module, host, layout = layout) > 0
+ assert 1 < len(layout) < len(host)
+ names = [name for _, entries in layout for name, _, _ in entries]
+ assert sorted(names) == sorted(host)
+ for buf, entries in layout:
+ for name, start, view in entries:
+ assert view is host[name]
+ assert view.data_ptr() == buf.data_ptr() + start
+
+
+def test_a_device_that_cannot_back_host_parameters_takes_the_stock_copy(fake_pin):
+ # meta tensors cannot be set as a CPU Parameter's data; the stock move hands back new Parameters there.
+ pipe = _pipe("meta", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ module = _onload(pipe)
+ assert module.__dict__[dm._KEEP_ATTR]["pin_layout"]
+ assert all(p.device.type == "meta" for p in module.parameters())
+ assert all(p.untyped_storage().nbytes() == p.nbytes for p in module.parameters())
+
+
+ "value,on", [("", True), ("1", True), ("0", False), ("off", False), ("False", False)]
+)
+def test_fast_upload_switch(monkeypatch, value, on):
+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, value)
+ assert dm._fast_upload_enabled() is on
+
+
+def _fake_pipeline_utils(monkeypatch):
+ calls: list = []
+ module = types.ModuleType("diffusers.pipelines.pipeline_utils")
+ module.empty_device_cache = lambda *a, **k: calls.append(a)
+ monkeypatch.setitem(sys.modules, "diffusers.pipelines.pipeline_utils", module)
+ return module, calls
+
+
+def test_the_re_enable_keeps_the_device_cache_only_once_chunks_are_in_use(monkeypatch):
+ module, calls = _fake_pipeline_utils(monkeypatch)
+ lin = torch.nn.Linear(2, 2)
+ pipe = types.SimpleNamespace(components = {"transformer": lin})
+ with dm._keep_device_cache(pipe):
+ module.empty_device_cache("cuda")
+ assert calls == [("cuda",)] # nothing pinned yet: the stock empty_cache runs
+ lin.__dict__[dm._KEEP_ATTR] = {"pin_layout": [object()]}
+ with dm._keep_device_cache(pipe):
+ module.empty_device_cache("cuda")
+ assert calls == [("cuda",)]
+ module.empty_device_cache("cuda") # restored on exit
+ assert len(calls) == 2
+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, "0")
+ with dm._keep_device_cache(pipe):
+ module.empty_device_cache("cuda")
+ assert len(calls) == 3
+
+
+def test_the_device_cache_is_restored_when_the_re_enable_raises(monkeypatch):
+ module, calls = _fake_pipeline_utils(monkeypatch)
+ original = module.empty_device_cache
+ lin = torch.nn.Linear(2, 2)
+ lin.__dict__[dm._KEEP_ATTR] = {"pin_layout": [object()]}
+ with pytest.raises(RuntimeError):
+ with dm._keep_device_cache(types.SimpleNamespace(components = {"t": lin})):
+ raise RuntimeError("boom")
+ assert module.empty_device_cache is original
+
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
+
+
+def _render(
+ pipe,
+ x,
+ ctx = torch.no_grad,
+):
+ out = x
+ with ctx():
+ for module in pipe.components.values():
+ out = module(out.to(next(module.parameters()).device))
+ out = out.detach().cpu()
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ pipe.enable_model_cpu_offload()
+ return out
+
+
+def _device_allocs():
+ return torch.cuda.memory_stats().get("num_device_alloc", 0)
+
+
+@cuda
[email protected]("mode", ["no_grad", "inference_mode"])
+def test_chunk_onload_is_bit_identical_and_stops_allocating(monkeypatch, mode):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)
+ ctx = torch.inference_mode if mode == "inference_mode" else torch.no_grad
+ x = torch.randn(4, 16)
+ stock = _pipe("cuda", "text_encoder", "transformer")
+ stock.enable_model_cpu_offload()
+ ref = _render(stock, x, ctx)
+ pipe = _pipe("cuda", "text_encoder", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ assert torch.equal(_render(pipe, x, ctx), ref) # the first render pins
+ layouts = {n: m.__dict__[dm._KEEP_ATTR]["pin_layout"] for n, m in pipe.components.items()}
+ assert all(1 < len(layout) < N_LAYERS * 2 for layout in layouts.values())
+ seen = []
+
+ def spy(module, *args):
+ # A post-hook: accelerate onloads inside forward, after any pre-hook.
+ assert all(p.device.type == "cuda" for p in module.parameters())
+ seen.append({p.untyped_storage().data_ptr() for p in module.parameters()})
+
+ for module in pipe.components.values():
+ module.register_forward_hook(spy)
+ allocs = _device_allocs()
+ for _ in range(3):
+ assert torch.equal(_render(pipe, x, ctx), ref)
+ # Onloaded weights are views of one allocation per chunk (per tensor would be one storage per weight) ...
+ assert [len(s) for s in seen] == [len(layouts["text_encoder"]), len(layouts["transformer"])] * 3
+ # ... and the chunks come back out of the caching allocator: no cudaMalloc per swap.
+ assert _device_allocs() - allocs <= 2
+ host = pipe.transformer.__dict__[dm._KEEP_ATTR]["host"]
+ assert all(
+ p.data_ptr() == host[n].data_ptr() and p.is_pinned()
+ for n, p in pipe.transformer.named_parameters()
+ )
+
+
+@cuda
+def test_kill_switch_restores_the_per_tensor_onload(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, "0")
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ module = pipe.transformer
+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))
+ assert all(p.device.type == "cuda" for p in module.parameters())
+ assert all(p.untyped_storage().nbytes() == p.nbytes for p in module.parameters())
+
+
+@cuda
+def test_a_weight_replaced_while_offloaded_takes_the_stock_copy(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ module = pipe.transformer
+ _render(pipe, torch.randn(4, 16))
+ fresh = torch.nn.Parameter(torch.full((16, 16), 0.5))
+ module[0].weight = fresh
+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))
+ assert all(p.device.type == "cuda" for p in module.parameters())
+ assert module[0].weight.untyped_storage().nbytes() == fresh.nbytes
+ assert torch.equal(module[0].weight.cpu(), torch.full((16, 16), 0.5))
+ assert module[1].weight.untyped_storage().nbytes() > module[1].weight.nbytes
+
+
+@cuda
+def test_a_weight_written_on_the_device_after_a_chunk_onload_is_copied_back(monkeypatch):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
+ pipe = _pipe("cuda", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ module = pipe.transformer
+ _render(pipe, torch.randn(4, 16))
+ before = module[2].weight.detach().clone()
+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))
+ with torch.no_grad():
+ module[2].weight.add_(1)
+ module._hf_hook.init_hook(module)
+ assert module[2].weight.device.type == "cpu"
+ assert torch.equal(module[2].weight.detach(), before + 1)
+
+
+def test_overlapping_re_enables_restore_the_real_device_cache(monkeypatch):
+ module, calls = _fake_pipeline_utils(monkeypatch)
+ original = module.empty_device_cache
+ lin = torch.nn.Linear(2, 2)
+ lin.__dict__[dm._KEEP_ATTR] = {"pin_layout": [object()]}
+ pipe = types.SimpleNamespace(components = {"t": lin})
+ first, second = dm._keep_device_cache(pipe), dm._keep_device_cache(pipe)
+ first.__enter__()
+ second.__enter__()
+ first.__exit__(None, None, None) # the first out must not end the second's patch ...
+ module.empty_device_cache("cuda")
+ assert calls == []
+ second.__exit__(None, None, None) # ... nor may the last out leave a no-op behind
+ assert module.empty_device_cache is original
+
+
+def test_a_replaced_weight_leaves_no_stale_hole_in_the_chunk_upload(fake_pin):
+ module = _stack()
+ host = {n: p.data for n, p in module.named_parameters()}
+ owner = dict(module.named_parameters())
+ layout: list = []
+ assert dm._pin_host_weights(module, host, layout = layout) > 0
+ buf, entries = layout[0]
+ assert len(entries) > 1
+ first, last = entries[0][0], entries[-1][0]
+ expect = {n: p.detach().clone() for n, p in module.named_parameters()}
+ prefix, _, leaf = first.rpartition(".")
+ setattr(module.get_submodule(prefix), leaf, torch.nn.Parameter(torch.zeros_like(expect[first])))
+ copied = dm._upload_pinned_chunks(module, layout[:1], host, owner, {}, torch.device("cpu"))
+ live = [view for name, _, view in entries if name != first]
+ assert copied < max(start + view.nbytes for _, start, view in entries)
+ assert copied <= sum(-(-v.nbytes // dm._PIN_ALIGN) * dm._PIN_ALIGN for v in live)
+ params = dict(module.named_parameters())
+ assert params[last].data_ptr() != host[last].data_ptr() # re-pointed at the device chunk
+ for name, _, _ in entries[1:]:
+ assert torch.equal(params[name].detach(), expect[name])
+
+
+def test_an_instance_level_mover_is_kept_and_used(fake_pin):
+ pipe = _pipe("meta", "transformer")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ module = pipe.transformer
+ calls: list = []
+ stock = type(module).to
+
+ def mover(*args, **kwargs):
+ calls.append(args)
+ return stock(module, *args, **kwargs)
+
+ module.__dict__["to"] = mover # e.g. a stable arena's placement
+ _onload(pipe)
+ assert module.__dict__[dm._KEEP_ATTR]["pin_layout"] # the chunk path was armed ...
+ assert calls # ... yet the onload went through the installed mover
+ assert module.__dict__.get("to") is mover
+
+
+def test_tensors_left_out_of_the_layout_are_not_uploaded_with_the_chunk(fake_pin):
+ # A torchao wrapper weight is pinned into the chunk but left out of its entries (the stock module.to moves it).
+ module = _stack()
+ host = {n: p.data for n, p in module.named_parameters()}
+ owner = dict(module.named_parameters())
+ layout: list = []
+ assert dm._pin_host_weights(module, host, layout = layout) > 0
+ buf, entries = layout[0]
+ assert len(entries) > 1 and entries[0][1] == 0
+ span = max(start + view.nbytes for _, start, view in entries)
+ kept = entries[1:]
+ expect = {name: view.clone() for name, _, view in kept}
+ copied = dm._upload_pinned_chunks(module, [(buf, kept)], host, owner, {}, torch.device("cpu"))
+ assert copied <= span - entries[1][1]
+ params = dict(module.named_parameters())
+ for name, _, view in kept:
+ assert params[name].data_ptr() != view.data_ptr()
+ assert torch.equal(params[name].detach(), expect[name])