Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: fast whole-component swap for low_vram offload (#14767)
Daniel Han4e2838ec5f0c
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex bebb55f8873..7d3e0aab92f 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -21,6 +21,7 @@ import contextvarsimport functoolsimport osimport sys+import threadingfrom dataclasses import dataclass, field, replacefrom typing import Any, Callable, Optionalfrom .rocm_bf16 import is_rocm_torch@@ -137,7 +138,8 @@ def keep_cpu_weights_on_offload(pipe: Any, logger: Any = None) -> int:def _enable(*args: Any, **kwargs: Any) -> Any:# The re-enable after every call starts with pipe.to("cpu"), which would copy back the module run last._offload_through_kept_hooks(pipe, CpuOffload, logger)- out = enable(*args, **kwargs)+ with _keep_device_cache(pipe):+ out = enable(*args, **kwargs)_wrap_cpu_offload_hooks(pipe, CpuOffload, logger)return out@@ -305,8 +307,10 @@ def _pin_host_weights(host: dict,logger: Any = None,buffer_host: Optional[dict] = None,+ layout: Optional[list] = None,) -> int:- """Pack kept host weights and buffers into page-locked chunks; returns bytes pinned. Each copy keeps its source strides."""+ """Pack kept host weights and buffers into page-locked chunks; returns bytes pinned. Each copy keeps its source strides.+ ``layout`` (if given) receives ``(chunk, [(name, start, view), ...])`` per chunk for ``_upload_pinned_chunks``."""import torchmode = _pin_mode()@@ -381,7 +385,7 @@ def _pin_host_weights(else view.as_strided(src.shape, src.stride()))view.copy_(src)- placed.append((name, p, attr, view))+ placed.append((name, p, attr, view, index, start))except Exception as exc: # noqa: BLE001 - e.g. a WSL pinned-memory cap: keep the pageable weightsbufs.clear()placed.clear()@@ -394,8 +398,12 @@ def _pin_host_weights(exc,)return 0+ members: list = [[] for _ in bufs]wrapped: dict = {}- for name, p, attr, view in placed:+ for name, p, attr, view, index, start in placed:+ if attr is None:+ # Wrapped (torchao) inner tensors stay with the stock module.to: the chunk upload re-points p.data only.+ members[index].append((name, start, view))if name.startswith("\0"):name = name[1:]_set_buffer(module, name, view)@@ -409,9 +417,163 @@ def _pin_host_weights(host[name] = viewfor name, p in wrapped.items():host[name] = p.data # the kept host weight now reads the pinned inner tensors+ if layout is not None:+ layout[:] = [(buf, entries) for buf, entries in zip(bufs, members) if entries]return need+OFFLOAD_FAST_UPLOAD_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_FAST_UPLOAD"+++def _on_device(module: Any) -> bool:+ first = next(iter(module.parameters()), None)+ return first is not None and first.device.type != "cpu"+++def _keep_device_cache(pipe: Any):+ """Skip the empty_cache inside diffusers' enable_model_cpu_offload (re-run after every call) once the+ components onload through pinned chunks: the chunks freed by the last offload are reused by the next onload,+ while an emptied cache makes every onload cudaMalloc again and every offload cudaFree."""+ module = sys.modules.get("diffusers.pipelines.pipeline_utils")+ original = getattr(module, "empty_device_cache", None) if module is not None else None+ components = getattr(pipe, "components", None) or {}+ engaged = _fast_upload_enabled() and any(+ bool((getattr(m, "__dict__", {}).get(_KEEP_ATTR) or {}).get("pin_layout"))+ for m in components.values()+ )+ if not engaged or not callable(original):+ yield+ return+ # Process-global patch: overlapping enables share one count, and the last one out restores the real function.+ with _cache_patch_lock:+ if not _cache_patch:+ _cache_patch.update(module = module, original = original, depth = 0)+ module.empty_device_cache = _skip_empty_device_cache+ _cache_patch["depth"] += 1+ try:+ yield+ finally:+ with _cache_patch_lock:+ _cache_patch["depth"] -= 1+ if _cache_patch["depth"] <= 0:+ if _cache_patch["module"].empty_device_cache is _skip_empty_device_cache:+ _cache_patch["module"].empty_device_cache = _cache_patch["original"]+ _cache_patch.clear()+++_cache_patch_lock = threading.Lock()+_cache_patch: dict = {}+++def _skip_empty_device_cache(*args: Any, **kwargs: Any) -> None:+ return None+++def _fast_upload_enabled() -> bool:+ return (os.environ.get(OFFLOAD_FAST_UPLOAD_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _same_view(tensor: Any, view: Any) -> bool:+ return (+ tensor.device.type == "cpu"+ and tensor.data_ptr() == view.data_ptr()+ and tensor.dtype == view.dtype+ and tensor.shape == view.shape+ and tensor.stride() == view.stride()+ )+++def _upload_pinned_chunks(+ module: Any, layout: list, host: dict, owner: dict, buffer_host: dict, device: Any+) -> int:+ """Onload a module's pinned weights chunk by chunk: one device allocation and one non_blocking copy per pinned+ chunk, then every weight is re-pointed at its slice. The stock ``module.to`` allocates every tensor on its own,+ and with the device cache emptied between components each of those was a cudaMalloc (and a cudaFree on the way+ out): under host load that ran at 57-120 MiB/s against a 50 GiB/s link. Chunks are near-uniform, so the next+ component reuses the previous one's cached chunks. Same bytes, same strides, so outputs are bit-identical.+ Weights no longer at their pinned host view (replaced, reassigned) are left for the stock copy. Returns bytes."""+ import torch++ params = dict(module.named_parameters())+ buffers = dict(_named_buffers(module))+ copied = 0+ moved: list = []+ try:+ for buf, entries in layout:+ live = []+ for name, start, view in entries:+ # By address, not identity: ``p.data`` and the re-captured host map hand out fresh tensor objects.+ if name.startswith("\0"):+ leaf = name[1:]+ b = buffers.get(leaf)+ if b is not None and buffer_host.get(leaf) is b and _same_view(b, view):+ live.append((name, start, view))+ continue+ p = params.get(name)+ kept = host.get(name)+ if (+ p is not None+ and owner.get(name) is p+ and kept is not None+ and _same_view(kept, view)+ and _same_view(p.data, view)+ ):+ live.append((name, start, view))+ if not live:+ continue+ # Pack the movable weights: replaced weights and torchao wrapper tensors (left to the stock module.to)+ # would otherwise leave holes that are allocated and uploaded for nothing.+ used, slots = 0, []+ for name, start, view in sorted(live, key = lambda e: e[1]):+ used = -(-used // _PIN_ALIGN) * _PIN_ALIGN+ slots.append((name, start, view, used))+ used += view.nbytes+ dense = all(start == at for _, start, _, at in slots)+ # Same stream as the forward that follows, so no sync: kernels queue behind the copy.+ dev_buf = torch.empty(used, dtype = torch.uint8, device = device)+ if not torch._has_compatible_shallow_copy_type(buf, dev_buf):+ # e.g. meta: the stock move hands back new Parameters there instead of re-pointing them.+ raise TypeError("device tensors cannot back host parameters")+ if dense:+ dev_buf.copy_(buf[:used], non_blocking = True)+ else:+ for _, start, view, at in slots:+ dev_buf[at : at + view.nbytes].copy_(+ buf[start : start + view.nbytes], non_blocking = True+ )+ copied += used+ for name, _, view, at in slots:+ dev = dev_buf[at : at + view.nbytes].view(view.dtype)+ dev = (+ dev.view(view.shape)+ if view.is_contiguous()+ else dev.as_strided(view.shape, view.stride())+ )+ if name.startswith("\0"):+ _set_buffer(module, name[1:], dev)+ else:+ params[name].data = dev+ moved.append((name, view))+ except Exception:+ # Put every weight back on its host view; the stock per-tensor copy then runs (and raises a real OOM itself).+ for name, view in moved:+ try:+ if name.startswith("\0"):+ _set_buffer(module, name[1:], view)+ else:+ params[name].data = view+ except Exception: # noqa: BLE001+ pass+ return 0+ return copied++def _named_buffers(module: Any) -> list:named = getattr(module, "named_buffers", None)return list(named()) if callable(named) else []@@ -530,17 +692,65 @@ def _wrap_cpu_offload_hook(if onload and not state.get("pin_tried"):# Once per module, on its first onload, so loading pays nothing.state["pin_tried"] = True+ layout: list = []try:- pinned = _pin_host_weights(mod, host, logger, buffer_host = buffer_host)+ pinned = _pin_host_weights(+ mod, host, logger, buffer_host = buffer_host, layout = layout+ )except Exception: # noqa: BLE001 - pinning is an optimisation, never a failurepinned = 0+ if pinned:+ state["pin_layout"] = layoutif pinned and logger is not None:logger.info("diffusion.memory: pinned %.1f GiB of %s host weights",pinned / 2**30,type(mod).__name__,)- out = pre_forward(mod, *args, **kwargs)+ layout = state.get("pin_layout") if onload and _fast_upload_enabled() else None+ if layout and "to" in mod.__dict__:+ # An instance-level mover is already installed (stable arena, pinned swap): defer to it.+ layout = None+ if not layout:+ out = pre_forward(mod, *args, **kwargs)+ else:+ # Offload the previous component through its kept hook here, so the stock pre_forward finds it on the+ # host and skips its empty_cache: the freed chunks stay in the caching allocator for this onload.+ prev = getattr(hook, "prev_module_hook", None)+ prev_model = getattr(prev, "model", None)+ if (+ prev_model is not None+ and getattr(getattr(prev, "hook", None), _KEEP_ATTR, False)+ and callable(getattr(prev, "offload", None))+ and _on_device(prev_model)+ ):+ prev.offload()++ # The stock pre_forward offloads the previous component, then calls module.to(device) only if the module+ # is not there yet. Shadow that one call so the pinned chunks go up first; module.to then moves whatever+ # the chunks did not cover (unpinned or replaced weights) exactly as before.+ def _to(*to_args: Any, **to_kwargs: Any) -> Any:+ mod.__dict__.pop("to", None)+ import torch++ target = to_args[0] if to_args else to_kwargs.get("device")+ try:+ target = torch.device(target)+ except Exception: # noqa: BLE001 - not a plain device move: stock only+ target = None+ if (+ target is not None+ and target.type != "cpu"+ and len(to_args) + len(to_kwargs) == 1+ ):+ _upload_pinned_chunks(mod, layout, host, owner, buffer_host, target)+ return mod.to(*to_args, **to_kwargs)++ mod.__dict__["to"] = _to+ try:+ out = pre_forward(mod, *args, **kwargs)+ finally:+ mod.__dict__.pop("to", None)if onload:for name, p in mod.named_parameters():if name in host and owner.get(name) is p and p.device.type != "cpu":diff --git a/studio/backend/tests/test_diffusion_offload_fast_upload.py b/studio/backend/tests/test_diffusion_offload_fast_upload.pynew file mode 100644index 00000000000..e719be70ade--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_fast_upload.py@@ -0,0 +1,335 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Whole-component offload onloads pinned weights one chunk at a time, not one tensor at a time.++The offload chain empties the device cache between components, so a per-tensor onload pays one cudaMalloc per+weight (and one cudaFree on the way out). Under host load that ran at 57-120 MiB/s instead of the link's 50 GiB/s.+CPU tests fake the pinning and onload to ``meta``: a per-chunk onload leaves every weight a view of one shared+allocation, a per-tensor onload gives every weight its own.+"""++from __future__ import annotations++import sys+import types++import pytest++torch = pytest.importorskip("torch")+hooks = pytest.importorskip("accelerate.hooks")++from core.inference import diffusion_memory as dm # noqa: E402++N_LAYERS = 6+++def _stack():+ torch.manual_seed(0)+ return torch.nn.Sequential(*(torch.nn.Linear(16, 16) for _ in range(N_LAYERS)))+++def _pipe(device, *names):+ comps = {name: _stack() for name in names}+ pipe = types.SimpleNamespace(components = comps)++ def enable_model_cpu_offload(device = device):+ prev = None+ for module in comps.values():+ hooks.remove_hook_from_module(module)+ module.to("cpu")+ hook = hooks.CpuOffload(execution_device = device, prev_module_hook = prev)+ hooks.add_hook_to_module(module, hook)+ prev = hooks.UserCpuOffloadHook(module, hook)++ pipe.enable_model_cpu_offload = enable_model_cpu_offload+ for name, module in comps.items():+ setattr(pipe, name, module)+ return pipe+++def fake_pin(monkeypatch):+ """Pinning without CUDA: plain host chunks, small enough that one module spans several."""+ real_empty = torch.empty+ device_allocs: list = []++ def empty(*args, **kwargs):+ kwargs.pop("pin_memory", None)+ out = real_empty(*args, **kwargs)+ if out.device.type != "cpu":+ device_allocs.append(out.nbytes)+ return out++ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch, "empty", empty)+ return device_allocs+++def _storages(module):+ return {p.untyped_storage().nbytes() for p in module.parameters()}+++def _onload(pipe):+ module = pipe.transformer+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))+ return module+++def test_pinning_records_the_chunk_layout(fake_pin):+ module = _stack()+ host = {n: p.data for n, p in module.named_parameters()}+ layout: list = []+ assert dm._pin_host_weights(module, host, layout = layout) > 0+ assert 1 < len(layout) < len(host)+ names = [name for _, entries in layout for name, _, _ in entries]+ assert sorted(names) == sorted(host)+ for buf, entries in layout:+ for name, start, view in entries:+ assert view is host[name]+ assert view.data_ptr() == buf.data_ptr() + start+++def test_a_device_that_cannot_back_host_parameters_takes_the_stock_copy(fake_pin):+ # meta tensors cannot be set as a CPU Parameter's data; the stock move hands back new Parameters there.+ pipe = _pipe("meta", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ module = _onload(pipe)+ assert module.__dict__[dm._KEEP_ATTR]["pin_layout"]+ assert all(p.device.type == "meta" for p in module.parameters())+ assert all(p.untyped_storage().nbytes() == p.nbytes for p in module.parameters())+++ "value,on", [("", True), ("1", True), ("0", False), ("off", False), ("False", False)]+)+def test_fast_upload_switch(monkeypatch, value, on):+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, value)+ assert dm._fast_upload_enabled() is on+++def _fake_pipeline_utils(monkeypatch):+ calls: list = []+ module = types.ModuleType("diffusers.pipelines.pipeline_utils")+ module.empty_device_cache = lambda *a, **k: calls.append(a)+ monkeypatch.setitem(sys.modules, "diffusers.pipelines.pipeline_utils", module)+ return module, calls+++def test_the_re_enable_keeps_the_device_cache_only_once_chunks_are_in_use(monkeypatch):+ module, calls = _fake_pipeline_utils(monkeypatch)+ lin = torch.nn.Linear(2, 2)+ pipe = types.SimpleNamespace(components = {"transformer": lin})+ with dm._keep_device_cache(pipe):+ module.empty_device_cache("cuda")+ assert calls == [("cuda",)] # nothing pinned yet: the stock empty_cache runs+ lin.__dict__[dm._KEEP_ATTR] = {"pin_layout": [object()]}+ with dm._keep_device_cache(pipe):+ module.empty_device_cache("cuda")+ assert calls == [("cuda",)]+ module.empty_device_cache("cuda") # restored on exit+ assert len(calls) == 2+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, "0")+ with dm._keep_device_cache(pipe):+ module.empty_device_cache("cuda")+ assert len(calls) == 3+++def test_the_device_cache_is_restored_when_the_re_enable_raises(monkeypatch):+ module, calls = _fake_pipeline_utils(monkeypatch)+ original = module.empty_device_cache+ lin = torch.nn.Linear(2, 2)+ lin.__dict__[dm._KEEP_ATTR] = {"pin_layout": [object()]}+ with pytest.raises(RuntimeError):+ with dm._keep_device_cache(types.SimpleNamespace(components = {"t": lin})):+ raise RuntimeError("boom")+ assert module.empty_device_cache is original+++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")+++def _render(+ pipe,+ x,+ ctx = torch.no_grad,+):+ out = x+ with ctx():+ for module in pipe.components.values():+ out = module(out.to(next(module.parameters()).device))+ out = out.detach().cpu()+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ pipe.enable_model_cpu_offload()+ return out+++def _device_allocs():+ return torch.cuda.memory_stats().get("num_device_alloc", 0)+++@cuda[email protected]("mode", ["no_grad", "inference_mode"])+def test_chunk_onload_is_bit_identical_and_stops_allocating(monkeypatch, mode):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)+ ctx = torch.inference_mode if mode == "inference_mode" else torch.no_grad+ x = torch.randn(4, 16)+ stock = _pipe("cuda", "text_encoder", "transformer")+ stock.enable_model_cpu_offload()+ ref = _render(stock, x, ctx)+ pipe = _pipe("cuda", "text_encoder", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ assert torch.equal(_render(pipe, x, ctx), ref) # the first render pins+ layouts = {n: m.__dict__[dm._KEEP_ATTR]["pin_layout"] for n, m in pipe.components.items()}+ assert all(1 < len(layout) < N_LAYERS * 2 for layout in layouts.values())+ seen = []++ def spy(module, *args):+ # A post-hook: accelerate onloads inside forward, after any pre-hook.+ assert all(p.device.type == "cuda" for p in module.parameters())+ seen.append({p.untyped_storage().data_ptr() for p in module.parameters()})++ for module in pipe.components.values():+ module.register_forward_hook(spy)+ allocs = _device_allocs()+ for _ in range(3):+ assert torch.equal(_render(pipe, x, ctx), ref)+ # Onloaded weights are views of one allocation per chunk (per tensor would be one storage per weight) ...+ assert [len(s) for s in seen] == [len(layouts["text_encoder"]), len(layouts["transformer"])] * 3+ # ... and the chunks come back out of the caching allocator: no cudaMalloc per swap.+ assert _device_allocs() - allocs <= 2+ host = pipe.transformer.__dict__[dm._KEEP_ATTR]["host"]+ assert all(+ p.data_ptr() == host[n].data_ptr() and p.is_pinned()+ for n, p in pipe.transformer.named_parameters()+ )+++@cuda+def test_kill_switch_restores_the_per_tensor_onload(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ monkeypatch.setenv(dm.OFFLOAD_FAST_UPLOAD_ENV, "0")+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ module = pipe.transformer+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))+ assert all(p.device.type == "cuda" for p in module.parameters())+ assert all(p.untyped_storage().nbytes() == p.nbytes for p in module.parameters())+++@cuda+def test_a_weight_replaced_while_offloaded_takes_the_stock_copy(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 4096)+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ module = pipe.transformer+ _render(pipe, torch.randn(4, 16))+ fresh = torch.nn.Parameter(torch.full((16, 16), 0.5))+ module[0].weight = fresh+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))+ assert all(p.device.type == "cuda" for p in module.parameters())+ assert module[0].weight.untyped_storage().nbytes() == fresh.nbytes+ assert torch.equal(module[0].weight.cpu(), torch.full((16, 16), 0.5))+ assert module[1].weight.untyped_storage().nbytes() > module[1].weight.nbytes+++@cuda+def test_a_weight_written_on_the_device_after_a_chunk_onload_is_copied_back(monkeypatch):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")+ pipe = _pipe("cuda", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ module = pipe.transformer+ _render(pipe, torch.randn(4, 16))+ before = module[2].weight.detach().clone()+ module._hf_hook.pre_forward(module, torch.zeros(1, 16))+ with torch.no_grad():+ module[2].weight.add_(1)+ module._hf_hook.init_hook(module)+ assert module[2].weight.device.type == "cpu"+ assert torch.equal(module[2].weight.detach(), before + 1)+++def test_overlapping_re_enables_restore_the_real_device_cache(monkeypatch):+ module, calls = _fake_pipeline_utils(monkeypatch)+ original = module.empty_device_cache+ lin = torch.nn.Linear(2, 2)+ lin.__dict__[dm._KEEP_ATTR] = {"pin_layout": [object()]}+ pipe = types.SimpleNamespace(components = {"t": lin})+ first, second = dm._keep_device_cache(pipe), dm._keep_device_cache(pipe)+ first.__enter__()+ second.__enter__()+ first.__exit__(None, None, None) # the first out must not end the second's patch ...+ module.empty_device_cache("cuda")+ assert calls == []+ second.__exit__(None, None, None) # ... nor may the last out leave a no-op behind+ assert module.empty_device_cache is original+++def test_a_replaced_weight_leaves_no_stale_hole_in_the_chunk_upload(fake_pin):+ module = _stack()+ host = {n: p.data for n, p in module.named_parameters()}+ owner = dict(module.named_parameters())+ layout: list = []+ assert dm._pin_host_weights(module, host, layout = layout) > 0+ buf, entries = layout[0]+ assert len(entries) > 1+ first, last = entries[0][0], entries[-1][0]+ expect = {n: p.detach().clone() for n, p in module.named_parameters()}+ prefix, _, leaf = first.rpartition(".")+ setattr(module.get_submodule(prefix), leaf, torch.nn.Parameter(torch.zeros_like(expect[first])))+ copied = dm._upload_pinned_chunks(module, layout[:1], host, owner, {}, torch.device("cpu"))+ live = [view for name, _, view in entries if name != first]+ assert copied < max(start + view.nbytes for _, start, view in entries)+ assert copied <= sum(-(-v.nbytes // dm._PIN_ALIGN) * dm._PIN_ALIGN for v in live)+ params = dict(module.named_parameters())+ assert params[last].data_ptr() != host[last].data_ptr() # re-pointed at the device chunk+ for name, _, _ in entries[1:]:+ assert torch.equal(params[name].detach(), expect[name])+++def test_an_instance_level_mover_is_kept_and_used(fake_pin):+ pipe = _pipe("meta", "transformer")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ module = pipe.transformer+ calls: list = []+ stock = type(module).to++ def mover(*args, **kwargs):+ calls.append(args)+ return stock(module, *args, **kwargs)++ module.__dict__["to"] = mover # e.g. a stable arena's placement+ _onload(pipe)+ assert module.__dict__[dm._KEEP_ATTR]["pin_layout"] # the chunk path was armed ...+ assert calls # ... yet the onload went through the installed mover+ assert module.__dict__.get("to") is mover+++def test_tensors_left_out_of_the_layout_are_not_uploaded_with_the_chunk(fake_pin):+ # A torchao wrapper weight is pinned into the chunk but left out of its entries (the stock module.to moves it).+ module = _stack()+ host = {n: p.data for n, p in module.named_parameters()}+ owner = dict(module.named_parameters())+ layout: list = []+ assert dm._pin_host_weights(module, host, layout = layout) > 0+ buf, entries = layout[0]+ assert len(entries) > 1 and entries[0][1] == 0+ span = max(start + view.nbytes for _, start, view in entries)+ kept = entries[1:]+ expect = {name: view.clone() for name, _, view in kept}+ copied = dm._upload_pinned_chunks(module, [(buf, kept)], host, owner, {}, torch.device("cpu"))+ assert copied <= span - entries[1][1]+ params = dict(module.named_parameters())+ for name, _, view in kept:+ assert params[name].data_ptr() != view.data_ptr()+ assert torch.equal(params[name].detach(), expect[name])