Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: replay a model-offloaded denoiser's CUDA graph across renders and pin torchao int8 host weights (#14666)

Daniel Hancaa55746e5ae
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 877b44dbc14..1a319ba973c 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -33,7 +33,9 @@ OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_CUDA_GRAPH"
# =0: a block-streamed whole-step graph records its copies into the graph pool instead of the prefetcher's slot ring.
STEP_SLOTS_ENV = "UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS"
MODEL_OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_MODEL_OFFLOAD_CUDA_GRAPH"
-# Default off: replay saved ~1% of a step (FLUX.2-klein 4B, 1024^2) but the per-render re-record + ~0.6 GB pool cost more.
+# =1: record over the stock onloads, re-recording whenever they move (replay saved ~1% of a step on FLUX.2-klein 4B
+# 1024^2 while each re-record + ~0.6 GB pool cost more). Default: record only over the stable range, see
+# diffusion_stable_arena.
MODEL_OFFLOAD_REASON = (
"model offload re-uploads the denoiser every render at new addresses, so its step graph would record again on "
"every render; set " + MODEL_OFFLOAD_CUDA_GRAPH_ENV + "=1 to record it anyway"
@@ -1728,7 +1730,9 @@ class OffloadPlacement:
and the resident groups read in place. The pool holds the in-flight window at fixed addresses, so it plays the
prefetch ring. Recorded again when a residency change, a release / restore or a host-copy swap moves an address.
- ``model``: whole-module CPU offload (accelerate), only with ``MODEL_OFFLOAD_CUDA_GRAPH_ENV=1`` (see
+ ``model``: whole-module CPU offload (accelerate). By default only over the stable device range
+ (``diffusion_stable_arena``: every onload lands at the same addresses, so render 1's recording replays on every
+ later render); ``MODEL_OFFLOAD_CUDA_GRAPH_ENV=1`` records over the stock onloads instead (see
``MODEL_OFFLOAD_REASON``). The hook moves the module on the eager path first and the replay sits under it, in
``_old_forward``. Recorded again when the onload placed the weights at new addresses; after
``MODEL_MAX_INVALIDATIONS`` re-records the module runs eager (each re-record costs a forward)."""
@@ -2002,14 +2006,23 @@ def offload_placement(module: Any) -> tuple:
None,
f"offload hook {type(hf_hook).__name__} moves the weights inside the forward",
)
- if not model_offload_graphs_requested():
- return None, MODEL_OFFLOAD_REASON
+ # Every other refusal first: the stable range below swaps the module's onload, so it attaches only for a
+ # placement that will record.
why = _forward_refusal(module, "model")
if why is not None:
return None, why
+ arena = None
+ if not model_offload_graphs_requested():
+ # Recorded only where the onloads land at the same addresses every render (diffusion_stable_arena).
+ from .diffusion_stable_arena import attach
+ arena, _arena_why = attach(module)
+ if arena is None:
+ return None, MODEL_OFFLOAD_REASON
try:
return OffloadPlacement(module, "model"), None
except Exception as exc: # noqa: BLE001
+ if arena is not None:
+ arena.detach()
return None, f"model offload hook unreadable ({exc})"
registry = getattr(module, "_diffusers_hook", None)
hooks = getattr(registry, "hooks", None) or {}
@@ -2049,6 +2062,25 @@ def offload_placement(module: Any) -> tuple:
return placement, None
+def _detach_stable_range(module: Any) -> None:
+ """Give a module whose step stays eager its stock onload back (no stable range, no per-offload card sync)."""
+ try:
+ from .diffusion_stable_arena import arena_of
+ arena = arena_of(module)
+ if arena is not None:
+ arena.detach()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _stable_range(module: Any) -> bool:
+ try:
+ from .diffusion_stable_arena import arena_of
+ return arena_of(module) is not None
+ except Exception: # noqa: BLE001
+ return False
+
+
def _forward_refusal(module: Any, mode: str) -> Optional[str]:
"""Why ``module``'s own forward cannot record under ``mode`` offload, else None (see GraphedForward.__init__)."""
try:
@@ -2144,8 +2176,11 @@ def arm_after_placement(
module, max_graphs = max_graphs, logger = logger, placement = placement
).enable()
)
- modes.append(placement.mode)
+ stable = placement.mode == "model" and _stable_range(module)
+ modes.append("model_stable" if stable else placement.mode)
except Exception as exc: # noqa: BLE001
+ if placement is not None and placement.mode == "model":
+ _detach_stable_range(module)
refusals.append(f"{type(module).__name__}: {exc}")
installed = tuple(handles)
try:
@@ -2159,6 +2194,7 @@ def arm_after_placement(
"resident": "resident",
"group": "block-streamed (copies recorded in the graph)",
"model": "model offload",
+ "model_stable": "model offload over a stable device range",
}
reason = "captured per input shape: " + ", ".join(sorted({labels.get(m, m) for m in modes}))
if logger is not None:
@@ -2435,6 +2471,11 @@ def stats(handles: Any) -> dict:
out["replay_ms"] = float(handle.stats["replay_ms"])
placement = getattr(handle, "placement", None)
out["placements"].append("resident" if placement is None else placement.mode)
+ if placement is not None and placement.mode == "model":
+ from .diffusion_stable_arena import arena_of
+ arena = arena_of(placement.module)
+ if arena is not None:
+ out["stable_range"] = arena.describe()
if handle.poisoned:
out["poisoned"] = True
error = handle.capture_error
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 453b8c961b7..e8ee9cc48a7 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -176,12 +176,56 @@ def _gguf_parameter_class() -> Optional[type]:
return getattr(module, "GGUFParameter", None) if module is not None else None
+def _wrapper_inner(t: Any) -> Optional[list]:
+ """``[(attr, tensor), ...]`` of a traceable wrapper subclass (torchao int8 / fp8 weights) whose inner tensors are
+ all plain tensors, else None."""
+ import torch
+
+ flatten = getattr(type(t), "__tensor_flatten__", None)
+ if flatten is None or type(t) in (torch.Tensor, torch.nn.Parameter):
+ return None
+ try:
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
+
+ if not is_traceable_wrapper_subclass(t):
+ return None
+ names, _ctx = t.__tensor_flatten__()
+ inner = [(name, getattr(t, name)) for name in names]
+ except Exception: # noqa: BLE001
+ return None
+ if not inner or any(type(x) is not torch.Tensor for _, x in inner):
+ return None
+ return inner
+
+
+def _tensor_ptr(t: Any) -> Any:
+ """Storage address of ``t``; for a wrapper subclass, the tuple of its inner tensors' addresses."""
+ inner = _wrapper_inner(t)
+ if inner is not None:
+ return tuple(int(x.data_ptr()) for _, x in inner)
+ return t.data_ptr()
+
+
+def _repoint(param: Any, kept: Any) -> None:
+ """Point ``param`` at its kept host tensor without a copy. A wrapper subclass cannot change device through
+ ``.data``; it swaps with a parameter over the kept tensors, as ``Module.to`` moves it."""
+ if _wrapper_inner(kept) is None:
+ param.data = kept
+ return
+ import torch
+
+ torch.utils.swap_tensors(param, torch.nn.Parameter(kept, requires_grad = param.requires_grad))
+
+
def _keepable(param: Any) -> bool:
import torch
data = param.data
if type(data) is torch.Tensor:
return True
+ # torchao weights (int8 / fp8): a wrapper of plain tensors; kept and re-pointed like a dense weight.
+ if _wrapper_inner(data) is not None and type(data) is type(param):
+ return True
# GGUF weights are packed bytes; the quant type lives on the Parameter, which the swap never replaces.
gguf = _gguf_parameter_class()
return gguf is not None and type(param) is gguf and type(data) is gguf
@@ -267,15 +311,19 @@ def _pin_host_weights(
mode = _pin_mode()
if mode == "off" or not torch.cuda.is_available():
return 0
- params = [
- (name, p)
- for name, p in module.named_parameters()
- if host.get(name) is not None
- and p.device.type == "cpu"
- and _keepable(p)
- and _pinnable_layout(p.data)
- and not p.data.is_pinned()
- ]
+ # (name, owner tensor, inner attribute or None, the tensor to pin)
+ params: list = []
+ for name, p in module.named_parameters():
+ if host.get(name) is None or p.device.type != "cpu" or not _keepable(p):
+ continue
+ inner = _wrapper_inner(p.data)
+ if inner is None:
+ if _pinnable_layout(p.data) and not p.data.is_pinned():
+ params.append((name, p, None, p.data))
+ continue
+ for attr, t in inner:
+ if t.device.type == "cpu" and _pinnable_layout(t) and not t.is_pinned():
+ params.append((name, p, attr, t))
buffers = [
(name, b)
for name, b in _named_buffers(module)
@@ -285,8 +333,8 @@ def _pin_host_weights(
and b.is_contiguous()
and not b.is_pinned()
]
- params = params + [("\0" + name, b) for name, b in buffers]
- sizes = [-(-p.data.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, p in params]
+ params = params + [("\0" + name, b, None, b) for name, b in buffers]
+ sizes = [-(-t.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, _, _, t in params]
if not sizes or sum(sizes) == 0:
return 0
# Lay out chunks first so the RAM gate counts chunk tails and last-chunk rounding, not just weight bytes.
@@ -322,17 +370,17 @@ def _pin_host_weights(
bufs: list = []
placed: list = []
try:
- for (name, p), (index, start) in zip(params, slots):
+ for (name, p, attr, src), (index, start) in zip(params, slots):
if index == len(bufs):
bufs.append(torch.empty(chunks[index], dtype = torch.uint8, pin_memory = True))
- view = bufs[index][start : start + p.data.nbytes].view(p.dtype)
+ view = bufs[index][start : start + src.nbytes].view(src.dtype)
view = (
- view.view(p.shape)
- if p.data.is_contiguous()
- else view.as_strided(p.shape, p.data.stride())
+ view.view(src.shape)
+ if src.is_contiguous()
+ else view.as_strided(src.shape, src.stride())
)
- view.copy_(p.data)
- placed.append((name, p, view))
+ view.copy_(src)
+ placed.append((name, p, attr, view))
except Exception as exc: # noqa: BLE001 - e.g. a WSL pinned-memory cap: keep the pageable weights
bufs.clear()
placed.clear()
@@ -345,14 +393,21 @@ def _pin_host_weights(
exc,
)
return 0
- for name, p, view in placed:
+ wrapped: dict = {}
+ for name, p, attr, view in placed:
if name.startswith("\0"):
name = name[1:]
_set_buffer(module, name, view)
buffer_host[name] = view
continue
+ if attr is not None:
+ setattr(p, attr, view)
+ wrapped[name] = p
+ continue
p.data = view
host[name] = view
+ for name, p in wrapped.items():
+ host[name] = p.data # the kept host weight now reads the pinned inner tensors
return need
@@ -441,11 +496,11 @@ def _wrap_cpu_offload_hook(
or seen is None
or seen[0] is not p
or seen[1] != p._version
- or seen[2] != p.data_ptr()
+ or seen[2] != _tensor_ptr(p)
):
continue
try:
- p.data = kept
+ _repoint(p, kept)
except Exception: # noqa: BLE001 - incompatible tensor types: the stock copy below handles it
pass
out = init_hook(mod)
@@ -463,7 +518,7 @@ def _wrap_cpu_offload_hook(
if kept is not None and (
owner.get(name) is not p
or p.device.type != "cpu"
- or p.data.data_ptr() != kept.data_ptr()
+ or _tensor_ptr(p.data) != _tensor_ptr(kept)
):
host.pop(name, None)
owner.pop(name, None)
@@ -488,7 +543,7 @@ def _wrap_cpu_offload_hook(
if onload:
for name, p in mod.named_parameters():
if name in host and owner.get(name) is p and p.device.type != "cpu":
- version[name] = (p, p._version, p.data_ptr())
+ version[name] = (p, p._version, _tensor_ptr(p))
for name, b in _named_buffers(mod):
if (
name in buffer_host
diff --git a/studio/backend/core/inference/diffusion_stable_arena.py b/studio/backend/core/inference/diffusion_stable_arena.py
new file mode 100644
index 00000000000..2ac22ccda8b
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_stable_arena.py
@@ -0,0 +1,586 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Stable device addresses for a whole-module CPU-offloaded denoiser (``memory_mode=low_vram``).
+
+accelerate's ``CpuOffload`` hook onloads the denoiser with ``module.to(device)`` at the start of every render, so the
+caching allocator hands its weights new addresses each time and a CUDA graph recorded on render 1 cannot replay on
+render 2 (``diffusion_cuda_graph.MODEL_OFFLOAD_REASON``). Here the denoiser's device copy lives in one CUDA virtual
+address range reserved once for the load (driver VMM: ``cuMemAddressReserve``). An onload creates physical memory,
+maps it at that range and copies the kept host weights in; the offload syncs the card, unmaps and releases the
+physical memory, so the text encoder and VAE phases get the VRAM back exactly as with the stock hook. The weights
+land at the same addresses on every render, so the recorded step graph keeps replaying.
+
+ComfyUI does the same for its graph replays over offloaded weights (comfy_aimdo ``model_vbar``).
+
+Plain dense tensors (with their strides) and torchao-style wrapper weights (int8 / fp8: their plain inner tensors,
+rebuilt over the range) are placed; a module holding any other parameter type (GGUF) is left to the stock hook.
+Opt out with ``UNSLOTH_DIFFUSION_MODEL_OFFLOAD_ARENA=0``.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import os
+import threading
+import weakref
+from typing import Any, Optional
+
+ARENA_ENV = "UNSLOTH_DIFFUSION_MODEL_OFFLOAD_ARENA"
+_ARENA_ATTR = "_unsloth_stable_arena"
+_ALIGN = 512
+_OFF = ("0", "off", "false", "no")
+
+# CUDA driver enums
+_CU_MEM_ALLOCATION_TYPE_PINNED = 1
+_CU_MEM_LOCATION_TYPE_DEVICE = 1
+_CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 3
+_CU_MEM_ALLOC_GRANULARITY_RECOMMENDED = 1
+
+
+class _Location(ctypes.Structure):
+ _fields_ = [("type", ctypes.c_int), ("id", ctypes.c_int)]
+
+
+class _AllocFlags(ctypes.Structure):
+ _fields_ = [
+ ("compressionType", ctypes.c_ubyte),
+ ("gpuDirectRDMACapable", ctypes.c_ubyte),
+ ("usage", ctypes.c_ushort),
+ ("reserved", ctypes.c_ubyte * 4),
+ ]
+
+
+class _AllocProp(ctypes.Structure):
+ _fields_ = [
+ ("type", ctypes.c_int),
+ ("requestedHandleTypes", ctypes.c_int),
+ ("location", _Location),
+ ("win32HandleMetaData", ctypes.c_void_p),
+ ("allocFlags", _AllocFlags),
+ ]
+
+
+class _AccessDesc(ctypes.Structure):
+ _fields_ = [("location", _Location), ("flags", ctypes.c_int)]
+
+
+_LIB: Any = None
+_LIB_LOCK = threading.Lock()
+
+
+def _driver() -> Any:
+ global _LIB
+ with _LIB_LOCK:
+ if _LIB is None:
+ lib = None
+ for name in ("libcuda.so.1", "libcuda.so", "nvcuda.dll"):
+ try:
+ lib = ctypes.CDLL(name)
+ break
+ except OSError:
+ continue
+ if lib is None:
+ raise OSError("CUDA driver library not found")
+ u64, sz = ctypes.c_uint64, ctypes.c_size_t
+ for fn, args in (
+ (
+ "cuMemGetAllocationGranularity",
+ [ctypes.POINTER(sz), ctypes.POINTER(_AllocProp), ctypes.c_int],
+ ),
+ ("cuMemAddressReserve", [ctypes.POINTER(u64), sz, sz, u64, ctypes.c_ulonglong]),
+ ("cuMemAddressFree", [u64, sz]),
+ (
+ "cuMemCreate",
+ [
+ ctypes.POINTER(ctypes.c_ulonglong),
+ sz,
+ ctypes.POINTER(_AllocProp),
+ ctypes.c_ulonglong,
+ ],
+ ),
+ ("cuMemRelease", [ctypes.c_ulonglong]),
+ ("cuMemMap", [u64, sz, sz, ctypes.c_ulonglong, ctypes.c_ulonglong]),
+ ("cuMemUnmap", [u64, sz]),
+ ("cuMemSetAccess", [u64, sz, ctypes.POINTER(_AccessDesc), sz]),
+ ("cuCtxGetCurrent", [ctypes.POINTER(ctypes.c_void_p)]),
+ ):
+ f = getattr(lib, fn)
+ f.argtypes = args
+ f.restype = ctypes.c_int
+ _LIB = lib
+ return _LIB
+
+
+def _check(rc: int, what: str) -> None:
+ if rc != 0:
+ raise RuntimeError(f"{what} failed (CUresult {rc})")
+
+
+def arena_disabled() -> bool:
+ return (os.environ.get(ARENA_ENV) or "").strip().lower() in _OFF
+
+
+class _CudaArray:
+ """A ``__cuda_array_interface__`` over raw device bytes, so ``torch.as_tensor`` wraps them without a copy."""
+
+ def __init__(self, ptr: int, nbytes: int) -> None:
+ self.__cuda_array_interface__ = {
+ "shape": (int(nbytes),),
+ "typestr": "|u1",
+ "data": (int(ptr), False),
+ "version": 3,
+ "strides": None,
+ "stream": None,
+ }
+
+
+class VmmArena:
+ """One reserved device VA range of ``nbytes`` (rounded to the allocation granularity) on ``device_index``; physical
+ memory exists only between ``map()`` and ``unmap()``."""
+
+ def __init__(self, nbytes: int, device_index: int) -> None:
+ import torch
+
+ self.device_index = int(device_index)
+ self.device = torch.device("cuda", self.device_index)
+ lib = _driver()
+ self._ensure_context()
+ prop = self._prop()
+ gran = ctypes.c_size_t(0)
+ _check(
+ lib.cuMemGetAllocationGranularity(
+ ctypes.byref(gran), ctypes.byref(prop), _CU_MEM_ALLOC_GRANULARITY_RECOMMENDED
+ ),
+ "cuMemGetAllocationGranularity",
+ )
+ self.granularity = int(gran.value) or (2 << 20)
+ self.size = -(-int(nbytes) // self.granularity) * self.granularity
+ va = ctypes.c_uint64(0)
+ _check(
+ lib.cuMemAddressReserve(ctypes.byref(va), self.size, self.granularity, 0, 0),
+ "cuMemAddressReserve",
+ )
+ self.va = int(va.value)
+ self._handle: Optional[int] = None
+ self._base: Any = None
+ self.maps = 0
+ self.unmaps = 0
+ self.closed = False
+
+ def _prop(self) -> _AllocProp:
+ prop = _AllocProp()
+ prop.type = _CU_MEM_ALLOCATION_TYPE_PINNED
+ prop.location.type = _CU_MEM_LOCATION_TYPE_DEVICE
+ prop.location.id = self.device_index
+ return prop
+
+ def _ensure_context(self) -> None:
+ import torch
+
+ ctx = ctypes.c_void_p(0)
+ _driver().cuCtxGetCurrent(ctypes.byref(ctx))
+ if not ctx.value:
+ # The calling thread has no current context yet: any runtime call on the device makes the primary one current.
+ with torch.cuda.device(self.device_index):
+ torch.cuda.current_stream().query()
+
+ @property
+ def mapped(self) -> bool:
+ return self._handle is not None
+
+ def map(self) -> Any:
+ """Back the range with new physical memory; returns a uint8 tensor over the whole range."""
+ import torch
+
+ if self.closed:
+ raise RuntimeError("arena closed")
+ if self._handle is not None:
+ return self._base
+ lib = _driver()
+ with torch.cuda.device(self.device_index):
+ self._ensure_context()
+ prop = self._prop()
+ handle = ctypes.c_ulonglong(0)
+ rc = lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)
+ if rc != 0:
+ # The caching allocator may hold the memory as free cached blocks.
+ torch.cuda.empty_cache()
+ rc = lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)
+ _check(rc, "cuMemCreate")
+ try:
+ _check(lib.cuMemMap(self.va, self.size, 0, handle.value, 0), "cuMemMap")
+ access = _AccessDesc()
+ access.location.type = _CU_MEM_LOCATION_TYPE_DEVICE
+ access.location.id = self.device_index
+ access.flags = _CU_MEM_ACCESS_FLAGS_PROT_READWRITE
+ rc = lib.cuMemSetAccess(self.va, self.size, ctypes.byref(access), 1)
+ if rc != 0:
+ lib.cuMemUnmap(self.va, self.size)
+ _check(rc, "cuMemSetAccess")
+ except Exception:
+ lib.cuMemRelease(handle.value)
+ raise
+ self._handle = int(handle.value)
+ self._base = torch.as_tensor(_CudaArray(self.va, self.size), device = self.device)
+ self.maps += 1
+ return self._base
+
+ def view(
+ self,
+ offset: int,
+ shape: tuple,
+ dtype: Any,
+ stride: Optional[tuple] = None,
+ ) -> Any:
+ import torch
+
+ base = self.map()
+ numel = 1
+ for s in shape:
+ numel *= int(s)
+ itemsize = torch.empty((), dtype = dtype).element_size()
+ if stride is None:
+ span = numel
+ else:
+ span = 1 + sum((int(s) - 1) * int(st) for s, st in zip(shape, stride)) if numel else 0
+ flat = base[offset : offset + span * itemsize].view(dtype)
+ if stride is None:
+ return flat.view(shape)
+ return flat.as_strided(shape, stride)
+
+ def unmap(self) -> None:
+ """Release the physical memory. The caller guarantees no tensor over the range is used again before ``map()``
+ (pending work on the card is waited for here)."""
+ import torch
+
+ if self._handle is None:
+ return
+ lib = _driver()
+ with torch.cuda.device(self.device_index):
+ torch.cuda.synchronize(self.device_index)
+ self._base = None
+ _check(lib.cuMemUnmap(self.va, self.size), "cuMemUnmap")
+ _check(lib.cuMemRelease(self._handle), "cuMemRelease")
+ self._handle = None
+ self.unmaps += 1
+
+ def close(self) -> None:
+ if self.closed:
+ return
+ try:
+ self.unmap()
+ finally:
+ self.closed = True
+ try:
+ _driver().cuMemAddressFree(self.va, self.size)
+ except Exception: # noqa: BLE001 - process teardown
+ pass
+
+
+def _plain(t: Any) -> bool:
+ import torch
+ return type(t) is torch.Tensor or (
+ type(t) is torch.nn.Parameter and type(t.data) is torch.Tensor
+ )
+
+
+def _inner(t: Any) -> Optional[list]:
+ """A torchao-style wrapper weight's plain inner tensors (``diffusion_memory._wrapper_inner``), else None."""
+ from .diffusion_memory import _wrapper_inner
+ return _wrapper_inner(t)
+
+
+def _placeable(t: Any) -> bool:
+ return _plain(t) or _inner(t) is not None
+
+
+def _pieces(name: str, t: Any) -> list:
+ """(layout key, tensor) for each storage of ``t``: its own data, or a wrapper weight's inner tensors."""
+ if _plain(t):
+ return [(name, t.data)]
+ return [(f"{name}#{attr}", x) for attr, x in (_inner(t) or ())]
+
+
+def _set_tensor(owner: Any, leaf: str, kind: str, value: Any) -> None:
+ if kind == "p":
+ owner._parameters[leaf].data = value
+ else:
+ owner._buffers[leaf] = value
+
+
+def _entries(module: Any) -> list:
+ """(qualified name, registrations, kind, tensor) for every parameter and buffer, each tensor once.
+
+ ``registrations`` lists every (owner module, leaf name, qualified name) the tensor is registered under: a buffer
+ shared by several submodules (or names) has one registry slot per registration, and each must point at the placed
+ copy."""
+ out: list = []
+ index: dict = {}
+ for mod_name, mod in module.named_modules(remove_duplicate = False):
+ for kind, table in (("p", mod._parameters), ("b", mod._buffers)):
+ for leaf, t in table.items():
+ if t is None:
+ continue
+ qual = f"{mod_name}.{leaf}" if mod_name else leaf
+ at = index.get(id(t))
+ if at is not None:
+ regs = out[at][1]
+ if not any(o is mod and l == leaf for o, l, _q in regs):
+ regs.append((mod, leaf, qual))
+ continue
+ index[id(t)] = len(out)
+ out.append((qual, [(mod, leaf, qual)], kind, t))
+ return out
+
+
+class ModuleArena:
+ """The stable device range of one whole-module offloaded denoiser, installed as the module's own ``to``."""
+
+ def __init__(
+ self,
+ module: Any,
+ device: Any,
+ logger: Any = None,
+ ) -> None:
+ import torch
+
+ self.device = torch.device(device)
+ if self.device.index is None:
+ self.device = torch.device("cuda", torch.cuda.current_device())
+ self.logger = logger
+ self.layout: Optional[dict] = None
+ self.arena: Optional[VmmArena] = None
+ self.failed: Optional[str] = None
+ self.stats = {"onloads": 0, "placed": 0, "stock": 0, "maps": 0, "unmaps": 0}
+ self._module = weakref.ref(module)
+ self._stock_to = type(module).to
+ module.__dict__["to"] = self._to
+ # accelerate's remove_hook_from_module pops ``to`` from the instance (its _accelerate_added_attributes) and
+ # diffusers re-hooks before every render: put it back before the hook's pre-forward onloads the module.
+ self._reinstall_handle = module.register_forward_pre_hook(self._reinstall)
+ weakref.finalize(module, ModuleArena._finalize, self.__dict__)
+
+ def _reinstall(self, module: Any, _args: Any) -> None:
+ if module.__dict__.get("to") != self._to and self.failed is None:
+ module.__dict__["to"] = self._to
+ self.stats["reinstalls"] = self.stats.get("reinstalls", 0) + 1
+
+ @staticmethod
+ def _finalize(state: dict) -> None:
+ arena = state.get("arena")
+ if arena is not None:
+ try:
+ arena.close()
+ except Exception: # noqa: BLE001 - process teardown
+ pass
+
+ def detach(self) -> None:
+ module = self._module()
+ if module is not None and module.__dict__.get("to") == self._to:
+ module.__dict__.pop("to", None)
+ try:
+ self._reinstall_handle.remove()
+ except Exception: # noqa: BLE001
+ pass
+ if module is not None and module.__dict__.get(_ARENA_ATTR) is self:
+ module.__dict__.pop(_ARENA_ATTR, None)
+ if self.arena is not None and module is not None and not self._referenced(module):
+ self.arena.close()
+ self.arena = None
+
+ def _build(self, entries: list) -> None:
+ offset = 0
+ layout: dict = {}
+ for name, regs, _kind, whole in entries:
+ if whole.device.type != "cpu" or not _placeable(whole):
+ continue
+ for key, t in _pieces(name, whole):
+ # strided layouts (channels_last) keep their strides over the span they reach
+ span = (
+ (1 + sum((n - 1) * st for n, st in zip(t.shape, t.stride()))) * t.element_size()
+ if t.numel()
+ else 0
+ )
+ slot = (offset, tuple(t.shape), tuple(t.stride()), t.dtype, span)
+ # Every registration name of a shared tensor maps to its one slot: Module.to() moves each
+ # registration on its own, so after an offload they come back as separate (equal) host copies.
+ suffix = key[len(name) :]
+ for _owner, _leaf, qual in regs:
+ layout[qual + suffix] = slot
+ offset += -(-max(span, 1) // _ALIGN) * _ALIGN
+ self.layout = layout
+ self.arena = VmmArena(max(offset, _ALIGN), self.device.index)
+
+ def _referenced(self, module: Any) -> bool:
+ if self.arena is None or not self.arena.mapped:
+ return False
+ lo, hi = self.arena.va, self.arena.va + self.arena.size
+ for name, _regs, _kind, whole in _entries(module):
+ try:
+ if whole.device.type != "cuda":
+ continue
+ pieces = _pieces(name, whole) if _placeable(whole) else [(name, whole)]
+ if any(lo <= x.data_ptr() < hi for _, x in pieces):
+ return True
+ except Exception: # noqa: BLE001
+ return True
+ return False
+
+ @staticmethod
+ def _on_host(module: Any) -> bool:
+ for t in module.parameters():
+ return t.device.type == "cpu"
+ return False
+
+ def _onload(self, module: Any, non_blocking: bool) -> None:
+ import torch
+
+ entries = _entries(module)
+ if self.layout is None:
+ self._build(entries)
+ assert self.arena is not None and self.layout is not None
+ self.arena.map()
+ placed = 0
+ filled: dict = {} # slot offset -> (host tensor copied into it during this onload, its view)
+ for name, regs, kind, whole in entries:
+ if whole.device.type != "cpu" or not _placeable(whole):
+ continue
+ pieces = _pieces(name, whole)
+ slots = [self.layout.get(key) for key, _ in pieces]
+ if any(
+ slot is None
+ or (tuple(t.shape), tuple(t.stride()), t.dtype) != (slot[1], slot[2], slot[3])
+ for slot, (_, t) in zip(slots, pieces)
+ ):
+ continue
+ # A slot already filled by another registration of a once-shared buffer is reused only when the copies
+ # still hold the same values; otherwise this one keeps the stock move.
+ if any(
+ slot[0] in filled and not torch.equal(filled[slot[0]][0], t)
+ for slot, (_, t) in zip(slots, pieces)
+ ):
+ continue
+ views = []
+ for slot, (_, t) in zip(slots, pieces):
+ if slot[0] in filled:
+ # the same view object: the registrations share one tensor again, as on the first onload, so the
+ # graph's placement fingerprint (one entry per tensor) does not change
+ views.append(filled[slot[0]][1])
+ continue
+ view = self.arena.view(slot[0], slot[1], slot[3], slot[2])
+ view.copy_(t, non_blocking = non_blocking)
+ filled[slot[0]] = (t, view)
+ views.append(view)
+ if _plain(whole):
+ for owner, leaf, _qual in regs:
+ _set_tensor(owner, leaf, kind, views[0])
+ else:
+ # a wrapper weight is rebuilt over its placed inner tensors and swapped in, as Module.to moves it
+ names, ctx = whole.__tensor_flatten__()
+ rebuilt = type(whole).__tensor_unflatten__(
+ dict(zip(names, views)), ctx, whole.shape, whole.stride()
+ )
+ if kind == "p":
+ torch.utils.swap_tensors(
+ whole, torch.nn.Parameter(rebuilt, requires_grad = whole.requires_grad)
+ )
+ else:
+ for owner, leaf, _qual in regs:
+ owner._buffers[leaf] = rebuilt
+ placed += 1
+ self.stats["placed"] = placed
+
+ def _to(self, *args: Any, **kwargs: Any) -> Any:
+ import torch
+
+ module = self._module()
+ try:
+ device, dtype, non_blocking, fmt = torch._C._nn._parse_to(*args, **kwargs)
+ except Exception: # noqa: BLE001 - let the stock call raise its own error
+ device = dtype = fmt = None
+ non_blocking = False
+ if (
+ module is not None
+ and self.failed is None
+ and dtype is None
+ and fmt is None
+ and device is not None
+ and device.type == "cuda"
+ and (device.index if device.index is not None else torch.cuda.current_device())
+ == self.device.index
+ ):
+ try:
+ # accelerate calls ``to`` on every forward (its device check compares "cuda" with "cuda:0"): only a
+ # module whose weights are on the host is placed
+ if self._on_host(module):
+ self._onload(module, True)
+ self.stats["onloads"] += 1
+ except Exception as exc: # noqa: BLE001 - the stock move still places the module
+ self.failed = f"{type(exc).__name__}: {exc}"
+ if self.logger is not None:
+ self.logger.warning(
+ "diffusion.stable_arena: onload into the stable range failed (%s)", exc
+ )
+ self._release(module)
+ out = self._stock_to(module, *args, **kwargs)
+ if module is not None:
+ self._release(module)
+ return out
+
+ def _release(self, module: Any) -> None:
+ """Unmap the range once no tensor of the module lives in it (its weights went back to the host)."""
+ if self.arena is None or not self.arena.mapped or self._referenced(module):
+ return
+ self.arena.unmap()
+ self.stats["unmaps"] += 1
+
+ def describe(self) -> dict:
+ arena = self.arena
+ return {
+ **self.stats,
+ "mib": None if arena is None else arena.size >> 20,
+ "mapped": bool(arena is not None and arena.mapped),
+ "failed": self.failed,
+ }
+
+
+def attach(
+ module: Any,
+ device: Any = None,
+ logger: Any = None,
+) -> tuple:
+ """``(arena, None)`` when ``module``'s onloads will land in one stable range from now on, else ``(None, why)``."""
+ if arena_disabled():
+ return None, f"disabled by {ARENA_ENV}=0"
+ existing = module.__dict__.get(_ARENA_ATTR)
+ if existing is not None:
+ return (existing, None) if existing.failed is None else (None, existing.failed)
+ try:
+ import torch
+ if not torch.cuda.is_available():
+ return None, "no CUDA device"
+ _driver()
+ except Exception as exc: # noqa: BLE001
+ return None, f"CUDA virtual memory unavailable ({exc})"
+ odd = [name for name, _r, _k, t in _entries(module) if not _placeable(t)]
+ if odd:
+ return None, f"{len(odd)} parameter(s) are not plain tensors ({odd[0]})"
+ if device is None:
+ hook = getattr(module, "_hf_hook", None)
+ device = getattr(hook, "execution_device", None)
+ if device is None:
+ return None, "no execution device"
+ try:
+ arena = ModuleArena(module, device, logger)
+ except Exception as exc: # noqa: BLE001
+ return None, f"stable range unavailable ({exc})"
+ module.__dict__[_ARENA_ATTR] = arena
+ return arena, None
+
+
+def arena_of(module: Any) -> Optional[ModuleArena]:
+ try:
+ return module.__dict__.get(_ARENA_ATTR)
+ except Exception: # noqa: BLE001
+ return None
diff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
index 6d9493245b3..b13d1d47050 100644
--- a/studio/backend/tests/test_diffusion_offload_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py
@@ -17,6 +17,7 @@ import core.inference.diffusion_cuda_graph as cg # noqa: E402
import core.inference.diffusion_memory as dm # noqa: E402
import core.inference.diffusion_offload_prefetch as op # noqa: E402
import core.inference.diffusion_speed as ds # noqa: E402
+import core.inference.diffusion_stable_arena as sa # noqa: E402
@pytest.fixture(autouse = True)
@@ -27,6 +28,7 @@ def _clean_env(monkeypatch):
cg.OFFLOAD_CUDA_GRAPH_ENV,
cg.CUDA_GRAPH_DISABLE_ENV,
cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV,
+ sa.ARENA_ENV,
cg.SPEED_CHECK_ENV,
"UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",
"UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",
@@ -282,9 +284,10 @@ def _model_offloaded():
return ref, net, hook
-def test_model_offload_keeps_the_step_eager_by_default_with_the_reason():
+def test_model_offload_keeps_the_step_eager_without_the_stable_range(monkeypatch):
_cuda()
pytest.importorskip("accelerate")
+ monkeypatch.setenv(sa.ARENA_ENV, "0")
_ref, net, _hook = _model_offloaded()
placement, why = cg.offload_placement(net)
assert placement is None and why == cg.MODEL_OFFLOAD_REASON
@@ -302,7 +305,7 @@ def test_model_offload_records_nothing_after_the_first_render(monkeypatch, opt_i
monkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")
ref, net, hook = _model_offloaded()
handles, reason = cg.arm_after_placement(_pipe(net))
- assert len(handles) == (1 if opt_in else 0), reason
+ assert len(handles) == 1, reason
dev = torch.cuda.current_device()
later, hold = 0, []
for render in range(4):
@@ -320,7 +323,65 @@ def test_model_offload_records_nothing_after_the_first_render(monkeypatch, opt_i
if opt_in:
assert later >= 1 and handles[0].stats["invalidations"] >= 1
else:
- assert later == 0
+ # the stable range: the step recorded on render 1 replays on every later render
+ assert later == 0 and handles[0].stats["invalidations"] == 0
+ assert handles[0].stats["replays"] >= 4 * 5 - cg.WARMUP_ITERS - 1
+
+
+def test_model_offload_replays_over_the_stable_range_and_frees_it_between_renders():
+ """low_vram: the denoiser's onloads land in one reserved VA range, so render 1's graph replays on renders 2+ with
+ no capture, bit-identical, while each offload returns the range's memory to the card for the other stages."""
+ _cuda()
+ pytest.importorskip("accelerate")
+ from accelerate import cpu_offload_with_hook
+
+ torch.manual_seed(0)
+ ref = _Net(width = 1024).cuda()
+ torch.manual_seed(0)
+ net, hook = cpu_offload_with_hook(_Net(width = 1024), execution_device = "cuda")
+ other, other_hook = cpu_offload_with_hook(
+ torch.nn.Linear(1024, 1024), execution_device = "cuda", prev_module_hook = hook
+ )
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == 1 and "stable device range" in reason, reason
+ arena = sa.arena_of(net)
+ assert arena is not None
+ dev = torch.cuda.current_device()
+ weight_bytes = sum(p.numel() * p.element_size() for p in net.parameters())
+ ptrs, later = None, 0
+ for render in range(4):
+ seeds = [10 * render + i for i in range(5)]
+ before = cg.capture_generation(dev)
+ got = _run(net, seeds)
+ if render:
+ later += cg.capture_generation(dev) - before
+ now = tuple(p.data_ptr() for p in net.parameters())
+ assert ptrs is None or now == ptrs # same addresses on every onload
+ ptrs = now
+ for a, b in zip(got, _run(ref, seeds)):
+ assert torch.equal(a, b)
+ other(torch.zeros(2, 1024)) # the next stage's pre-forward offloads the denoiser
+ assert net.proj_in.weight.device.type == "cpu"
+ assert not arena.arena.mapped # its physical memory went back to the card
+ assert arena.arena.size >= weight_bytes
+ other_hook.offload()
+ torch.cuda.empty_cache()
+ _hold = torch.empty(4 << 20, device = "cuda") # the freed VRAM is reused meanwhile
+ del _hold
+ # diffusers re-hooks every render (remove_hook_from_module also drops instance ``to`` overrides)
+ from accelerate.hooks import remove_hook_from_module
+
+ remove_hook_from_module(net)
+ remove_hook_from_module(other)
+ net, hook = cpu_offload_with_hook(net, execution_device = "cuda")
+ other, other_hook = cpu_offload_with_hook(
+ other, execution_device = "cuda", prev_module_hook = hook
+ )
+ assert later == 0
+ assert handles[0].stats["captures"] == 1 and handles[0].stats["invalidations"] == 0
+ assert arena.stats["onloads"] == 4 and arena.stats["unmaps"] == 4
+ assert arena.stats.get("reinstalls", 0) >= 3
+ cg.uninstall_all(handles)
def test_model_offload_stops_recording_only_when_moves_outpace_replays():
@@ -765,3 +826,158 @@ def test_status_names_graphs_dropped_after_replaying():
handle.cache = {"k": object()}
handle.capture_error = None
assert cg.live_status(resolved, ["cuda_graph"], [handle])[0]["cuda_graph"]["value"] == "on"
+
+
+def _inner_of(t):
+ """A torchao wrapper weight's inner tensors, read without the module under test."""
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
+
+ if not is_traceable_wrapper_subclass(t):
+ return None
+ names, _ctx = t.__tensor_flatten__()
+ inner = [(n, getattr(t, n)) for n in names]
+ return inner if all(type(x) is torch.Tensor for _, x in inner) else None
+
+
+def test_model_offload_stable_range_holds_torchao_int8_weights():
+ """Studio's compiled low_vram tier quantizes the denoiser to torchao int8: those wrapper weights are rebuilt over
+ the stable range too (same inner addresses on every onload, memory released between renders), exactly.
+ The eager torchao int8 matmul here is not capture-safe, so this checks placement, not the replay; the compiled
+ Studio step records over it (scripts/graphs_split_offload)."""
+ _cuda()
+ pytest.importorskip("accelerate")
+ quant = pytest.importorskip("torchao.quantization")
+ from accelerate import cpu_offload_with_hook
+
+ def build():
+ torch.manual_seed(0)
+ net = _Net(width = 256).to(torch.bfloat16)
+ try:
+ config = quant.Int8DynamicActivationInt8WeightConfig(version = 2)
+ except TypeError:
+ config = quant.Int8DynamicActivationInt8WeightConfig()
+ quant.quantize_(net.blocks, config)
+ return net
+
+ probe = build()
+ if _inner_of(probe.blocks[0].weight.data) is None:
+ pytest.skip("this torchao build stores int8 weights without a traceable wrapper")
+ ref = build().cuda()
+ net, hook = cpu_offload_with_hook(build(), execution_device = "cuda")
+ arena, why = sa.attach(net)
+ assert arena is not None, why
+
+ def run(module, seeds):
+ outs = []
+ with torch.no_grad():
+ for seed in seeds:
+ x, t = _inputs(seed)
+ outs.append(
+ module(x.to(torch.bfloat16), t.to(torch.bfloat16), return_dict = False)[0].clone()
+ )
+ torch.cuda.synchronize()
+ return outs
+
+ def inner_ptrs():
+ return [x.data_ptr() for b in net.blocks for _, x in _inner_of(b.weight.data)]
+
+ ptrs = None
+ for render in range(3):
+ seeds = [10 * render + i for i in range(3)]
+ got = run(net, seeds)
+ assert type(net.blocks[0].weight.data) is type(probe.blocks[0].weight.data)
+ now = inner_ptrs()
+ lo, hi = arena.arena.va, arena.arena.va + arena.arena.size
+ assert all(lo <= q < hi for q in now)
+ assert ptrs is None or now == ptrs
+ ptrs = now
+ for a, b in zip(got, run(ref, seeds)):
+ assert torch.equal(a, b)
+ hook.offload()
+ assert net.blocks[0].weight.device.type == "cpu"
+ assert not arena.arena.mapped
+ assert arena.stats["onloads"] == 3 and arena.stats["unmaps"] == 3
+
+
[email protected]("where", ["forward_refusal", "placement"])
+def test_a_refused_model_offload_leaves_the_stock_onload(monkeypatch, where):
+ """A denoiser whose step stays eager keeps accelerate's own onload: no stable range, no instance ``to``, no
+ re-install hook (so no per-offload card sync for a step that never replays)."""
+ _cuda()
+ pytest.importorskip("accelerate")
+ _ref, net, _hook = _model_offloaded()
+ if where == "forward_refusal":
+ monkeypatch.setattr(cg, "_forward_refusal", lambda module, mode: "not capture-safe")
+ else:
+
+ def boom(*a, **k):
+ raise RuntimeError("no hook forward")
+
+ monkeypatch.setattr(cg, "OffloadPlacement", boom)
+ placement, why = cg.offload_placement(net)
+ assert placement is None and why
+ assert sa.arena_of(net) is None
+ assert "to" not in net.__dict__
+ assert not net._forward_pre_hooks
+
+
+class _SharedBuffer(_Net):
+ """One buffer tensor registered on two submodules, as tied rotary / scale tables are."""
+
+ def __init__(self):
+ super().__init__(width = 512)
+ scale = torch.linspace(0.5, 1.5, 512)
+ self.blocks[0].register_buffer("scale", scale)
+ self.blocks[-1].register_buffer("scale", scale)
+
+ def forward(
+ self,
+ x,
+ t,
+ return_dict = True,
+ ):
+ x = self.proj_in(x) * t
+ for i, block in enumerate(self.blocks):
+ x = torch.nn.functional.gelu(block(x))
+ if i in (0, len(self.blocks) - 1):
+ x = x * block.scale
+ out = self.proj_out(x)
+ return (out,) if not return_dict else {"sample": out}
+
+
+def test_model_offload_stable_range_places_every_registration_of_a_shared_buffer():
+ _cuda()
+ pytest.importorskip("accelerate")
+ from accelerate import cpu_offload_with_hook
+ from accelerate.hooks import remove_hook_from_module
+
+ torch.manual_seed(0)
+ ref = _SharedBuffer().cuda()
+ torch.manual_seed(0)
+ net, hook = cpu_offload_with_hook(_SharedBuffer(), execution_device = "cuda")
+ handles, reason = cg.arm_after_placement(_pipe(net))
+ assert len(handles) == 1 and "stable device range" in reason, reason
+ arena = sa.arena_of(net)
+ later = 0
+ dev = torch.cuda.current_device()
+ for render in range(4):
+ seeds = [10 * render + i for i in range(5)]
+ before = cg.capture_generation(dev)
+ got = _run(net, seeds)
+ if render:
+ later += cg.capture_generation(dev) - before
+ first, last = net.blocks[0].scale, net.blocks[-1].scale
+ lo, hi = arena.arena.va, arena.arena.va + arena.arena.size
+ # Module.to() splits a shared buffer into two host copies; the onload shares one placed tensor again
+ assert first is last and lo <= first.data_ptr() < hi
+ for a, b in zip(got, _run(ref, seeds)):
+ assert torch.equal(a, b)
+ hook.offload()
+ assert net.blocks[-1].scale.device.type == "cpu" and not arena.arena.mapped
+ torch.cuda.empty_cache()
+ _hold = torch.empty(4 << 20, device = "cuda") # the freed VRAM is reused meanwhile
+ del _hold
+ remove_hook_from_module(net)
+ net, hook = cpu_offload_with_hook(net, execution_device = "cuda")
+ assert later == 0 and handles[0].stats["invalidations"] == 0, handles[0].stats
+ cg.uninstall_all(handles)
diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
index 958edec5113..9b4cacf1400 100644
--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py
+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
@@ -608,3 +608,65 @@ def test_buffers_onloaded_under_inference_mode_do_not_break_the_hook():
for _ in range(3):
with torch.inference_mode():
assert torch.equal(_render(pipe, x), ref)
+
+
+def _int8_pipe(device):
+ """The _pipe layout with torchao Int8Tensor weights (Studio's int8 denoisers under low_vram)."""
+ quant = pytest.importorskip("torchao.quantization")
+ pipe = _pipe(device, "text_encoder", "transformer")
+ lin = torch.nn.Linear(64, 64, dtype = torch.bfloat16)
+ torch.manual_seed(1)
+ torch.nn.init.normal_(lin.weight)
+ try:
+ config = quant.Int8DynamicActivationInt8WeightConfig(version = 2)
+ except TypeError:
+ config = quant.Int8DynamicActivationInt8WeightConfig()
+ quant.quantize_(torch.nn.Sequential(lin), config)
+ if _inner_of(lin.weight.data) is None:
+ pytest.skip("this torchao build stores int8 weights without a traceable wrapper")
+ pipe.components["text_encoder"] = torch.nn.Linear(8, 64, dtype = torch.bfloat16)
+ pipe.components["transformer"] = lin
+ pipe.text_encoder, pipe.transformer = pipe.components["text_encoder"], lin
+ return pipe
+
+
+def _inner_ptrs(module):
+ return [x.data_ptr() for _, x in _inner_of(module.weight.data)]
+
+
+def _inner_of(t):
+ """A torchao wrapper weight's inner tensors, read without the module under test."""
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
+
+ if not is_traceable_wrapper_subclass(t):
+ return None
+ names, _ctx = t.__tensor_flatten__()
+ inner = [(n, getattr(t, n)) for n in names]
+ return inner if all(type(x) is torch.Tensor for _, x in inner) else None
+
+
+@cuda
[email protected]("pin", ["0", "1"])
+def test_torchao_int8_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):
+ """Each offload re-points the int8 weight at its kept host tensors (no device-to-host copy, no new host
+ allocation), the inner tensors are pinned once, and every render matches the stock offload bit for bit."""
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)
+ x = torch.randn(4, 8, dtype = torch.bfloat16)
+ stock = _int8_pipe("cuda")
+ stock.enable_model_cpu_offload()
+ ref = _render(stock, x)
+ pipe = _int8_pipe("cuda")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ assert _wrapped(pipe.transformer)
+ outs, host = [], None
+ for _ in range(3):
+ outs.append(_render(pipe, x))
+ assert pipe.transformer.weight.device.type == "cpu"
+ now = _inner_ptrs(pipe.transformer)
+ assert host is None or now == host # the same host tensors every render
+ host = now
+ for out in outs:
+ assert torch.equal(out, ref)
+ inner = _inner_of(pipe.transformer.weight.data)
+ assert all(t.is_pinned() for _, t in inner) == (pin == "1")