Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: replay a model-offloaded denoiser's CUDA graph across renders and pin torchao int8 host weights (#14666)
Daniel Hancaa55746e5ae
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 877b44dbc14..1a319ba973c 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -33,7 +33,9 @@ OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_OFFLOAD_CUDA_GRAPH"# =0: a block-streamed whole-step graph records its copies into the graph pool instead of the prefetcher's slot ring.STEP_SLOTS_ENV = "UNSLOTH_DIFFUSION_STEP_GRAPH_SLOTS"MODEL_OFFLOAD_CUDA_GRAPH_ENV = "UNSLOTH_DIFFUSION_MODEL_OFFLOAD_CUDA_GRAPH"-# Default off: replay saved ~1% of a step (FLUX.2-klein 4B, 1024^2) but the per-render re-record + ~0.6 GB pool cost more.+# =1: record over the stock onloads, re-recording whenever they move (replay saved ~1% of a step on FLUX.2-klein 4B+# 1024^2 while each re-record + ~0.6 GB pool cost more). Default: record only over the stable range, see+# diffusion_stable_arena.MODEL_OFFLOAD_REASON = ("model offload re-uploads the denoiser every render at new addresses, so its step graph would record again on ""every render; set " + MODEL_OFFLOAD_CUDA_GRAPH_ENV + "=1 to record it anyway"@@ -1728,7 +1730,9 @@ class OffloadPlacement:and the resident groups read in place. The pool holds the in-flight window at fixed addresses, so it plays theprefetch ring. Recorded again when a residency change, a release / restore or a host-copy swap moves an address.- ``model``: whole-module CPU offload (accelerate), only with ``MODEL_OFFLOAD_CUDA_GRAPH_ENV=1`` (see+ ``model``: whole-module CPU offload (accelerate). By default only over the stable device range+ (``diffusion_stable_arena``: every onload lands at the same addresses, so render 1's recording replays on every+ later render); ``MODEL_OFFLOAD_CUDA_GRAPH_ENV=1`` records over the stock onloads instead (see``MODEL_OFFLOAD_REASON``). The hook moves the module on the eager path first and the replay sits under it, in``_old_forward``. Recorded again when the onload placed the weights at new addresses; after``MODEL_MAX_INVALIDATIONS`` re-records the module runs eager (each re-record costs a forward)."""@@ -2002,14 +2006,23 @@ def offload_placement(module: Any) -> tuple:None,f"offload hook {type(hf_hook).__name__} moves the weights inside the forward",)- if not model_offload_graphs_requested():- return None, MODEL_OFFLOAD_REASON+ # Every other refusal first: the stable range below swaps the module's onload, so it attaches only for a+ # placement that will record.why = _forward_refusal(module, "model")if why is not None:return None, why+ arena = None+ if not model_offload_graphs_requested():+ # Recorded only where the onloads land at the same addresses every render (diffusion_stable_arena).+ from .diffusion_stable_arena import attach+ arena, _arena_why = attach(module)+ if arena is None:+ return None, MODEL_OFFLOAD_REASONtry:return OffloadPlacement(module, "model"), Noneexcept Exception as exc: # noqa: BLE001+ if arena is not None:+ arena.detach()return None, f"model offload hook unreadable ({exc})"registry = getattr(module, "_diffusers_hook", None)hooks = getattr(registry, "hooks", None) or {}@@ -2049,6 +2062,25 @@ def offload_placement(module: Any) -> tuple:return placement, None+def _detach_stable_range(module: Any) -> None:+ """Give a module whose step stays eager its stock onload back (no stable range, no per-offload card sync)."""+ try:+ from .diffusion_stable_arena import arena_of+ arena = arena_of(module)+ if arena is not None:+ arena.detach()+ except Exception: # noqa: BLE001+ pass+++def _stable_range(module: Any) -> bool:+ try:+ from .diffusion_stable_arena import arena_of+ return arena_of(module) is not None+ except Exception: # noqa: BLE001+ return False++def _forward_refusal(module: Any, mode: str) -> Optional[str]:"""Why ``module``'s own forward cannot record under ``mode`` offload, else None (see GraphedForward.__init__)."""try:@@ -2144,8 +2176,11 @@ def arm_after_placement(module, max_graphs = max_graphs, logger = logger, placement = placement).enable())- modes.append(placement.mode)+ stable = placement.mode == "model" and _stable_range(module)+ modes.append("model_stable" if stable else placement.mode)except Exception as exc: # noqa: BLE001+ if placement is not None and placement.mode == "model":+ _detach_stable_range(module)refusals.append(f"{type(module).__name__}: {exc}")installed = tuple(handles)try:@@ -2159,6 +2194,7 @@ def arm_after_placement("resident": "resident","group": "block-streamed (copies recorded in the graph)","model": "model offload",+ "model_stable": "model offload over a stable device range",}reason = "captured per input shape: " + ", ".join(sorted({labels.get(m, m) for m in modes}))if logger is not None:@@ -2435,6 +2471,11 @@ def stats(handles: Any) -> dict:out["replay_ms"] = float(handle.stats["replay_ms"])placement = getattr(handle, "placement", None)out["placements"].append("resident" if placement is None else placement.mode)+ if placement is not None and placement.mode == "model":+ from .diffusion_stable_arena import arena_of+ arena = arena_of(placement.module)+ if arena is not None:+ out["stable_range"] = arena.describe()if handle.poisoned:out["poisoned"] = Trueerror = handle.capture_errordiff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 453b8c961b7..e8ee9cc48a7 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -176,12 +176,56 @@ def _gguf_parameter_class() -> Optional[type]:return getattr(module, "GGUFParameter", None) if module is not None else None+def _wrapper_inner(t: Any) -> Optional[list]:+ """``[(attr, tensor), ...]`` of a traceable wrapper subclass (torchao int8 / fp8 weights) whose inner tensors are+ all plain tensors, else None."""+ import torch++ flatten = getattr(type(t), "__tensor_flatten__", None)+ if flatten is None or type(t) in (torch.Tensor, torch.nn.Parameter):+ return None+ try:+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass++ if not is_traceable_wrapper_subclass(t):+ return None+ names, _ctx = t.__tensor_flatten__()+ inner = [(name, getattr(t, name)) for name in names]+ except Exception: # noqa: BLE001+ return None+ if not inner or any(type(x) is not torch.Tensor for _, x in inner):+ return None+ return inner+++def _tensor_ptr(t: Any) -> Any:+ """Storage address of ``t``; for a wrapper subclass, the tuple of its inner tensors' addresses."""+ inner = _wrapper_inner(t)+ if inner is not None:+ return tuple(int(x.data_ptr()) for _, x in inner)+ return t.data_ptr()+++def _repoint(param: Any, kept: Any) -> None:+ """Point ``param`` at its kept host tensor without a copy. A wrapper subclass cannot change device through+ ``.data``; it swaps with a parameter over the kept tensors, as ``Module.to`` moves it."""+ if _wrapper_inner(kept) is None:+ param.data = kept+ return+ import torch++ torch.utils.swap_tensors(param, torch.nn.Parameter(kept, requires_grad = param.requires_grad))++def _keepable(param: Any) -> bool:import torchdata = param.dataif type(data) is torch.Tensor:return True+ # torchao weights (int8 / fp8): a wrapper of plain tensors; kept and re-pointed like a dense weight.+ if _wrapper_inner(data) is not None and type(data) is type(param):+ return True# GGUF weights are packed bytes; the quant type lives on the Parameter, which the swap never replaces.gguf = _gguf_parameter_class()return gguf is not None and type(param) is gguf and type(data) is gguf@@ -267,15 +311,19 @@ def _pin_host_weights(mode = _pin_mode()if mode == "off" or not torch.cuda.is_available():return 0- params = [- (name, p)- for name, p in module.named_parameters()- if host.get(name) is not None- and p.device.type == "cpu"- and _keepable(p)- and _pinnable_layout(p.data)- and not p.data.is_pinned()- ]+ # (name, owner tensor, inner attribute or None, the tensor to pin)+ params: list = []+ for name, p in module.named_parameters():+ if host.get(name) is None or p.device.type != "cpu" or not _keepable(p):+ continue+ inner = _wrapper_inner(p.data)+ if inner is None:+ if _pinnable_layout(p.data) and not p.data.is_pinned():+ params.append((name, p, None, p.data))+ continue+ for attr, t in inner:+ if t.device.type == "cpu" and _pinnable_layout(t) and not t.is_pinned():+ params.append((name, p, attr, t))buffers = [(name, b)for name, b in _named_buffers(module)@@ -285,8 +333,8 @@ def _pin_host_weights(and b.is_contiguous()and not b.is_pinned()]- params = params + [("\0" + name, b) for name, b in buffers]- sizes = [-(-p.data.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, p in params]+ params = params + [("\0" + name, b, None, b) for name, b in buffers]+ sizes = [-(-t.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, _, _, t in params]if not sizes or sum(sizes) == 0:return 0# Lay out chunks first so the RAM gate counts chunk tails and last-chunk rounding, not just weight bytes.@@ -322,17 +370,17 @@ def _pin_host_weights(bufs: list = []placed: list = []try:- for (name, p), (index, start) in zip(params, slots):+ for (name, p, attr, src), (index, start) in zip(params, slots):if index == len(bufs):bufs.append(torch.empty(chunks[index], dtype = torch.uint8, pin_memory = True))- view = bufs[index][start : start + p.data.nbytes].view(p.dtype)+ view = bufs[index][start : start + src.nbytes].view(src.dtype)view = (- view.view(p.shape)- if p.data.is_contiguous()- else view.as_strided(p.shape, p.data.stride())+ view.view(src.shape)+ if src.is_contiguous()+ else view.as_strided(src.shape, src.stride()))- view.copy_(p.data)- placed.append((name, p, view))+ view.copy_(src)+ placed.append((name, p, attr, view))except Exception as exc: # noqa: BLE001 - e.g. a WSL pinned-memory cap: keep the pageable weightsbufs.clear()placed.clear()@@ -345,14 +393,21 @@ def _pin_host_weights(exc,)return 0- for name, p, view in placed:+ wrapped: dict = {}+ for name, p, attr, view in placed:if name.startswith("\0"):name = name[1:]_set_buffer(module, name, view)buffer_host[name] = viewcontinue+ if attr is not None:+ setattr(p, attr, view)+ wrapped[name] = p+ continuep.data = viewhost[name] = view+ for name, p in wrapped.items():+ host[name] = p.data # the kept host weight now reads the pinned inner tensorsreturn need@@ -441,11 +496,11 @@ def _wrap_cpu_offload_hook(or seen is Noneor seen[0] is not por seen[1] != p._version- or seen[2] != p.data_ptr()+ or seen[2] != _tensor_ptr(p)):continuetry:- p.data = kept+ _repoint(p, kept)except Exception: # noqa: BLE001 - incompatible tensor types: the stock copy below handles itpassout = init_hook(mod)@@ -463,7 +518,7 @@ def _wrap_cpu_offload_hook(if kept is not None and (owner.get(name) is not por p.device.type != "cpu"- or p.data.data_ptr() != kept.data_ptr()+ or _tensor_ptr(p.data) != _tensor_ptr(kept)):host.pop(name, None)owner.pop(name, None)@@ -488,7 +543,7 @@ def _wrap_cpu_offload_hook(if onload:for name, p in mod.named_parameters():if name in host and owner.get(name) is p and p.device.type != "cpu":- version[name] = (p, p._version, p.data_ptr())+ version[name] = (p, p._version, _tensor_ptr(p))for name, b in _named_buffers(mod):if (name in buffer_hostdiff --git a/studio/backend/core/inference/diffusion_stable_arena.py b/studio/backend/core/inference/diffusion_stable_arena.pynew file mode 100644index 00000000000..2ac22ccda8b--- /dev/null+++ b/studio/backend/core/inference/diffusion_stable_arena.py@@ -0,0 +1,586 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Stable device addresses for a whole-module CPU-offloaded denoiser (``memory_mode=low_vram``).++accelerate's ``CpuOffload`` hook onloads the denoiser with ``module.to(device)`` at the start of every render, so the+caching allocator hands its weights new addresses each time and a CUDA graph recorded on render 1 cannot replay on+render 2 (``diffusion_cuda_graph.MODEL_OFFLOAD_REASON``). Here the denoiser's device copy lives in one CUDA virtual+address range reserved once for the load (driver VMM: ``cuMemAddressReserve``). An onload creates physical memory,+maps it at that range and copies the kept host weights in; the offload syncs the card, unmaps and releases the+physical memory, so the text encoder and VAE phases get the VRAM back exactly as with the stock hook. The weights+land at the same addresses on every render, so the recorded step graph keeps replaying.++ComfyUI does the same for its graph replays over offloaded weights (comfy_aimdo ``model_vbar``).++Plain dense tensors (with their strides) and torchao-style wrapper weights (int8 / fp8: their plain inner tensors,+rebuilt over the range) are placed; a module holding any other parameter type (GGUF) is left to the stock hook.+Opt out with ``UNSLOTH_DIFFUSION_MODEL_OFFLOAD_ARENA=0``.+"""++from __future__ import annotations++import ctypes+import os+import threading+import weakref+from typing import Any, Optional++ARENA_ENV = "UNSLOTH_DIFFUSION_MODEL_OFFLOAD_ARENA"+_ARENA_ATTR = "_unsloth_stable_arena"+_ALIGN = 512+_OFF = ("0", "off", "false", "no")++# CUDA driver enums+_CU_MEM_ALLOCATION_TYPE_PINNED = 1+_CU_MEM_LOCATION_TYPE_DEVICE = 1+_CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 3+_CU_MEM_ALLOC_GRANULARITY_RECOMMENDED = 1+++class _Location(ctypes.Structure):+ _fields_ = [("type", ctypes.c_int), ("id", ctypes.c_int)]+++class _AllocFlags(ctypes.Structure):+ _fields_ = [+ ("compressionType", ctypes.c_ubyte),+ ("gpuDirectRDMACapable", ctypes.c_ubyte),+ ("usage", ctypes.c_ushort),+ ("reserved", ctypes.c_ubyte * 4),+ ]+++class _AllocProp(ctypes.Structure):+ _fields_ = [+ ("type", ctypes.c_int),+ ("requestedHandleTypes", ctypes.c_int),+ ("location", _Location),+ ("win32HandleMetaData", ctypes.c_void_p),+ ("allocFlags", _AllocFlags),+ ]+++class _AccessDesc(ctypes.Structure):+ _fields_ = [("location", _Location), ("flags", ctypes.c_int)]+++_LIB: Any = None+_LIB_LOCK = threading.Lock()+++def _driver() -> Any:+ global _LIB+ with _LIB_LOCK:+ if _LIB is None:+ lib = None+ for name in ("libcuda.so.1", "libcuda.so", "nvcuda.dll"):+ try:+ lib = ctypes.CDLL(name)+ break+ except OSError:+ continue+ if lib is None:+ raise OSError("CUDA driver library not found")+ u64, sz = ctypes.c_uint64, ctypes.c_size_t+ for fn, args in (+ (+ "cuMemGetAllocationGranularity",+ [ctypes.POINTER(sz), ctypes.POINTER(_AllocProp), ctypes.c_int],+ ),+ ("cuMemAddressReserve", [ctypes.POINTER(u64), sz, sz, u64, ctypes.c_ulonglong]),+ ("cuMemAddressFree", [u64, sz]),+ (+ "cuMemCreate",+ [+ ctypes.POINTER(ctypes.c_ulonglong),+ sz,+ ctypes.POINTER(_AllocProp),+ ctypes.c_ulonglong,+ ],+ ),+ ("cuMemRelease", [ctypes.c_ulonglong]),+ ("cuMemMap", [u64, sz, sz, ctypes.c_ulonglong, ctypes.c_ulonglong]),+ ("cuMemUnmap", [u64, sz]),+ ("cuMemSetAccess", [u64, sz, ctypes.POINTER(_AccessDesc), sz]),+ ("cuCtxGetCurrent", [ctypes.POINTER(ctypes.c_void_p)]),+ ):+ f = getattr(lib, fn)+ f.argtypes = args+ f.restype = ctypes.c_int+ _LIB = lib+ return _LIB+++def _check(rc: int, what: str) -> None:+ if rc != 0:+ raise RuntimeError(f"{what} failed (CUresult {rc})")+++def arena_disabled() -> bool:+ return (os.environ.get(ARENA_ENV) or "").strip().lower() in _OFF+++class _CudaArray:+ """A ``__cuda_array_interface__`` over raw device bytes, so ``torch.as_tensor`` wraps them without a copy."""++ def __init__(self, ptr: int, nbytes: int) -> None:+ self.__cuda_array_interface__ = {+ "shape": (int(nbytes),),+ "typestr": "|u1",+ "data": (int(ptr), False),+ "version": 3,+ "strides": None,+ "stream": None,+ }+++class VmmArena:+ """One reserved device VA range of ``nbytes`` (rounded to the allocation granularity) on ``device_index``; physical+ memory exists only between ``map()`` and ``unmap()``."""++ def __init__(self, nbytes: int, device_index: int) -> None:+ import torch++ self.device_index = int(device_index)+ self.device = torch.device("cuda", self.device_index)+ lib = _driver()+ self._ensure_context()+ prop = self._prop()+ gran = ctypes.c_size_t(0)+ _check(+ lib.cuMemGetAllocationGranularity(+ ctypes.byref(gran), ctypes.byref(prop), _CU_MEM_ALLOC_GRANULARITY_RECOMMENDED+ ),+ "cuMemGetAllocationGranularity",+ )+ self.granularity = int(gran.value) or (2 << 20)+ self.size = -(-int(nbytes) // self.granularity) * self.granularity+ va = ctypes.c_uint64(0)+ _check(+ lib.cuMemAddressReserve(ctypes.byref(va), self.size, self.granularity, 0, 0),+ "cuMemAddressReserve",+ )+ self.va = int(va.value)+ self._handle: Optional[int] = None+ self._base: Any = None+ self.maps = 0+ self.unmaps = 0+ self.closed = False++ def _prop(self) -> _AllocProp:+ prop = _AllocProp()+ prop.type = _CU_MEM_ALLOCATION_TYPE_PINNED+ prop.location.type = _CU_MEM_LOCATION_TYPE_DEVICE+ prop.location.id = self.device_index+ return prop++ def _ensure_context(self) -> None:+ import torch++ ctx = ctypes.c_void_p(0)+ _driver().cuCtxGetCurrent(ctypes.byref(ctx))+ if not ctx.value:+ # The calling thread has no current context yet: any runtime call on the device makes the primary one current.+ with torch.cuda.device(self.device_index):+ torch.cuda.current_stream().query()++ @property+ def mapped(self) -> bool:+ return self._handle is not None++ def map(self) -> Any:+ """Back the range with new physical memory; returns a uint8 tensor over the whole range."""+ import torch++ if self.closed:+ raise RuntimeError("arena closed")+ if self._handle is not None:+ return self._base+ lib = _driver()+ with torch.cuda.device(self.device_index):+ self._ensure_context()+ prop = self._prop()+ handle = ctypes.c_ulonglong(0)+ rc = lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)+ if rc != 0:+ # The caching allocator may hold the memory as free cached blocks.+ torch.cuda.empty_cache()+ rc = lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)+ _check(rc, "cuMemCreate")+ try:+ _check(lib.cuMemMap(self.va, self.size, 0, handle.value, 0), "cuMemMap")+ access = _AccessDesc()+ access.location.type = _CU_MEM_LOCATION_TYPE_DEVICE+ access.location.id = self.device_index+ access.flags = _CU_MEM_ACCESS_FLAGS_PROT_READWRITE+ rc = lib.cuMemSetAccess(self.va, self.size, ctypes.byref(access), 1)+ if rc != 0:+ lib.cuMemUnmap(self.va, self.size)+ _check(rc, "cuMemSetAccess")+ except Exception:+ lib.cuMemRelease(handle.value)+ raise+ self._handle = int(handle.value)+ self._base = torch.as_tensor(_CudaArray(self.va, self.size), device = self.device)+ self.maps += 1+ return self._base++ def view(+ self,+ offset: int,+ shape: tuple,+ dtype: Any,+ stride: Optional[tuple] = None,+ ) -> Any:+ import torch++ base = self.map()+ numel = 1+ for s in shape:+ numel *= int(s)+ itemsize = torch.empty((), dtype = dtype).element_size()+ if stride is None:+ span = numel+ else:+ span = 1 + sum((int(s) - 1) * int(st) for s, st in zip(shape, stride)) if numel else 0+ flat = base[offset : offset + span * itemsize].view(dtype)+ if stride is None:+ return flat.view(shape)+ return flat.as_strided(shape, stride)++ def unmap(self) -> None:+ """Release the physical memory. The caller guarantees no tensor over the range is used again before ``map()``+ (pending work on the card is waited for here)."""+ import torch++ if self._handle is None:+ return+ lib = _driver()+ with torch.cuda.device(self.device_index):+ torch.cuda.synchronize(self.device_index)+ self._base = None+ _check(lib.cuMemUnmap(self.va, self.size), "cuMemUnmap")+ _check(lib.cuMemRelease(self._handle), "cuMemRelease")+ self._handle = None+ self.unmaps += 1++ def close(self) -> None:+ if self.closed:+ return+ try:+ self.unmap()+ finally:+ self.closed = True+ try:+ _driver().cuMemAddressFree(self.va, self.size)+ except Exception: # noqa: BLE001 - process teardown+ pass+++def _plain(t: Any) -> bool:+ import torch+ return type(t) is torch.Tensor or (+ type(t) is torch.nn.Parameter and type(t.data) is torch.Tensor+ )+++def _inner(t: Any) -> Optional[list]:+ """A torchao-style wrapper weight's plain inner tensors (``diffusion_memory._wrapper_inner``), else None."""+ from .diffusion_memory import _wrapper_inner+ return _wrapper_inner(t)+++def _placeable(t: Any) -> bool:+ return _plain(t) or _inner(t) is not None+++def _pieces(name: str, t: Any) -> list:+ """(layout key, tensor) for each storage of ``t``: its own data, or a wrapper weight's inner tensors."""+ if _plain(t):+ return [(name, t.data)]+ return [(f"{name}#{attr}", x) for attr, x in (_inner(t) or ())]+++def _set_tensor(owner: Any, leaf: str, kind: str, value: Any) -> None:+ if kind == "p":+ owner._parameters[leaf].data = value+ else:+ owner._buffers[leaf] = value+++def _entries(module: Any) -> list:+ """(qualified name, registrations, kind, tensor) for every parameter and buffer, each tensor once.++ ``registrations`` lists every (owner module, leaf name, qualified name) the tensor is registered under: a buffer+ shared by several submodules (or names) has one registry slot per registration, and each must point at the placed+ copy."""+ out: list = []+ index: dict = {}+ for mod_name, mod in module.named_modules(remove_duplicate = False):+ for kind, table in (("p", mod._parameters), ("b", mod._buffers)):+ for leaf, t in table.items():+ if t is None:+ continue+ qual = f"{mod_name}.{leaf}" if mod_name else leaf+ at = index.get(id(t))+ if at is not None:+ regs = out[at][1]+ if not any(o is mod and l == leaf for o, l, _q in regs):+ regs.append((mod, leaf, qual))+ continue+ index[id(t)] = len(out)+ out.append((qual, [(mod, leaf, qual)], kind, t))+ return out+++class ModuleArena:+ """The stable device range of one whole-module offloaded denoiser, installed as the module's own ``to``."""++ def __init__(+ self,+ module: Any,+ device: Any,+ logger: Any = None,+ ) -> None:+ import torch++ self.device = torch.device(device)+ if self.device.index is None:+ self.device = torch.device("cuda", torch.cuda.current_device())+ self.logger = logger+ self.layout: Optional[dict] = None+ self.arena: Optional[VmmArena] = None+ self.failed: Optional[str] = None+ self.stats = {"onloads": 0, "placed": 0, "stock": 0, "maps": 0, "unmaps": 0}+ self._module = weakref.ref(module)+ self._stock_to = type(module).to+ module.__dict__["to"] = self._to+ # accelerate's remove_hook_from_module pops ``to`` from the instance (its _accelerate_added_attributes) and+ # diffusers re-hooks before every render: put it back before the hook's pre-forward onloads the module.+ self._reinstall_handle = module.register_forward_pre_hook(self._reinstall)+ weakref.finalize(module, ModuleArena._finalize, self.__dict__)++ def _reinstall(self, module: Any, _args: Any) -> None:+ if module.__dict__.get("to") != self._to and self.failed is None:+ module.__dict__["to"] = self._to+ self.stats["reinstalls"] = self.stats.get("reinstalls", 0) + 1++ @staticmethod+ def _finalize(state: dict) -> None:+ arena = state.get("arena")+ if arena is not None:+ try:+ arena.close()+ except Exception: # noqa: BLE001 - process teardown+ pass++ def detach(self) -> None:+ module = self._module()+ if module is not None and module.__dict__.get("to") == self._to:+ module.__dict__.pop("to", None)+ try:+ self._reinstall_handle.remove()+ except Exception: # noqa: BLE001+ pass+ if module is not None and module.__dict__.get(_ARENA_ATTR) is self:+ module.__dict__.pop(_ARENA_ATTR, None)+ if self.arena is not None and module is not None and not self._referenced(module):+ self.arena.close()+ self.arena = None++ def _build(self, entries: list) -> None:+ offset = 0+ layout: dict = {}+ for name, regs, _kind, whole in entries:+ if whole.device.type != "cpu" or not _placeable(whole):+ continue+ for key, t in _pieces(name, whole):+ # strided layouts (channels_last) keep their strides over the span they reach+ span = (+ (1 + sum((n - 1) * st for n, st in zip(t.shape, t.stride()))) * t.element_size()+ if t.numel()+ else 0+ )+ slot = (offset, tuple(t.shape), tuple(t.stride()), t.dtype, span)+ # Every registration name of a shared tensor maps to its one slot: Module.to() moves each+ # registration on its own, so after an offload they come back as separate (equal) host copies.+ suffix = key[len(name) :]+ for _owner, _leaf, qual in regs:+ layout[qual + suffix] = slot+ offset += -(-max(span, 1) // _ALIGN) * _ALIGN+ self.layout = layout+ self.arena = VmmArena(max(offset, _ALIGN), self.device.index)++ def _referenced(self, module: Any) -> bool:+ if self.arena is None or not self.arena.mapped:+ return False+ lo, hi = self.arena.va, self.arena.va + self.arena.size+ for name, _regs, _kind, whole in _entries(module):+ try:+ if whole.device.type != "cuda":+ continue+ pieces = _pieces(name, whole) if _placeable(whole) else [(name, whole)]+ if any(lo <= x.data_ptr() < hi for _, x in pieces):+ return True+ except Exception: # noqa: BLE001+ return True+ return False++ @staticmethod+ def _on_host(module: Any) -> bool:+ for t in module.parameters():+ return t.device.type == "cpu"+ return False++ def _onload(self, module: Any, non_blocking: bool) -> None:+ import torch++ entries = _entries(module)+ if self.layout is None:+ self._build(entries)+ assert self.arena is not None and self.layout is not None+ self.arena.map()+ placed = 0+ filled: dict = {} # slot offset -> (host tensor copied into it during this onload, its view)+ for name, regs, kind, whole in entries:+ if whole.device.type != "cpu" or not _placeable(whole):+ continue+ pieces = _pieces(name, whole)+ slots = [self.layout.get(key) for key, _ in pieces]+ if any(+ slot is None+ or (tuple(t.shape), tuple(t.stride()), t.dtype) != (slot[1], slot[2], slot[3])+ for slot, (_, t) in zip(slots, pieces)+ ):+ continue+ # A slot already filled by another registration of a once-shared buffer is reused only when the copies+ # still hold the same values; otherwise this one keeps the stock move.+ if any(+ slot[0] in filled and not torch.equal(filled[slot[0]][0], t)+ for slot, (_, t) in zip(slots, pieces)+ ):+ continue+ views = []+ for slot, (_, t) in zip(slots, pieces):+ if slot[0] in filled:+ # the same view object: the registrations share one tensor again, as on the first onload, so the+ # graph's placement fingerprint (one entry per tensor) does not change+ views.append(filled[slot[0]][1])+ continue+ view = self.arena.view(slot[0], slot[1], slot[3], slot[2])+ view.copy_(t, non_blocking = non_blocking)+ filled[slot[0]] = (t, view)+ views.append(view)+ if _plain(whole):+ for owner, leaf, _qual in regs:+ _set_tensor(owner, leaf, kind, views[0])+ else:+ # a wrapper weight is rebuilt over its placed inner tensors and swapped in, as Module.to moves it+ names, ctx = whole.__tensor_flatten__()+ rebuilt = type(whole).__tensor_unflatten__(+ dict(zip(names, views)), ctx, whole.shape, whole.stride()+ )+ if kind == "p":+ torch.utils.swap_tensors(+ whole, torch.nn.Parameter(rebuilt, requires_grad = whole.requires_grad)+ )+ else:+ for owner, leaf, _qual in regs:+ owner._buffers[leaf] = rebuilt+ placed += 1+ self.stats["placed"] = placed++ def _to(self, *args: Any, **kwargs: Any) -> Any:+ import torch++ module = self._module()+ try:+ device, dtype, non_blocking, fmt = torch._C._nn._parse_to(*args, **kwargs)+ except Exception: # noqa: BLE001 - let the stock call raise its own error+ device = dtype = fmt = None+ non_blocking = False+ if (+ module is not None+ and self.failed is None+ and dtype is None+ and fmt is None+ and device is not None+ and device.type == "cuda"+ and (device.index if device.index is not None else torch.cuda.current_device())+ == self.device.index+ ):+ try:+ # accelerate calls ``to`` on every forward (its device check compares "cuda" with "cuda:0"): only a+ # module whose weights are on the host is placed+ if self._on_host(module):+ self._onload(module, True)+ self.stats["onloads"] += 1+ except Exception as exc: # noqa: BLE001 - the stock move still places the module+ self.failed = f"{type(exc).__name__}: {exc}"+ if self.logger is not None:+ self.logger.warning(+ "diffusion.stable_arena: onload into the stable range failed (%s)", exc+ )+ self._release(module)+ out = self._stock_to(module, *args, **kwargs)+ if module is not None:+ self._release(module)+ return out++ def _release(self, module: Any) -> None:+ """Unmap the range once no tensor of the module lives in it (its weights went back to the host)."""+ if self.arena is None or not self.arena.mapped or self._referenced(module):+ return+ self.arena.unmap()+ self.stats["unmaps"] += 1++ def describe(self) -> dict:+ arena = self.arena+ return {+ **self.stats,+ "mib": None if arena is None else arena.size >> 20,+ "mapped": bool(arena is not None and arena.mapped),+ "failed": self.failed,+ }+++def attach(+ module: Any,+ device: Any = None,+ logger: Any = None,+) -> tuple:+ """``(arena, None)`` when ``module``'s onloads will land in one stable range from now on, else ``(None, why)``."""+ if arena_disabled():+ return None, f"disabled by {ARENA_ENV}=0"+ existing = module.__dict__.get(_ARENA_ATTR)+ if existing is not None:+ return (existing, None) if existing.failed is None else (None, existing.failed)+ try:+ import torch+ if not torch.cuda.is_available():+ return None, "no CUDA device"+ _driver()+ except Exception as exc: # noqa: BLE001+ return None, f"CUDA virtual memory unavailable ({exc})"+ odd = [name for name, _r, _k, t in _entries(module) if not _placeable(t)]+ if odd:+ return None, f"{len(odd)} parameter(s) are not plain tensors ({odd[0]})"+ if device is None:+ hook = getattr(module, "_hf_hook", None)+ device = getattr(hook, "execution_device", None)+ if device is None:+ return None, "no execution device"+ try:+ arena = ModuleArena(module, device, logger)+ except Exception as exc: # noqa: BLE001+ return None, f"stable range unavailable ({exc})"+ module.__dict__[_ARENA_ATTR] = arena+ return arena, None+++def arena_of(module: Any) -> Optional[ModuleArena]:+ try:+ return module.__dict__.get(_ARENA_ATTR)+ except Exception: # noqa: BLE001+ return Nonediff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.pyindex 6d9493245b3..b13d1d47050 100644--- a/studio/backend/tests/test_diffusion_offload_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py@@ -17,6 +17,7 @@ import core.inference.diffusion_cuda_graph as cg # noqa: E402import core.inference.diffusion_memory as dm # noqa: E402import core.inference.diffusion_offload_prefetch as op # noqa: E402import core.inference.diffusion_speed as ds # noqa: E402+import core.inference.diffusion_stable_arena as sa # noqa: E402@pytest.fixture(autouse = True)@@ -27,6 +28,7 @@ def _clean_env(monkeypatch):cg.OFFLOAD_CUDA_GRAPH_ENV,cg.CUDA_GRAPH_DISABLE_ENV,cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV,+ sa.ARENA_ENV,cg.SPEED_CHECK_ENV,"UNSLOTH_DIFFUSION_PARTIAL_RESIDENT","UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",@@ -282,9 +284,10 @@ def _model_offloaded():return ref, net, hook-def test_model_offload_keeps_the_step_eager_by_default_with_the_reason():+def test_model_offload_keeps_the_step_eager_without_the_stable_range(monkeypatch):_cuda()pytest.importorskip("accelerate")+ monkeypatch.setenv(sa.ARENA_ENV, "0")_ref, net, _hook = _model_offloaded()placement, why = cg.offload_placement(net)assert placement is None and why == cg.MODEL_OFFLOAD_REASON@@ -302,7 +305,7 @@ def test_model_offload_records_nothing_after_the_first_render(monkeypatch, opt_imonkeypatch.setenv(cg.MODEL_OFFLOAD_CUDA_GRAPH_ENV, "1")ref, net, hook = _model_offloaded()handles, reason = cg.arm_after_placement(_pipe(net))- assert len(handles) == (1 if opt_in else 0), reason+ assert len(handles) == 1, reasondev = torch.cuda.current_device()later, hold = 0, []for render in range(4):@@ -320,7 +323,65 @@ def test_model_offload_records_nothing_after_the_first_render(monkeypatch, opt_iif opt_in:assert later >= 1 and handles[0].stats["invalidations"] >= 1else:- assert later == 0+ # the stable range: the step recorded on render 1 replays on every later render+ assert later == 0 and handles[0].stats["invalidations"] == 0+ assert handles[0].stats["replays"] >= 4 * 5 - cg.WARMUP_ITERS - 1+++def test_model_offload_replays_over_the_stable_range_and_frees_it_between_renders():+ """low_vram: the denoiser's onloads land in one reserved VA range, so render 1's graph replays on renders 2+ with+ no capture, bit-identical, while each offload returns the range's memory to the card for the other stages."""+ _cuda()+ pytest.importorskip("accelerate")+ from accelerate import cpu_offload_with_hook++ torch.manual_seed(0)+ ref = _Net(width = 1024).cuda()+ torch.manual_seed(0)+ net, hook = cpu_offload_with_hook(_Net(width = 1024), execution_device = "cuda")+ other, other_hook = cpu_offload_with_hook(+ torch.nn.Linear(1024, 1024), execution_device = "cuda", prev_module_hook = hook+ )+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == 1 and "stable device range" in reason, reason+ arena = sa.arena_of(net)+ assert arena is not None+ dev = torch.cuda.current_device()+ weight_bytes = sum(p.numel() * p.element_size() for p in net.parameters())+ ptrs, later = None, 0+ for render in range(4):+ seeds = [10 * render + i for i in range(5)]+ before = cg.capture_generation(dev)+ got = _run(net, seeds)+ if render:+ later += cg.capture_generation(dev) - before+ now = tuple(p.data_ptr() for p in net.parameters())+ assert ptrs is None or now == ptrs # same addresses on every onload+ ptrs = now+ for a, b in zip(got, _run(ref, seeds)):+ assert torch.equal(a, b)+ other(torch.zeros(2, 1024)) # the next stage's pre-forward offloads the denoiser+ assert net.proj_in.weight.device.type == "cpu"+ assert not arena.arena.mapped # its physical memory went back to the card+ assert arena.arena.size >= weight_bytes+ other_hook.offload()+ torch.cuda.empty_cache()+ _hold = torch.empty(4 << 20, device = "cuda") # the freed VRAM is reused meanwhile+ del _hold+ # diffusers re-hooks every render (remove_hook_from_module also drops instance ``to`` overrides)+ from accelerate.hooks import remove_hook_from_module++ remove_hook_from_module(net)+ remove_hook_from_module(other)+ net, hook = cpu_offload_with_hook(net, execution_device = "cuda")+ other, other_hook = cpu_offload_with_hook(+ other, execution_device = "cuda", prev_module_hook = hook+ )+ assert later == 0+ assert handles[0].stats["captures"] == 1 and handles[0].stats["invalidations"] == 0+ assert arena.stats["onloads"] == 4 and arena.stats["unmaps"] == 4+ assert arena.stats.get("reinstalls", 0) >= 3+ cg.uninstall_all(handles)def test_model_offload_stops_recording_only_when_moves_outpace_replays():@@ -765,3 +826,158 @@ def test_status_names_graphs_dropped_after_replaying():handle.cache = {"k": object()}handle.capture_error = Noneassert cg.live_status(resolved, ["cuda_graph"], [handle])[0]["cuda_graph"]["value"] == "on"+++def _inner_of(t):+ """A torchao wrapper weight's inner tensors, read without the module under test."""+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass++ if not is_traceable_wrapper_subclass(t):+ return None+ names, _ctx = t.__tensor_flatten__()+ inner = [(n, getattr(t, n)) for n in names]+ return inner if all(type(x) is torch.Tensor for _, x in inner) else None+++def test_model_offload_stable_range_holds_torchao_int8_weights():+ """Studio's compiled low_vram tier quantizes the denoiser to torchao int8: those wrapper weights are rebuilt over+ the stable range too (same inner addresses on every onload, memory released between renders), exactly.+ The eager torchao int8 matmul here is not capture-safe, so this checks placement, not the replay; the compiled+ Studio step records over it (scripts/graphs_split_offload)."""+ _cuda()+ pytest.importorskip("accelerate")+ quant = pytest.importorskip("torchao.quantization")+ from accelerate import cpu_offload_with_hook++ def build():+ torch.manual_seed(0)+ net = _Net(width = 256).to(torch.bfloat16)+ try:+ config = quant.Int8DynamicActivationInt8WeightConfig(version = 2)+ except TypeError:+ config = quant.Int8DynamicActivationInt8WeightConfig()+ quant.quantize_(net.blocks, config)+ return net++ probe = build()+ if _inner_of(probe.blocks[0].weight.data) is None:+ pytest.skip("this torchao build stores int8 weights without a traceable wrapper")+ ref = build().cuda()+ net, hook = cpu_offload_with_hook(build(), execution_device = "cuda")+ arena, why = sa.attach(net)+ assert arena is not None, why++ def run(module, seeds):+ outs = []+ with torch.no_grad():+ for seed in seeds:+ x, t = _inputs(seed)+ outs.append(+ module(x.to(torch.bfloat16), t.to(torch.bfloat16), return_dict = False)[0].clone()+ )+ torch.cuda.synchronize()+ return outs++ def inner_ptrs():+ return [x.data_ptr() for b in net.blocks for _, x in _inner_of(b.weight.data)]++ ptrs = None+ for render in range(3):+ seeds = [10 * render + i for i in range(3)]+ got = run(net, seeds)+ assert type(net.blocks[0].weight.data) is type(probe.blocks[0].weight.data)+ now = inner_ptrs()+ lo, hi = arena.arena.va, arena.arena.va + arena.arena.size+ assert all(lo <= q < hi for q in now)+ assert ptrs is None or now == ptrs+ ptrs = now+ for a, b in zip(got, run(ref, seeds)):+ assert torch.equal(a, b)+ hook.offload()+ assert net.blocks[0].weight.device.type == "cpu"+ assert not arena.arena.mapped+ assert arena.stats["onloads"] == 3 and arena.stats["unmaps"] == 3++[email protected]("where", ["forward_refusal", "placement"])+def test_a_refused_model_offload_leaves_the_stock_onload(monkeypatch, where):+ """A denoiser whose step stays eager keeps accelerate's own onload: no stable range, no instance ``to``, no+ re-install hook (so no per-offload card sync for a step that never replays)."""+ _cuda()+ pytest.importorskip("accelerate")+ _ref, net, _hook = _model_offloaded()+ if where == "forward_refusal":+ monkeypatch.setattr(cg, "_forward_refusal", lambda module, mode: "not capture-safe")+ else:++ def boom(*a, **k):+ raise RuntimeError("no hook forward")++ monkeypatch.setattr(cg, "OffloadPlacement", boom)+ placement, why = cg.offload_placement(net)+ assert placement is None and why+ assert sa.arena_of(net) is None+ assert "to" not in net.__dict__+ assert not net._forward_pre_hooks+++class _SharedBuffer(_Net):+ """One buffer tensor registered on two submodules, as tied rotary / scale tables are."""++ def __init__(self):+ super().__init__(width = 512)+ scale = torch.linspace(0.5, 1.5, 512)+ self.blocks[0].register_buffer("scale", scale)+ self.blocks[-1].register_buffer("scale", scale)++ def forward(+ self,+ x,+ t,+ return_dict = True,+ ):+ x = self.proj_in(x) * t+ for i, block in enumerate(self.blocks):+ x = torch.nn.functional.gelu(block(x))+ if i in (0, len(self.blocks) - 1):+ x = x * block.scale+ out = self.proj_out(x)+ return (out,) if not return_dict else {"sample": out}+++def test_model_offload_stable_range_places_every_registration_of_a_shared_buffer():+ _cuda()+ pytest.importorskip("accelerate")+ from accelerate import cpu_offload_with_hook+ from accelerate.hooks import remove_hook_from_module++ torch.manual_seed(0)+ ref = _SharedBuffer().cuda()+ torch.manual_seed(0)+ net, hook = cpu_offload_with_hook(_SharedBuffer(), execution_device = "cuda")+ handles, reason = cg.arm_after_placement(_pipe(net))+ assert len(handles) == 1 and "stable device range" in reason, reason+ arena = sa.arena_of(net)+ later = 0+ dev = torch.cuda.current_device()+ for render in range(4):+ seeds = [10 * render + i for i in range(5)]+ before = cg.capture_generation(dev)+ got = _run(net, seeds)+ if render:+ later += cg.capture_generation(dev) - before+ first, last = net.blocks[0].scale, net.blocks[-1].scale+ lo, hi = arena.arena.va, arena.arena.va + arena.arena.size+ # Module.to() splits a shared buffer into two host copies; the onload shares one placed tensor again+ assert first is last and lo <= first.data_ptr() < hi+ for a, b in zip(got, _run(ref, seeds)):+ assert torch.equal(a, b)+ hook.offload()+ assert net.blocks[-1].scale.device.type == "cpu" and not arena.arena.mapped+ torch.cuda.empty_cache()+ _hold = torch.empty(4 << 20, device = "cuda") # the freed VRAM is reused meanwhile+ del _hold+ remove_hook_from_module(net)+ net, hook = cpu_offload_with_hook(net, execution_device = "cuda")+ assert later == 0 and handles[0].stats["invalidations"] == 0, handles[0].stats+ cg.uninstall_all(handles)diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.pyindex 958edec5113..9b4cacf1400 100644--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py@@ -608,3 +608,65 @@ def test_buffers_onloaded_under_inference_mode_do_not_break_the_hook():for _ in range(3):with torch.inference_mode():assert torch.equal(_render(pipe, x), ref)+++def _int8_pipe(device):+ """The _pipe layout with torchao Int8Tensor weights (Studio's int8 denoisers under low_vram)."""+ quant = pytest.importorskip("torchao.quantization")+ pipe = _pipe(device, "text_encoder", "transformer")+ lin = torch.nn.Linear(64, 64, dtype = torch.bfloat16)+ torch.manual_seed(1)+ torch.nn.init.normal_(lin.weight)+ try:+ config = quant.Int8DynamicActivationInt8WeightConfig(version = 2)+ except TypeError:+ config = quant.Int8DynamicActivationInt8WeightConfig()+ quant.quantize_(torch.nn.Sequential(lin), config)+ if _inner_of(lin.weight.data) is None:+ pytest.skip("this torchao build stores int8 weights without a traceable wrapper")+ pipe.components["text_encoder"] = torch.nn.Linear(8, 64, dtype = torch.bfloat16)+ pipe.components["transformer"] = lin+ pipe.text_encoder, pipe.transformer = pipe.components["text_encoder"], lin+ return pipe+++def _inner_ptrs(module):+ return [x.data_ptr() for _, x in _inner_of(module.weight.data)]+++def _inner_of(t):+ """A torchao wrapper weight's inner tensors, read without the module under test."""+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass++ if not is_traceable_wrapper_subclass(t):+ return None+ names, _ctx = t.__tensor_flatten__()+ inner = [(n, getattr(t, n)) for n in names]+ return inner if all(type(x) is torch.Tensor for _, x in inner) else None+++@cuda[email protected]("pin", ["0", "1"])+def test_torchao_int8_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):+ """Each offload re-points the int8 weight at its kept host tensors (no device-to-host copy, no new host+ allocation), the inner tensors are pinned once, and every render matches the stock offload bit for bit."""+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)+ x = torch.randn(4, 8, dtype = torch.bfloat16)+ stock = _int8_pipe("cuda")+ stock.enable_model_cpu_offload()+ ref = _render(stock, x)+ pipe = _int8_pipe("cuda")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ assert _wrapped(pipe.transformer)+ outs, host = [], None+ for _ in range(3):+ outs.append(_render(pipe, x))+ assert pipe.transformer.weight.device.type == "cpu"+ now = _inner_ptrs(pipe.transformer)+ assert host is None or now == host # the same host tensors every render+ host = now+ for out in outs:+ assert torch.equal(out, ref)+ inner = _inner_of(pipe.transformer.weight.data)+ assert all(t.is_pinned() for _, t in inner) == (pin == "1")