Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Keeps offloaded denoiser weights at fixed VMM addresses so recorded CUDA graphs replay; GGUF or
other parameter types left to the stock hook"""
from __future__ import annotations
import ctypes
import os
import threading
import weakref
from typing import Any, Optional
ARENA_ENV = "UNSLOTH_DIFFUSION_MODEL_OFFLOAD_ARENA"
_ARENA_ATTR = "_unsloth_stable_arena"
_ALIGN = 512
_OFF = ("0", "off", "false", "no")
# CUDA driver enums
_CU_MEM_ALLOCATION_TYPE_PINNED = 1
_CU_MEM_LOCATION_TYPE_DEVICE = 1
_CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 3
_CU_MEM_ALLOC_GRANULARITY_RECOMMENDED = 1
class _Location(ctypes.Structure):
_fields_ = [("type", ctypes.c_int), ("id", ctypes.c_int)]
class _AllocFlags(ctypes.Structure):
_fields_ = [
("compressionType", ctypes.c_ubyte),
("gpuDirectRDMACapable", ctypes.c_ubyte),
("usage", ctypes.c_ushort),
("reserved", ctypes.c_ubyte * 4),
]
class _AllocProp(ctypes.Structure):
_fields_ = [
("type", ctypes.c_int),
("requestedHandleTypes", ctypes.c_int),
("location", _Location),
("win32HandleMetaData", ctypes.c_void_p),
("allocFlags", _AllocFlags),
]
class _AccessDesc(ctypes.Structure):
_fields_ = [("location", _Location), ("flags", ctypes.c_int)]
_LIB: Any = None
_LIB_LOCK = threading.Lock()
def _driver() -> Any:
global _LIB
with _LIB_LOCK:
if _LIB is None:
lib = None
for name in ("libcuda.so.1", "libcuda.so", "nvcuda.dll"):
try:
lib = ctypes.CDLL(name)
break
except OSError:
continue
if lib is None:
raise OSError("CUDA driver library not found")
u64, sz = ctypes.c_uint64, ctypes.c_size_t
for fn, args in (
(
"cuMemGetAllocationGranularity",
[ctypes.POINTER(sz), ctypes.POINTER(_AllocProp), ctypes.c_int],
),
("cuMemAddressReserve", [ctypes.POINTER(u64), sz, sz, u64, ctypes.c_ulonglong]),
("cuMemAddressFree", [u64, sz]),
(
"cuMemCreate",
[
ctypes.POINTER(ctypes.c_ulonglong),
sz,
ctypes.POINTER(_AllocProp),
ctypes.c_ulonglong,
],
),
("cuMemRelease", [ctypes.c_ulonglong]),
("cuMemMap", [u64, sz, sz, ctypes.c_ulonglong, ctypes.c_ulonglong]),
("cuMemUnmap", [u64, sz]),
("cuMemSetAccess", [u64, sz, ctypes.POINTER(_AccessDesc), sz]),
("cuCtxGetCurrent", [ctypes.POINTER(ctypes.c_void_p)]),
):
f = getattr(lib, fn)
f.argtypes = args
f.restype = ctypes.c_int
_LIB = lib
return _LIB
# Physical bytes mapped through every range, per CUDA ordinal: memory the caching allocator's counters do not see.
_MAPPED: dict = {}
# High-water mark of ``_MAPPED`` since the last ``reset_peak_mapped`` (the stable ranges' max_memory_allocated).
_PEAK_MAPPED: dict = {}
_MAPPED_LOCK = threading.Lock()
def mapped_bytes(device_index: Optional[int]) -> int:
"""Bytes currently mapped in stable ranges on ``device_index`` (outside ``torch.cuda.memory_allocated``)."""
with _MAPPED_LOCK:
return int(_MAPPED.get(device_index, 0))
def peak_mapped_bytes(device_index: Optional[int]) -> int:
"""Most bytes mapped at once on ``device_index`` since the last ``reset_peak_mapped``."""
with _MAPPED_LOCK:
return int(max(_PEAK_MAPPED.get(device_index, 0), _MAPPED.get(device_index, 0)))
def reset_peak_mapped(device_index: Optional[int]) -> None:
with _MAPPED_LOCK:
_PEAK_MAPPED[device_index] = _MAPPED.get(device_index, 0)
def _count_mapped(device_index: int, delta: int) -> None:
with _MAPPED_LOCK:
now = _MAPPED.get(device_index, 0) + int(delta)
_MAPPED[device_index] = now
_PEAK_MAPPED[device_index] = max(_PEAK_MAPPED.get(device_index, 0), now)
def _check(rc: int, what: str) -> None:
if rc != 0:
raise RuntimeError(f"{what} failed (CUresult {rc})")
def arena_disabled() -> bool:
return (os.environ.get(ARENA_ENV) or "").strip().lower() in _OFF
class _CudaArray:
"""A ``__cuda_array_interface__`` over raw device bytes, so ``torch.as_tensor`` wraps them without a copy."""
def __init__(self, ptr: int, nbytes: int) -> None:
self.__cuda_array_interface__ = {
"shape": (int(nbytes),),
"typestr": "|u1",
"data": (int(ptr), False),
"version": 3,
"strides": None,
"stream": None,
}
class VmmArena:
"""One reserved device VA range of ``nbytes`` (rounded to the allocation granularity) on ``device_index``; physical
memory exists only between ``map()`` and ``unmap()``."""
def __init__(self, nbytes: int, device_index: int) -> None:
import torch
self.device_index = int(device_index)
self.device = torch.device("cuda", self.device_index)
lib = _driver()
self._ensure_context()
prop = self._prop()
gran = ctypes.c_size_t(0)
_check(
lib.cuMemGetAllocationGranularity(
ctypes.byref(gran), ctypes.byref(prop), _CU_MEM_ALLOC_GRANULARITY_RECOMMENDED
),
"cuMemGetAllocationGranularity",
)
self.granularity = int(gran.value) or (2 << 20)
self.size = -(-int(nbytes) // self.granularity) * self.granularity
va = ctypes.c_uint64(0)
_check(
lib.cuMemAddressReserve(ctypes.byref(va), self.size, self.granularity, 0, 0),
"cuMemAddressReserve",
)
self.va = int(va.value)
self._handle: Optional[int] = None
self._base: Any = None
self.maps = 0
self.unmaps = 0
self.closed = False
def _prop(self) -> _AllocProp:
prop = _AllocProp()
prop.type = _CU_MEM_ALLOCATION_TYPE_PINNED
prop.location.type = _CU_MEM_LOCATION_TYPE_DEVICE
prop.location.id = self.device_index
return prop
def _ensure_context(self) -> None:
import torch
ctx = ctypes.c_void_p(0)
_driver().cuCtxGetCurrent(ctypes.byref(ctx))
if not ctx.value:
# The calling thread has no current context yet: any runtime call on the device makes the primary one current.
with torch.cuda.device(self.device_index):
torch.cuda.current_stream().query()
@property
def mapped(self) -> bool:
return self._handle is not None
def map(self) -> Any:
"""Back the range with new physical memory; returns a uint8 tensor over the whole range."""
import torch
if self.closed:
raise RuntimeError("arena closed")
if self._handle is not None:
return self._base
lib = _driver()
with torch.cuda.device(self.device_index):
self._ensure_context()
prop = self._prop()
handle = ctypes.c_ulonglong(0)
rc = lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)
if rc != 0:
# The caching allocator may hold the memory as free cached blocks.
torch.cuda.empty_cache()
rc = lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)
_check(rc, "cuMemCreate")
try:
_check(lib.cuMemMap(self.va, self.size, 0, handle.value, 0), "cuMemMap")
access = _AccessDesc()
access.location.type = _CU_MEM_LOCATION_TYPE_DEVICE
access.location.id = self.device_index
access.flags = _CU_MEM_ACCESS_FLAGS_PROT_READWRITE
rc = lib.cuMemSetAccess(self.va, self.size, ctypes.byref(access), 1)
if rc != 0:
lib.cuMemUnmap(self.va, self.size)
_check(rc, "cuMemSetAccess")
except Exception:
lib.cuMemRelease(handle.value)
raise
try:
base = torch.as_tensor(_CudaArray(self.va, self.size), device = self.device)
except Exception:
# nothing may keep the mapping a failed map() leaves no tensor over: unmap and release it, so the
# arena is unmapped and retryable and the caller's stock fallback holds the only copy
lib.cuMemUnmap(self.va, self.size)
lib.cuMemRelease(handle.value)
raise
self._handle = int(handle.value)
self._base = base
_count_mapped(self.device_index, self.size)
self.maps += 1
return self._base
def view(
self,
offset: int,
shape: tuple,
dtype: Any,
stride: Optional[tuple] = None,
) -> Any:
import torch
base = self.map()
numel = 1
for s in shape:
numel *= int(s)
itemsize = torch.empty((), dtype = dtype).element_size()
if stride is None:
span = numel
else:
span = 1 + sum((int(s) - 1) * int(st) for s, st in zip(shape, stride)) if numel else 0
flat = base[offset : offset + span * itemsize].view(dtype)
if stride is None:
return flat.view(shape)
return flat.as_strided(shape, stride)
def unmap(self) -> None:
"""Release the physical memory. The caller guarantees no tensor over the range is used again before ``map()``
(pending work on the card is waited for here)."""
import torch
if self._handle is None:
return
lib = _driver()
with torch.cuda.device(self.device_index):
from .diffusion_cuda_graph import synchronize_outside_capture
synchronize_outside_capture(self.device_index)
self._release_synced(lib)
def _release_synced(self, lib: Any) -> None:
"""Unmap and release, the card already synchronized with no capture recording."""
self._base = None
_check(lib.cuMemUnmap(self.va, self.size), "cuMemUnmap")
_check(lib.cuMemRelease(self._handle), "cuMemRelease")
self._handle = None
_count_mapped(self.device_index, -self.size)
self.unmaps += 1
def _close_idle(self) -> None:
"""``close`` for ``release_when_capture_idle``: run under the capture lock with no capture recording."""
import torch
if self.closed:
return
try:
if self._handle is not None:
with torch.cuda.device(self.device_index):
torch.cuda.synchronize(self.device_index)
self._release_synced(_driver())
finally:
self.closed = True
try:
_driver().cuMemAddressFree(self.va, self.size)
except Exception: # noqa: BLE001 - process teardown
pass
def close_from_finalizer(self) -> None:
"""``close`` for weakref finalizers: never blocks on the capture gate (the finalizer's thread may hold its
lock). Runs now when no capture records and the lock is free, else at the next gated point; the memory is
freed only after a device sync with no capture recording, as ``unmap`` does."""
if self.closed:
return
from .diffusion_cuda_graph import release_when_capture_idle
release_when_capture_idle(self._close_idle)
def close(self) -> None:
if self.closed:
return
try:
self.unmap()
finally:
self.closed = True
try:
_driver().cuMemAddressFree(self.va, self.size)
except Exception: # noqa: BLE001 - process teardown
pass
def _plain(t: Any) -> bool:
import torch
return type(t) is torch.Tensor or (
type(t) is torch.nn.Parameter and type(t.data) is torch.Tensor
)
def _inner(t: Any) -> Optional[list]:
"""A torchao-style wrapper weight's plain inner tensors (``diffusion_memory._wrapper_inner``), else None."""
from .diffusion_memory import _wrapper_inner
return _wrapper_inner(t)
def _placeable(t: Any) -> bool:
return _plain(t) or _inner(t) is not None
def _pieces(name: str, t: Any) -> list:
"""(layout key, tensor) for each storage of ``t``: its own data, or a wrapper weight's inner tensors."""
if _plain(t):
return [(name, t.data)]
return [(f"{name}#{attr}", x) for attr, x in (_inner(t) or ())]
def _set_tensor(owner: Any, leaf: str, kind: str, value: Any) -> None:
if kind == "p":
owner._parameters[leaf].data = value
else:
owner._buffers[leaf] = value
def _entries(module: Any) -> list:
"""Yields (qualified name, registrations, kind, tensor) per tensor, each once; a shared buffer's
registrations must each point at the placed copy."""
out: list = []
index: dict = {}
for mod_name, mod in module.named_modules(remove_duplicate = False):
for kind, table in (("p", mod._parameters), ("b", mod._buffers)):
for leaf, t in table.items():
if t is None:
continue
qual = f"{mod_name}.{leaf}" if mod_name else leaf
at = index.get(id(t))
if at is not None:
regs = out[at][1]
if not any(o is mod and l == leaf for o, l, _q in regs):
regs.append((mod, leaf, qual))
continue
index[id(t)] = len(out)
out.append((qual, [(mod, leaf, qual)], kind, t))
return out
class ModuleArena:
"""The stable device range of one whole-module offloaded denoiser, installed as the module's own ``to``."""
def __init__(
self,
module: Any,
device: Any,
logger: Any = None,
) -> None:
import torch
self.device = torch.device(device)
if self.device.index is None:
self.device = torch.device("cuda", torch.cuda.current_device())
self.logger = logger
self.layout: Optional[dict] = None
self.arena: Optional[VmmArena] = None
self.failed: Optional[str] = None
self.retiring = False
self.stats = {"onloads": 0, "placed": 0, "stock": 0, "maps": 0, "unmaps": 0, "rebuilds": 0}
self._module = weakref.ref(module)
self._stock_to = type(module).to
module.__dict__["to"] = self._to
# accelerate's remove_hook_from_module pops ``to`` from the instance (its _accelerate_added_attributes) and
# diffusers re-hooks before every render: put it back before the hook's pre-forward onloads the module.
self._reinstall_handle = module.register_forward_pre_hook(self._reinstall)
weakref.finalize(module, ModuleArena._finalize, self.__dict__)
def _reinstall(self, module: Any, _args: Any) -> None:
if self.retiring:
self.retire()
return
if module.__dict__.get("to") != self._to and self.failed is None:
module.__dict__["to"] = self._to
self.stats["reinstalls"] = self.stats.get("reinstalls", 0) + 1
@staticmethod
def _finalize(state: dict) -> None:
arena = state.get("arena")
if arena is not None:
try:
arena.close_from_finalizer()
except Exception: # noqa: BLE001 - process teardown
pass
def detach(self) -> None:
"""Restores the stock onload and closes the range; if weights still sit in it when detached,
placing stops and the offload finishes, so the range is never left mapped."""
module = self._module()
if self.arena is not None and module is not None and self._referenced(module):
self.failed = self.failed or "detached"
self.retiring = True
return
self.retiring = False
if module is not None and module.__dict__.get("to") == self._to:
module.__dict__.pop("to", None)
try:
self._reinstall_handle.remove()
except Exception: # noqa: BLE001
pass
if module is not None and module.__dict__.get(_ARENA_ATTR) is self:
module.__dict__.pop(_ARENA_ATTR, None)
if self.arena is not None and module is not None and not self._referenced(module):
self.arena.close()
self.arena = None
def retire(self) -> None:
"""Restore the stock onload; detach waits for the offload that takes mapped weights home."""
self.retiring = True
module = self._module()
if (
module is None
or self.arena is None
or not self.arena.mapped
or not self._referenced(module)
):
if self.arena is not None and self.arena.mapped and module is not None:
self._release(module)
self.detach()
def _build(self, entries: list) -> None:
offset = 0
layout: dict = {}
for name, regs, _kind, whole in entries:
if whole.device.type != "cpu" or not _placeable(whole):
continue
for key, t in _pieces(name, whole):
# strided layouts (channels_last) keep their strides over the span they reach
span = (
(1 + sum((n - 1) * st for n, st in zip(t.shape, t.stride()))) * t.element_size()
if t.numel()
else 0
)
slot = (offset, tuple(t.shape), tuple(t.stride()), t.dtype, span)
# Every registration name of a shared tensor maps to its one slot: Module.to() moves each
# registration on its own, so after an offload they come back as separate (equal) host copies.
suffix = key[len(name) :]
for _owner, _leaf, qual in regs:
layout[qual + suffix] = slot
offset += -(-max(span, 1) // _ALIGN) * _ALIGN
self.layout = layout
self.arena = VmmArena(max(offset, _ALIGN), self.device.index)
def _stale(self, entries: list) -> bool:
"""Whether a placeable host weight has no slot of its shape, stride and dtype in the layout, or a slot outlived
its weight (an unloaded adapter): a removed piece must not keep its share of the range mapped."""
assert self.layout is not None
seen: set = set()
for name, regs, _kind, whole in entries:
if whole.device.type != "cpu" or not _placeable(whole):
continue
for key, t in _pieces(name, whole):
slot = self.layout.get(key)
if slot is None or (tuple(t.shape), tuple(t.stride()), t.dtype) != (
slot[1],
slot[2],
slot[3],
):
return True
suffix = key[len(name) :]
seen.update(qual + suffix for _owner, _leaf, qual in regs)
return not seen.issuperset(self.layout)
def _referenced(self, module: Any) -> bool:
if self.arena is None or not self.arena.mapped:
return False
lo, hi = self.arena.va, self.arena.va + self.arena.size
for name, _regs, _kind, whole in _entries(module):
try:
if whole.device.type != "cuda":
continue
pieces = _pieces(name, whole) if _placeable(whole) else [(name, whole)]
if any(lo <= x.data_ptr() < hi for _, x in pieces):
return True
except Exception: # noqa: BLE001
return True
return False
@staticmethod
def _on_host(module: Any) -> bool:
for t in module.parameters():
return t.device.type == "cpu"
return False
def _onload(self, module: Any, non_blocking: bool) -> None:
import torch
entries = _entries(module)
if self.layout is not None and self._stale(entries):
# Host weights changed (adapter, dtype, shape): re-lay out, else the stale range stays mapped beside them.
if self.arena is not None:
self.arena.close()
self.arena = None
self.layout = None
self.stats["rebuilds"] += 1
if self.layout is None:
self._build(entries)
assert self.arena is not None and self.layout is not None
self.arena.map()
placed = 0
filled: dict = {} # slot offset -> (host tensor copied into it during this onload, its view)
for name, regs, kind, whole in entries:
if whole.device.type != "cpu" or not _placeable(whole):
continue
pieces = _pieces(name, whole)
slots = [self.layout.get(key) for key, _ in pieces]
if any(
slot is None
or (tuple(t.shape), tuple(t.stride()), t.dtype) != (slot[1], slot[2], slot[3])
for slot, (_, t) in zip(slots, pieces)
):
continue
# A slot already filled by another registration of a once-shared buffer is reused only when the copies
# still hold the same values; otherwise this one keeps the stock move.
if any(
slot[0] in filled and not torch.equal(filled[slot[0]][0], t)
for slot, (_, t) in zip(slots, pieces)
):
continue
views = []
for slot, (_, t) in zip(slots, pieces):
if slot[0] in filled:
# the same view object: the registrations share one tensor again, as on the first onload, so the
# graph's placement fingerprint (one entry per tensor) does not change
views.append(filled[slot[0]][1])
continue
view = self.arena.view(slot[0], slot[1], slot[3], slot[2])
view.copy_(t, non_blocking = non_blocking)
filled[slot[0]] = (t, view)
views.append(view)
if _plain(whole):
for owner, leaf, _qual in regs:
_set_tensor(owner, leaf, kind, views[0])
else:
# a wrapper weight is rebuilt over its placed inner tensors and swapped in, as Module.to moves it
names, ctx = whole.__tensor_flatten__()
rebuilt = type(whole).__tensor_unflatten__(
dict(zip(names, views)), ctx, whole.shape, whole.stride()
)
if kind == "p":
torch.utils.swap_tensors(
whole, torch.nn.Parameter(rebuilt, requires_grad = whole.requires_grad)
)
else:
for owner, leaf, _qual in regs:
owner._buffers[leaf] = rebuilt
placed += 1
self.stats["placed"] = placed
def _to(self, *args: Any, **kwargs: Any) -> Any:
import torch
module = self._module()
try:
device, dtype, non_blocking, fmt = torch._C._nn._parse_to(*args, **kwargs)
except Exception: # noqa: BLE001 - let the stock call raise its own error
device = dtype = fmt = None
non_blocking = False
if (
module is not None
and self.failed is None
and not self.retiring
and dtype is None
and fmt is None
and device is not None
and device.type == "cuda"
and (device.index if device.index is not None else torch.cuda.current_device())
== self.device.index
):
try:
# accelerate calls ``to`` on every forward (its device check compares "cuda" with "cuda:0"): only a
# module whose weights are on the host is placed
if self._on_host(module):
self._onload(module, True)
self.stats["onloads"] += 1
except Exception as exc: # noqa: BLE001 - the stock move still places the module
self.failed = f"{type(exc).__name__}: {exc}"
if self.logger is not None:
self.logger.warning(
"diffusion.stable_arena: onload into the stable range failed (%s)", exc
)
self._release(module)
out = self._stock_to(module, *args, **kwargs)
if module is not None:
self._release(module)
if self.retiring and not (self.arena is not None and self.arena.mapped):
self.detach()
return out
def _release(self, module: Any) -> None:
"""Unmap the range once no tensor of the module lives in it (its weights went back to the host)."""
if self.arena is None or not self.arena.mapped or self._referenced(module):
return
self.arena.unmap()
self.stats["unmaps"] += 1
def describe(self) -> dict:
arena = self.arena
return {
**self.stats,
"mib": None if arena is None else arena.size >> 20,
"mapped": bool(arena is not None and arena.mapped),
"failed": self.failed,
}
def attach(
module: Any,
device: Any = None,
logger: Any = None,
) -> tuple:
"""``(arena, None)`` when ``module``'s onloads will land in one stable range from now on, else ``(None, why)``."""
if arena_disabled():
return None, f"disabled by {ARENA_ENV}=0"
existing = module.__dict__.get(_ARENA_ATTR)
if existing is not None:
return (existing, None) if existing.failed is None else (None, existing.failed)
try:
import torch
if not torch.cuda.is_available():
return None, "no CUDA device"
_driver()
except Exception as exc: # noqa: BLE001
return None, f"CUDA virtual memory unavailable ({exc})"
odd = [name for name, _r, _k, t in _entries(module) if not _placeable(t)]
if odd:
return None, f"{len(odd)} parameter(s) are not plain tensors ({odd[0]})"
if device is None:
hook = getattr(module, "_hf_hook", None)
device = getattr(hook, "execution_device", None)
if device is None:
return None, "no execution device"
try:
arena = ModuleArena(module, device, logger)
except Exception as exc: # noqa: BLE001
return None, f"stable range unavailable ({exc})"
module.__dict__[_ARENA_ATTR] = arena
return arena, None
def arena_of(module: Any) -> Optional[ModuleArena]:
try:
return module.__dict__.get(_ARENA_ATTR)
except Exception: # noqa: BLE001
return None
# ---------------------------------------------------------------------------------------------------------------------
# Block offload gives resident groups a reserved address range, since the stock re-pin after an
# encode lands at new allocator addresses and re-records the step graph each prompt; opt out with
# UNSLOTH_DIFFUSION_GROUP_REPIN_RANGE=0.
REPIN_ENV = "UNSLOTH_DIFFUSION_GROUP_REPIN_RANGE"
_REPIN_ATTR = "_unsloth_repin_range"
_REPIN_WANTED = "_unsloth_repin_wanted"
_REPIN_STATS = "_unsloth_repin_stats"
def repin_disabled() -> bool:
return (os.environ.get(REPIN_ENV) or "").strip().lower() in _OFF
def want_stable_repin(module: Any) -> Optional[str]:
"""Mark ``module`` (a block-offloaded denoiser whose step records a graph) so its group re-pins land in stable
ranges from now on. Returns why not, else None."""
if repin_disabled():
return f"disabled by {REPIN_ENV}=0"
try:
import torch
if not torch.cuda.is_available():
return "no CUDA device"
_driver()
except Exception as exc: # noqa: BLE001
return f"CUDA virtual memory unavailable ({exc})"
try:
module.__dict__[_REPIN_WANTED] = True
module.__dict__.setdefault(
_REPIN_STATS,
{
"groups": 0,
"pins": 0,
"placed": 0,
"stock": 0,
"maps": 0,
"unmaps": 0,
"bytes": 0,
"rebuilds": 0,
},
)
except Exception as exc: # noqa: BLE001
return f"module not writable ({exc})"
return None
def drop_stable_repin(module: Any) -> None:
"""Graphing stopped for ``module``: later re-pins move the stock way. A mapped range is unmapped by the next
release as before; the address ranges themselves go with the module."""
try:
module.__dict__.pop(_REPIN_WANTED, None)
except Exception: # noqa: BLE001
pass
def stable_repin_wanted(module: Any) -> bool:
try:
return bool(module.__dict__.get(_REPIN_WANTED)) and not repin_disabled()
except Exception: # noqa: BLE001
return False
def repin_stats(module: Any) -> Optional[dict]:
try:
stats = module.__dict__.get(_REPIN_STATS)
except Exception: # noqa: BLE001
return None
return dict(stats) if isinstance(stats, dict) else None
def _host_pieces(t: Any) -> Optional[list]:
"""(attr or None, plain tensor) for each storage of a host copy, else None (a type the range cannot hold)."""
if _plain(t):
return [(None, t.data)]
inner = _inner(t)
return None if inner is None else list(inner)
def _span(t: Any) -> int:
if not t.numel():
return 0
return (1 + sum((n - 1) * st for n, st in zip(t.shape, t.stride()))) * t.element_size()
class GroupRange:
"""The reserved device range of one offload group's resident copy: ``layout[i]`` holds the (offset, shape, stride,
dtype) of each storage of the group's i-th tensor, fixed at the first re-pin."""
def __init__(self, layout: dict, nbytes: int, device_index: int) -> None:
self.layout = layout
self.arena = VmmArena(max(nbytes, _ALIGN), device_index)
def referenced(self, tensors: list) -> bool:
if not self.arena.mapped:
return False
lo, hi = self.arena.va, self.arena.va + self.arena.size
for t in tensors:
try:
pieces = _host_pieces(t) or [(None, t)]
if any(x.device.type == "cuda" and lo <= x.data_ptr() < hi for _, x in pieces):
return True
except Exception: # noqa: BLE001 - unreadable: keep the memory
return True
return False
def _close_ranges(ranges: list) -> None:
for rng in ranges:
try:
rng.arena.close_from_finalizer()
except Exception: # noqa: BLE001 - process teardown
pass
def _range_for(owner: Any, group: Any, sources: list, device: Any) -> Optional[GroupRange]:
"""``group``'s range, built on first use from the host copies ``sources`` (index -> tensor) it places."""
offset = 0
layout: dict = {}
for i, src in sources:
pieces = _host_pieces(src)
if pieces is None or any(x.device.type != "cpu" for _, x in pieces):
continue
slots = []
for _attr, x in pieces:
slots.append((offset, tuple(x.shape), tuple(x.stride()), x.dtype))
offset += -(-max(_span(x), 1) // _ALIGN) * _ALIGN
layout[i] = tuple(slots)
held = owner.__dict__.get("_unsloth_repin_ranges")
stats = owner.__dict__.get(_REPIN_STATS)
rng = getattr(group, "__dict__", {}).get(_REPIN_ATTR)
if rng is not None:
if rng.layout == layout or rng.arena.mapped:
# unchanged, or still holding weights (a release that could not move them all out): reused as is
return rng
# Host tensors changed (LoRA, dtype, shape): re-lay out, else the stale range stays mapped beside them.
rng.arena.close()
group.__dict__.pop(_REPIN_ATTR, None)
if held is not None and rng in held:
held.remove(rng)
if isinstance(stats, dict):
stats["rebuilds"] = stats.get("rebuilds", 0) + 1
stats["groups"] -= 1
stats["bytes"] -= rng.arena.size
if not layout:
return None
rng = GroupRange(layout, offset, device.index)
group.__dict__[_REPIN_ATTR] = rng
# The ranges outlive any one group object only as long as the denoiser does.
if held is None:
held = []
owner.__dict__["_unsloth_repin_ranges"] = held
weakref.finalize(owner, _close_ranges, held)
held.append(rng)
if isinstance(stats, dict):
stats["groups"] += 1
stats["bytes"] += rng.arena.size
return rng
def pin_group_in_range(
owner: Any,
group: Any,
tensors: list,
pending: Any,
host_of: Any,
device: Any,
swap_wrapper: Any,
) -> set:
"""Place ``group``'s ``tensors`` for which ``pending`` holds (not on the card yet) into its stable range, copied
from ``host_of(t)`` (queued on the current stream: the caller synchronizes). The range is laid out over the
tensors pending at the first re-pin (a tied tensor another group placed takes no room here); a slot is keyed on
the tensor's position in ``tensors``. Returns the ids placed; the caller moves the rest the stock way."""
import torch
device = torch.device(device)
if device.index is None:
device = torch.device("cuda", torch.cuda.current_device())
todo = [(i, t, host_of(t)) for i, t in enumerate(tensors) if pending(t)]
rng = _range_for(owner, group, [(i, src) for i, _t, src in todo], device)
if rng is None:
return set()
rng.arena.map()
stats = owner.__dict__.get(_REPIN_STATS)
placed: set = set()
undo: list = [] # how to point each tensor installed so far back at its original storage
try:
for i, t, src in todo:
slots = rng.layout.get(i)
pieces = _host_pieces(src)
if (
slots is None
or pieces is None
or len(slots) != len(pieces)
or any(
(tuple(x.shape), tuple(x.stride()), x.dtype) != slot[1:]
for slot, (_attr, x) in zip(slots, pieces)
)
):
continue
views = []
for slot, (_attr, x) in zip(slots, pieces):
view = rng.arena.view(slot[0], slot[1], slot[3], slot[2])
view.copy_(x, non_blocking = True)
views.append(view)
if pieces[0][0] is None:
prev = t.data
t.data = views[0]
undo.append((t, prev, False))
else:
_names, ctx = src.__tensor_flatten__()
rebuilt = type(src).__tensor_unflatten__(
dict(zip([a for a, _ in pieces], views)), ctx, src.shape, src.stride()
)
swap_wrapper(t, rebuilt) # a swap: ``rebuilt`` now holds t's original contents
undo.append((t, rebuilt, True))
placed.add(id(t))
except BaseException:
# Leave the range empty so the caller unmaps it and moves the whole group the stock way.
for t, prev, wrapped in reversed(undo):
if wrapped:
swap_wrapper(t, prev)
else:
t.data = prev
raise
if isinstance(stats, dict):
stats["pins"] += 1
stats["maps"] = sum(r.arena.maps for r in owner.__dict__.get("_unsloth_repin_ranges", ()))
stats["placed"] += len(placed)
stats["stock"] += len(todo) - len(placed)
return placed
def release_group_range(owner: Any, group: Any, tensors: list) -> bool:
"""After ``group`` streamed again (its tensors back on their host copies): unmap its range. False when a tensor
still lives in it (the memory then stays until the next re-pin reuses it)."""
rng = getattr(group, "__dict__", {}).get(_REPIN_ATTR)
if rng is None or not rng.arena.mapped:
return True
if rng.referenced(tensors):
return False
rng.arena.unmap()
stats = owner.__dict__.get(_REPIN_STATS) if owner is not None else None
if isinstance(stats, dict):
stats["unmaps"] += 1
return True