Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: pack up-front group-offload pins into slabs (#15105)

Daniel Han84d7833daf76
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 7f164b4ca42..73be7760f2d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -14888,6 +14888,7 @@ def _offload_controlnet_module(cn_model: Any, device: str, logger: Any) -> bool:
import torch
from diffusers.hooks import apply_group_offloading
+ from .diffusion_memory import _apply_group_offloading_packed
from .host_pinning import host_pinning_allowed
onload = torch.device(device)
@@ -14914,7 +14915,7 @@ def _offload_controlnet_module(cn_model: Any, device: str, logger: Any) -> bool:
install_group_offload_hooks_eager()
kwargs["non_blocking"] = True
kwargs["record_stream"] = True
- apply_group_offloading(cn_model, **kwargs)
+ _apply_group_offloading_packed(apply_group_offloading, cn_model, kwargs, logger)
if prefetch:
if not _pin_top_level_group(cn_model, logger):
_skip_top_level_copy_back(cn_model, logger)
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 8684ce7ec23..eae23158103 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2162,7 +2162,7 @@ def _torchao_group_offload_kwargs(
if forced in ("0", "off", "false", "no") or _pinned_memory_capped()
else _pin_budget_mib()
)
- need = _module_host_mib(module)
+ need = _module_pin_mib(module)
already = pinned_mib[0] if pinned_mib else 0
if budget is not None and already + need <= budget:
if pinned_mib is not None:
@@ -4665,7 +4665,7 @@ def hook_resident_denoiser(
onload = torch.device(device)
if onload.type != "cuda":
return False
- dit_mib = _module_host_mib(transformer)
+ dit_mib = _module_pin_mib(transformer)
# the load's _torchao_stream_pinnable rule: unpinned, the apply copies the whole transformer to pageable host RAM
if not _streamed_pin_plan(dit_mib, 0)[0]:
if logger is not None:
@@ -4696,8 +4696,11 @@ def hook_resident_denoiser(
if "low_cpu_mem_usage" in params:
gkwargs["low_cpu_mem_usage"] = False
pinned_mib = [0]
- apply_group_offloading(
- transformer, **_torchao_group_offload_kwargs(transformer, gkwargs, pinned_mib)
+ _apply_group_offloading_packed(
+ apply_group_offloading,
+ transformer,
+ _torchao_group_offload_kwargs(transformer, gkwargs, pinned_mib),
+ logger,
)
if not _pin_top_level_group(transformer, logger, pinned_mib):
_skip_top_level_copy_back(transformer, logger)
@@ -5358,6 +5361,71 @@ def _module_host_mib(module: Any) -> int:
return 0
+def _unique_tensors(module: Any) -> list:
+ seen: set[int] = set()
+ out = []
+ for tensor in list(module.parameters(recurse = True)) + list(module.buffers(recurse = True)):
+ if id(tensor) not in seen:
+ seen.add(id(tensor))
+ out.append(tensor)
+ return out
+
+
+def _module_pin_mib(module: Any) -> int:
+ """Pinned host MiB of an up-front pin of ``module``: slab packing when the arena is on, else per-tensor rounding."""
+ from .diffusion_pinned_arena import arena_reserved_bytes, pin_arena_enabled
+
+ if not pin_arena_enabled():
+ return _module_host_mib(module)
+ try:
+ return -(-arena_reserved_bytes(_unique_tensors(module)) // (1024 * 1024))
+ except Exception: # noqa: BLE001 - priced as the per-tensor pin
+ return _module_host_mib(module)
+
+
+def _apply_group_offloading_packed(
+ apply: Callable[..., Any],
+ module: Any,
+ kwargs: dict[str, Any],
+ logger: Any = None,
+) -> Any:
+ """``apply(module, **kwargs)`` with an up-front pin (stream, ``low_cpu_mem_usage=False``) packed into slabs.
+
+ diffusers pins each tensor through torch's host allocator, which rounds it up to a power of two: a 23.8 GB bf16
+ Flux DiT held 37.2 GB pinned. Slabs hold the same tensors in ~weight bytes; the H2D copies read the same pinned
+ pages, so overlap is unchanged. Lazy (``low_cpu_mem_usage``) and unstreamed applies run as before."""
+ from .diffusion_pinned_arena import packable_bytes, pinned_arena_for_group_offload
+
+ pack = (
+ bool(kwargs.get("use_stream"))
+ and not kwargs.get("low_cpu_mem_usage", False)
+ and host_pinning_allowed()
+ )
+ expected = None
+ if pack:
+ try:
+ expected = packable_bytes(_unique_tensors(module))
+ except Exception: # noqa: BLE001 - whole slabs, as without a size
+ expected = None
+ with pinned_arena_for_group_offload(
+ enabled = None if pack else False, expected_bytes = expected
+ ) as arena:
+ out = apply(module, **kwargs)
+ if arena is not None and arena.payload_bytes:
+ try:
+ module._unsloth_pin_arena_bytes = (arena.payload_bytes, arena.reserved_bytes)
+ except Exception: # noqa: BLE001 - only a diagnostic
+ pass
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: %s pinned in %.2f GB of slabs (%.2f GB of weights)",
+ type(module).__name__,
+ arena.reserved_bytes / 1e9,
+ arena.payload_bytes / 1e9,
+ )
+ return out
+
+
def _storage_nbytes(tensor: Any, depth: int = 0) -> list[int]:
"""Bytes per allocation; torchao subclasses report their logical bf16 size, so size the packed inner tensors."""
flatten = getattr(type(tensor), "__tensor_flatten__", None)
@@ -6294,14 +6362,25 @@ def _apply_group_offload(
gkwargs["non_blocking"] = True
if "record_stream" in _params:
gkwargs["record_stream"] = True
+ # background_pin: applied unpinned; a _GroupPinner pins it once the load commits (start_background_pins).
+ if background_pin is None:
+ background_pin = bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))
+ defer = (
+ background_pin
+ and use_stream
+ and "low_cpu_mem_usage" in _params
+ and _background_pin_enabled()
+ )
+ # an up-front pin packs into slabs (_apply_group_offloading_packed); a deferred one pins per tensor
+ pin_mib = _module_host_mib if defer else _module_pin_mib
pin_streamed = (True, True)
if stream_text_encoders and "low_cpu_mem_usage" in _params:
# The streamed path PINS every offloaded parameter in host RAM when a copy stream is in use (diffusers
# group_offloading `_init_cpu_param_dict`), which is a fine trade when group offload was already the plan.
# Pin only what host RAM covers (unbounded pinning hurt #8188); unpinned re-pins on every onload.
pin_streamed = _streamed_pin_plan(
- sum(_module_host_mib(m) for m in streamed.values()),
- sum(_module_host_mib(m) for m in streamed_encoders.values()),
+ sum(pin_mib(m) for m in streamed.values()),
+ sum(pin_mib(m) for m in streamed_encoders.values()),
logger,
)
gkwargs["low_cpu_mem_usage"] = not pin_streamed[0]
@@ -6309,16 +6388,6 @@ def _apply_group_offload(
# pinning would copy the small-host route's memory-mapped bytes back into host RAM
pin_streamed = (False, False)
gkwargs["low_cpu_mem_usage"] = True
- # ``background_pin``: a module the plan pins is applied unpinned and handed to a _GroupPinner, which the caller
- # starts with start_background_pins once the load has committed.
- if background_pin is None:
- background_pin = bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))
- defer = (
- background_pin
- and use_stream
- and "low_cpu_mem_usage" in _params
- and _background_pin_enabled()
- )
if defer:
install_group_pin_wait()
# Place the smaller components resident BEFORE attaching the transformer group-offload hooks: a companion .to()
@@ -6331,7 +6400,7 @@ def _apply_group_offload(
comp.to(onload)
# Encoders the plan pins count against the same budget as any torchao denoiser pinned below.
pinned_mib = [
- sum(_module_host_mib(m) for m in streamed_encoders.values())
+ sum(pin_mib(m) for m in streamed_encoders.values())
if stream_text_encoders and pin_streamed[1]
else 0
]
@@ -6345,8 +6414,11 @@ def _apply_group_offload(
apply_group_offloading(module, **{**gkwargs, "low_cpu_mem_usage": True})
_defer_pinning(pipe, module, onload, logger)
else:
- apply_group_offloading(
- module, **_torchao_group_offload_kwargs(module, gkwargs, pinned_mib)
+ _apply_group_offloading_packed(
+ apply_group_offloading,
+ module,
+ _torchao_group_offload_kwargs(module, gkwargs, pinned_mib),
+ logger,
)
installed += 1
if use_stream and not _pin_top_level_group(module, logger, pinned_mib):
@@ -6380,7 +6452,7 @@ def _apply_group_offload(
apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
_defer_pinning(pipe, module, onload, logger)
else:
- apply_group_offloading(module, **ekwargs)
+ _apply_group_offloading_packed(apply_group_offloading, module, ekwargs, logger)
installed += 1
_pin_vision_embedding_device(module)
if te_room > 0:
@@ -6496,14 +6568,14 @@ def stream_text_encoders_onto(
if str(n).startswith("text_encoder") and isinstance(m, torch.nn.Module)
}
if "low_cpu_mem_usage" in params:
- pin = _streamed_pin_plan(0, sum(_module_host_mib(m) for m in encoders.values()), logger)
+ pin = _streamed_pin_plan(0, sum(_module_pin_mib(m) for m in encoders.values()), logger)
kwargs["low_cpu_mem_usage"] = not pin[1]
done: list[str] = []
try:
for name, module in encoders.items():
module.to("cpu")
done.append(name)
- apply_group_offloading(module, **kwargs)
+ _apply_group_offloading_packed(apply_group_offloading, module, kwargs, logger)
_pin_vision_embedding_device(module)
except Exception:
for name in done:
@@ -7068,24 +7140,24 @@ def _apply_streaming_offload(
# Overlap needs a pinned host copy and record_stream (record_stream=False syncs the compute stream per group).
prefetch = use_stream and _streaming_prefetch_enabled()
pin_dits, pin_encoders = False, False
- if prefetch:
- pin_dits, pin_encoders = _streamed_pin_plan(
- sum(_module_host_mib(m) for m, t in streamed.values() if t == "block_level"),
- sum(_module_host_mib(m) for m, t in streamed.values() if t != "block_level"),
- logger,
- )
defer = (
prefetch
and bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))
and _background_pin_enabled()
)
+ # an up-front pin packs into slabs (_apply_group_offloading_packed); a deferred one pins per tensor
+ pin_mib = _module_host_mib if defer else _module_pin_mib
+ if prefetch:
+ pin_dits, pin_encoders = _streamed_pin_plan(
+ sum(pin_mib(m) for m, t in streamed.values() if t == "block_level"),
+ sum(pin_mib(m) for m, t in streamed.values() if t != "block_level"),
+ logger,
+ )
if defer:
install_group_pin_wait()
# pinned encoders count against the budget a torchao denoiser pins within, as on the group tier
if pin_encoders:
- pinned_mib[0] = sum(
- _module_host_mib(m) for m, t in streamed.values() if t != "block_level"
- )
+ pinned_mib[0] = sum(pin_mib(m) for m, t in streamed.values() if t != "block_level")
for module, offload_type in streamed.values():
kwargs: dict[str, Any] = {
@@ -7105,7 +7177,7 @@ def _apply_streaming_offload(
if use_stream and "low_cpu_mem_usage" in params:
kwargs["low_cpu_mem_usage"] = not (pin and not defer)
kwargs = _torchao_group_offload_kwargs(module, kwargs, pinned_mib)
- apply_group_offloading(module, **kwargs)
+ _apply_group_offloading_packed(apply_group_offloading, module, kwargs, logger)
if pin and defer and kwargs.get("use_stream") and kwargs.get("low_cpu_mem_usage"):
_defer_pinning(pipe, module, onload, logger)
installed += 1
diff --git a/studio/backend/core/inference/diffusion_pinned_arena.py b/studio/backend/core/inference/diffusion_pinned_arena.py
index 0a32e849762..0c9bd117bef 100644
--- a/studio/backend/core/inference/diffusion_pinned_arena.py
+++ b/studio/backend/core/inference/diffusion_pinned_arena.py
@@ -6,6 +6,9 @@
torch's host allocator rounds each pin up to a power of two (H3's 19.45 GB int8 denoiser held 33.8 GB pinned), and the
pageable source survived until each group's first onload. The slab copy replaces the parameter's storage on the spot.
+Used by every up-front group-offload pin (image streaming / group / resident tiers, streamed encoders, prequant, H3):
+a 23.8 GB bf16 Flux DiT held 37.2 GB pinned per tensor, 24.3 GB packed.
+
Kill switch: ``UNSLOTH_DIFFUSION_PIN_ARENA=0`` restores per-tensor ``pin_memory()``.
"""
@@ -13,7 +16,7 @@ from __future__ import annotations
import contextlib
import os
-from typing import Any, Iterator, Optional
+from typing import Any, Callable, Iterator, Optional
PIN_ARENA_ENV = "UNSLOTH_DIFFUSION_PIN_ARENA"
DEFAULT_SLAB_BYTES = 1 << 30
@@ -39,12 +42,50 @@ def _pow2_ceil(n: int) -> int:
return 1 << max(0, (int(n) - 1).bit_length())
+def _slab_size(nbytes: int, slab_bytes: int, remaining: Optional[int]) -> int:
+ """A new slab for an ``nbytes`` tensor. With ``remaining`` (bytes still to place, this tensor included) known, the
+ tail slabs halve with what is left (largest power of two not above it) instead of a whole slab per small module."""
+ size = slab_bytes
+ if remaining is not None and remaining < slab_bytes:
+ size = 1 << (max(int(remaining), 1).bit_length() - 1)
+ return max(size, _pow2_ceil(nbytes))
+
+
+def _first_fit(
+ entries: list[list[Any]], nbytes: int, capacity: Callable[[Any], int]
+) -> Optional[tuple[int, int]]:
+ """(slab index, offset) of ``nbytes`` in the first ``[slab, used]`` entry with room, marked taken; else None."""
+ for index, entry in enumerate(entries):
+ start = (entry[1] + _ALIGN - 1) // _ALIGN * _ALIGN
+ if start + nbytes <= capacity(entry[0]):
+ entry[1] = start + nbytes
+ return index, start
+ return None
+
+
+def _packable(tensor: Any) -> bool:
+ # Only CPU tensors: group offload's ``_to_cpu`` hands a device-resident one to the stock per-tensor pin.
+ return (
+ tensor.device.type == "cpu"
+ and tensor.is_contiguous()
+ and tensor.numel() > 0
+ and not tensor.dtype.is_complex
+ )
+
+
class PinnedArena:
"""First-fit bump allocator over pinned uint8 slabs. Views keep their slab alive, so dropping the
- last tensor that points into a slab returns it to torch's host cache like any pinned tensor."""
+ last tensor that points into a slab returns it to torch's host cache like any pinned tensor.
- def __init__(self, slab_bytes: int = DEFAULT_SLAB_BYTES) -> None:
+ ``expected_bytes`` (the packable bytes about to be pinned) shrinks the tail slabs to what is left."""
+
+ def __init__(
+ self,
+ slab_bytes: int = DEFAULT_SLAB_BYTES,
+ expected_bytes: Optional[int] = None,
+ ) -> None:
self.slab_bytes = _pow2_ceil(slab_bytes)
+ self.expected_bytes = expected_bytes
self._slabs: list[list[Any]] = [] # [slab tensor, used bytes]
self.payload_bytes = 0
@@ -53,26 +94,23 @@ class PinnedArena:
return sum(int(s.numel()) for s, _ in self._slabs)
def _take(self, nbytes: int) -> Any:
- for entry in self._slabs:
- slab, used = entry
- start = (used + _ALIGN - 1) // _ALIGN * _ALIGN
- if start + nbytes <= slab.numel():
- entry[1] = start + nbytes
- return slab[start : start + nbytes]
- size = max(self.slab_bytes, _pow2_ceil(nbytes))
- slab = _alloc_slab(size)
+ hit = _first_fit(self._slabs, nbytes, lambda slab: int(slab.numel()))
+ if hit is not None:
+ slab, start = self._slabs[hit[0]][0], hit[1]
+ return slab[start : start + nbytes]
+ remaining = (
+ None
+ if self.expected_bytes is None
+ else max(nbytes, self.expected_bytes - self.payload_bytes)
+ )
+ slab = _alloc_slab(_slab_size(nbytes, self.slab_bytes, remaining))
self._slabs.append([slab, nbytes])
return slab[:nbytes]
def pin_plain(self, tensor: Any) -> Any:
"""A pinned copy of a plain CPU tensor, as a view into a slab. Falls back to ``pin_memory()``
for anything the packing cannot represent exactly."""
- if (
- tensor.device.type != "cpu"
- or not tensor.is_contiguous()
- or tensor.numel() == 0
- or tensor.dtype.is_complex
- ):
+ if tensor.device.type != "cpu" or not _packable(tensor):
return tensor.pin_memory()
nbytes = tensor.numel() * tensor.element_size()
dst = self._take(nbytes).view(tensor.dtype).view(tensor.shape)
@@ -92,6 +130,50 @@ class PinnedArena:
return self.pin_plain(tensor)
+def _plain_leaves(tensor: Any, depth: int = 0) -> list:
+ """The plain tensors ``PinnedArena.pin`` would copy for ``tensor`` (inner tensors of a wrapper subclass)."""
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
+
+ if depth < 4 and is_traceable_wrapper_subclass(tensor):
+ names, _ = tensor.__tensor_flatten__()
+ return [leaf for name in names for leaf in _plain_leaves(getattr(tensor, name), depth + 1)]
+ return [tensor]
+
+
+def packable_bytes(tensors: Any) -> int:
+ """Bytes of ``tensors`` the arena packs (the rest goes to ``pin_memory()``): its ``expected_bytes``."""
+ return sum(
+ int(leaf.numel()) * int(leaf.element_size())
+ for tensor in tensors
+ for leaf in _plain_leaves(tensor)
+ if _packable(leaf)
+ )
+
+
+def arena_reserved_bytes(tensors: Any, slab_bytes: int = DEFAULT_SLAB_BYTES) -> int:
+ """Pinned bytes an arena sized by ``packable_bytes(tensors)`` reserves for them: its packing replayed in order, and
+ a tensor it hands back to ``pin_memory()`` priced at the host allocator's power-of-two rounding."""
+ tensors = list(tensors)
+ slab_bytes = _pow2_ceil(slab_bytes)
+ expected = packable_bytes(tensors)
+ slabs: list[list[int]] = [] # [capacity, used]
+ loose = placed = 0
+ for tensor in tensors:
+ for leaf in _plain_leaves(tensor):
+ nbytes = int(leaf.numel()) * int(leaf.element_size())
+ if nbytes <= 0:
+ continue
+ if not _packable(leaf):
+ loose += _pow2_ceil(nbytes)
+ continue
+ if _first_fit(slabs, nbytes, int) is None:
+ slabs.append(
+ [_slab_size(nbytes, slab_bytes, max(nbytes, expected - placed)), nbytes]
+ )
+ placed += nbytes
+ return loose + sum(cap for cap, _ in slabs)
+
+
def _repoint(param: Any, pinned: Any) -> None:
"""Make ``param`` read the pinned copy now, so its pageable storage is freed during setup.
Group offloading's own offload does exactly this after every forward."""
@@ -106,7 +188,10 @@ def _repoint(param: Any, pinned: Any) -> None:
@contextlib.contextmanager
def pinned_arena_for_group_offload(
- *, enabled: Optional[bool] = None, slab_bytes: int = DEFAULT_SLAB_BYTES
+ *,
+ enabled: Optional[bool] = None,
+ slab_bytes: int = DEFAULT_SLAB_BYTES,
+ expected_bytes: Optional[int] = None,
) -> Iterator[Optional[PinnedArena]]:
"""While active, diffusers' group offloading pins its up-front host copies through one arena.
@@ -128,7 +213,7 @@ def pinned_arena_for_group_offload(
if original is None:
yield None
return
- arena = PinnedArena(slab_bytes)
+ arena = PinnedArena(slab_bytes, expected_bytes)
def _to_cpu(tensor: Any, low_cpu_mem_usage: bool) -> Any:
if (
diff --git a/studio/backend/tests/test_diffusion_group_offload_pin_arena.py b/studio/backend/tests/test_diffusion_group_offload_pin_arena.py
new file mode 100644
index 00000000000..5a56eccdb38
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_group_offload_pin_arena.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Image / encoder group-offload tiers pin up front through the slab arena, and the pin plan prices the slabs."""
+
+import inspect
+
+import pytest
+import torch
+
+from core.inference import diffusion_memory as dm
+from core.inference import diffusion_pinned_arena as arena_mod
+
+
+def unpinned_slabs(monkeypatch):
+ requests = []
+
+ def alloc(nbytes):
+ requests.append(nbytes)
+ return torch.empty(nbytes, dtype = torch.uint8)
+
+ monkeypatch.setattr(arena_mod, "_alloc_slab", alloc)
+ return requests
+
+
+def _weights():
+ # Flux-like shapes: per-tensor pinning rounds 3072x12288 bf16 (72 MiB) up to 128 MiB
+ return [
+ torch.zeros(3072, 12288, dtype = torch.bfloat16),
+ torch.zeros(12288, 3072, dtype = torch.bfloat16),
+ torch.zeros(3072, dtype = torch.bfloat16),
+ torch.zeros(9216, 3072, dtype = torch.bfloat16),
+ torch.zeros(7, dtype = torch.float32),
+ ]
+
+
+def test_estimate_matches_the_arena_it_replays(unpinned_slabs):
+ weights = _weights() * 3
+ payload = sum(w.numel() * w.element_size() for w in weights)
+ assert arena_mod.packable_bytes(weights) == payload
+ arena = arena_mod.PinnedArena(slab_bytes = 1 << 28, expected_bytes = payload)
+ for w in weights:
+ arena.pin(w)
+ assert arena_mod.arena_reserved_bytes(weights, slab_bytes = 1 << 28) == arena.reserved_bytes
+ per_tensor = sum(arena_mod._pow2_ceil(w.numel() * w.element_size()) for w in weights)
+ assert arena.reserved_bytes < per_tensor
+ assert arena.reserved_bytes < payload * 1.15
+ assert all(n & (n - 1) == 0 for n in unpinned_slabs) # the host allocator rounds no slab
+
+
+def test_a_small_module_does_not_reserve_a_whole_slab(unpinned_slabs):
+ weights = [torch.zeros(1000, 1000) for _ in range(9)] # 36 MB, per-tensor 4 MiB each
+ payload = arena_mod.packable_bytes(weights)
+ arena = arena_mod.PinnedArena(expected_bytes = payload)
+ for w in weights:
+ arena.pin(w)
+ assert payload <= arena.reserved_bytes <= payload * 1.25
+ assert arena.reserved_bytes == arena_mod.arena_reserved_bytes(weights)
+ # without a size hint the whole first slab is taken, as before
+ assert (
+ arena_mod.PinnedArena()._take(16).untyped_storage().nbytes() == arena_mod.DEFAULT_SLAB_BYTES
+ )
+
+
+def test_estimate_prices_unpackable_tensors_per_tensor():
+ t = torch.zeros(300, 500).t() # not contiguous: the arena hands it to pin_memory()
+ assert arena_mod.arena_reserved_bytes([t]) == arena_mod._pow2_ceil(t.numel() * 4)
+ assert arena_mod.arena_reserved_bytes([torch.zeros(0)]) == 0
+
+
+def test_device_resident_tensors_are_priced_per_tensor():
+ # _to_cpu hands a tensor that is not on the CPU (a resident denoiser) to the stock per-tensor pin
+ t = torch.empty(300, 500, device = "meta")
+ assert arena_mod.packable_bytes([t]) == 0
+ assert arena_mod.arena_reserved_bytes([t]) == arena_mod._pow2_ceil(t.numel() * 4)
+
+
+def test_module_pin_mib_follows_the_kill_switch(monkeypatch):
+ net = torch.nn.Sequential(*(torch.nn.Linear(3072, 3000, bias = False) for _ in range(8)))
+ per_tensor = dm._module_host_mib(net)
+ packed = dm._module_pin_mib(net)
+ payload_mib = sum(p.numel() * p.element_size() for p in net.parameters()) / 2**20
+ assert payload_mib <= packed < per_tensor
+ monkeypatch.setenv(arena_mod.PIN_ARENA_ENV, "0")
+ assert dm._module_pin_mib(net) == per_tensor
+
+
+def _recording_apply(calls):
+ from diffusers.hooks import group_offloading as go
+ def apply(module, **kwargs):
+ calls.append(kwargs)
+ module._copies = {
+ p: go.ModuleGroup._to_cpu(p, kwargs.get("low_cpu_mem_usage", False))
+ for p in module.parameters()
+ }
+
+ return apply
+
+
+ "kwargs, packed",
+ [
+ ({"use_stream": True, "low_cpu_mem_usage": False}, True),
+ ({"use_stream": True}, True), # diffusers' default pins up front
+ ({"use_stream": True, "low_cpu_mem_usage": True}, False),
+ ({"use_stream": False, "low_cpu_mem_usage": False}, False),
+ ],
+)
+def test_packed_apply_packs_only_an_up_front_pin(monkeypatch, unpinned_slabs, kwargs, packed):
+ pytest.importorskip("diffusers.hooks.group_offloading")
+ from diffusers.hooks import group_offloading as go
+
+ stock = go.ModuleGroup.__dict__["_to_cpu"]
+ monkeypatch.setattr(dm, "host_pinning_allowed", lambda *a, **k: True)
+ if not packed:
+ # an unpacked apply must reach the stock host copy; stub its pin so no CUDA is needed
+ monkeypatch.setattr(torch.Tensor, "pin_memory", lambda self: self.clone())
+ calls = []
+ net = torch.nn.Linear(64, 32)
+ dm._apply_group_offloading_packed(_recording_apply(calls), net, dict(kwargs))
+ assert calls == [kwargs]
+ assert go.ModuleGroup.__dict__["_to_cpu"] is stock
+ if packed:
+ payload = sum(p.numel() * p.element_size() for p in net.parameters())
+ assert net._unsloth_pin_arena_bytes[0] == payload
+ for p, copy in net._copies.items():
+ assert p.data_ptr() == copy.data_ptr() # re-pointed at the slab view
+ else:
+ assert not hasattr(net, "_unsloth_pin_arena_bytes")
+ assert not unpinned_slabs
+
+
+def test_packed_apply_respects_integrated_gpus(monkeypatch, unpinned_slabs):
+ pytest.importorskip("diffusers.hooks.group_offloading")
+ monkeypatch.setattr(dm, "host_pinning_allowed", lambda *a, **k: False)
+ monkeypatch.setattr(torch.Tensor, "pin_memory", lambda self: self.clone())
+ calls = []
+ net = torch.nn.Linear(16, 16)
+ dm._apply_group_offloading_packed(
+ _recording_apply(calls), net, {"use_stream": True, "low_cpu_mem_usage": False}
+ )
+ assert not unpinned_slabs
+ assert not hasattr(net, "_unsloth_pin_arena_bytes")
+
+
+def test_every_up_front_group_offload_apply_goes_through_the_arena():
+ """The tiers that pin a whole module up front call the packed apply, and size the plan from the slabs."""
+ sources = {
+ name: inspect.getsource(getattr(dm, name))
+ for name in (
+ "_apply_group_offload",
+ "_apply_streaming_offload",
+ "stream_text_encoders_onto",
+ )
+ }
+ for name, src in sources.items():
+ assert "_apply_group_offloading_packed(" in src, name
+ assert "_streamed_pin_plan(" in src, name
+ plan = src[src.index("_streamed_pin_plan(") :][:400]
+ assert "_module_host_mib(m)" not in plan, name
+ from core.inference import diffusion as d
+
+ assert "_apply_group_offloading_packed(" in inspect.getsource(d._offload_controlnet_module)